diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..3b446daad --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +node_modules +dist +.git +.github +__tests__ +tests +src/keboola_mcp_server +3.10.venv +3.12.venv +.venv +.tox +.mypy_cache +.pytest_cache +*.venv +coverage.xml +.coverage +test-results.xml +feature_spec +docs +*.md +!README.md +.env +.mcp.json +uv.lock +pyproject.toml diff --git a/.github/actions/export-kbc-projects/action.yml b/.github/actions/export-kbc-projects/action.yml new file mode 100644 index 000000000..e00a7cc4d --- /dev/null +++ b/.github/actions/export-kbc-projects/action.yml @@ -0,0 +1,30 @@ +name: 'Export kbc projects with secrets' +description: 'Generates projects.json from .github/ci/projects.json, substituting TEST_KBC_PROJECT_* secrets' +inputs: + secrets: + description: workflow secrets (pass `toJSON(secrets)`) + required: true +outputs: + projects-file: + description: Absolute path to the generated projects.json + value: ${{ steps.gen.outputs.projects-file }} +runs: + using: 'composite' + steps: + - name: Generate projects.json file, replace secrets + id: gen + shell: bash + run: | + # Export every TEST_KBC_PROJECT_* secret as an env var, then envsubst the template. + jqQuery='to_entries[] | select(.key | startswith("TEST_KBC_PROJECT_")) | "\(.key)=\(.value|tostring) "' + export $( + (jq -r -j "$jqQuery") << EndOfSecrets + ${{ inputs.secrets }} + EndOfSecrets + ) + + envsubst < .github/ci/projects.json > projects.json + + # Stable mtime so test caching is not invalidated by regeneration. + touch -d '1970-01-01T00:00:01' projects.json + echo "projects-file=$(pwd)/projects.json" >> "$GITHUB_OUTPUT" diff --git a/.github/ci/projects.json b/.github/ci/projects.json new file mode 100644 index 000000000..92811b6b2 --- /dev/null +++ b/.github/ci/projects.json @@ -0,0 +1,37 @@ +[ + { + "host": "connection.europe-west3.gcp.keboola.com", + "project": 3053, + "stagingStorage": "gcs", + "backend": "snowflake", + "token": "$TEST_KBC_PROJECT_3053_TOKEN" + }, + { + "host": "connection.europe-west3.gcp.keboola.com", + "project": 3054, + "stagingStorage": "gcs", + "backend": "snowflake", + "token": "$TEST_KBC_PROJECT_3054_TOKEN" + }, + { + "host": "connection.europe-west3.gcp.keboola.com", + "project": 3056, + "stagingStorage": "gcs", + "backend": "bigquery", + "token": "$TEST_KBC_PROJECT_3056_TOKEN" + }, + { + "host": "connection.europe-west3.gcp.keboola.com", + "project": 3057, + "stagingStorage": "gcs", + "backend": "bigquery", + "token": "$TEST_KBC_PROJECT_3057_TOKEN" + }, + { + "host": "connection.europe-west3.gcp.keboola.com", + "project": 3055, + "stagingStorage": "gcs", + "backend": "snowflake", + "token": "$TEST_KBC_PROJECT_3055_TOKEN" + } +] diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e5f99620b..580003ea7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,13 +4,11 @@ # Cooldown (supply-chain hardening): Dependabot will not open an update PR for a # version until it has been published for at least `default-days` days. Most # malicious package releases are detected and yanked within this window, so we -# avoid pulling a freshly published (potentially compromised) version. This -# mirrors the `exclude-newer = "7 days"` setting in pyproject.toml [tool.uv], -# which guards manual `uv lock` regeneration. +# avoid pulling a freshly published (potentially compromised) version. version: 2 updates: - # Python dependencies managed via uv / pyproject.toml + uv.lock - - package-ecosystem: "uv" + # npm dependencies managed via package.json + package-lock.json + - package-ecosystem: "npm" directory: "/" schedule: interval: "weekly" diff --git a/.github/scripts/kaibench-cost-guardrails.py b/.github/scripts/kaibench-cost-guardrails.py deleted file mode 100644 index 5f50574b7..000000000 --- a/.github/scripts/kaibench-cost-guardrails.py +++ /dev/null @@ -1,42 +0,0 @@ -import json -from pathlib import Path - -MAX_TOKENS_PER_QUESTION = 100000 -MAX_TOTAL_TOKENS = 2000000 - -run_dirs = sorted(Path('results').glob('run_*'), key=lambda p: p.stat().st_mtime) -if not run_dirs: - print("::notice::No results/run_* directories found; skipping token usage checks.") - raise SystemExit(0) -latest = run_dirs[-1] -results = [] -results_file = latest / 'results.jsonl' -if results_file.exists(): - for line in results_file.read_text().splitlines(): - if line.strip(): - try: - results.append(json.loads(line)) - except json.JSONDecodeError: - continue - -evaluated = [r for r in results if r.get('status') != 'skipped'] -total_tokens = sum(r.get('trace', {}).get('total_tokens', 0) or 0 for r in evaluated) -violations = [] - -if total_tokens == 0: - print("::notice::Token usage not available — KaiClient may need updating") -else: - if total_tokens > MAX_TOTAL_TOKENS: - violations.append(f"Total tokens ({total_tokens:,}) exceeds limit ({MAX_TOTAL_TOKENS:,})") - - for r in evaluated: - qid = r.get('question_id', '?') - tokens = r.get('trace', {}).get('total_tokens', 0) or 0 - if tokens > MAX_TOKENS_PER_QUESTION: - violations.append(f"Q{qid}: {tokens:,} tokens exceeds per-question limit ({MAX_TOKENS_PER_QUESTION:,})") - - if violations: - for v in violations: - print(f"::warning::{v}") - else: - print(f"::notice::Token usage OK — {total_tokens:,} total tokens across {len(evaluated)} questions") diff --git a/.github/scripts/kaibench-parse-results.py b/.github/scripts/kaibench-parse-results.py deleted file mode 100644 index e6ea4382a..000000000 --- a/.github/scripts/kaibench-parse-results.py +++ /dev/null @@ -1,53 +0,0 @@ -import json -from pathlib import Path - -run_dirs = sorted(Path('results').glob('run_*'), key=lambda p: p.stat().st_mtime) -if not run_dirs: - raise SystemExit(0) -latest = run_dirs[-1] -if not (latest / 'summary.json').exists(): - raise SystemExit(0) -s = json.loads((latest / 'summary.json').read_text()) -m = s['metrics'] - -# Load results.jsonl to compute partial count (m dict never contains 'partial' key) -results = [] -results_file = latest / 'results.jsonl' -if results_file.exists(): - for line in results_file.read_text().splitlines(): - if line.strip(): - try: - results.append(json.loads(line)) - except json.JSONDecodeError: - continue -evaluated = [r for r in results if r.get('status') != 'skipped'] -partial_count = sum(1 for r in evaluated if r.get('status') == 'partial') - -print(f"passed={m['passed']}") -print(f"failed={m['failed']}") -print(f"total={m['total_questions']}") -print(f"pass_rate={m['overall_pass_rate']:.2f}") -print(f"duration={m['duration_seconds']:.0f}") -status = 'passed' if m['failed'] == 0 and m.get('errors', 0) == 0 and partial_count == 0 else 'failed' -print(f"status={status}") - -# Count regressions vs previous run (downloaded into prev-results/) -regressions = 0 -prev_runs = sorted(Path('prev-results').glob('run_*'), key=lambda p: p.stat().st_mtime) if Path('prev-results').exists() else [] -if prev_runs: - prev_file = prev_runs[-1] / 'results.jsonl' - if prev_file.exists() and results_file.exists(): - prev_by_qid = {} - for line in prev_file.read_text().splitlines(): - if line.strip(): - try: - pr = json.loads(line) - except json.JSONDecodeError: - continue - prev_by_qid[str(pr.get('question_id', ''))] = pr - for r in evaluated: - qid = str(r.get('question_id', '')) - if qid in prev_by_qid: - if prev_by_qid[qid].get('status') == 'passed' and r.get('status') not in ('passed', 'skipped'): - regressions += 1 -print(f"regressions={regressions}") diff --git a/.github/scripts/kaibench-step-summary.py b/.github/scripts/kaibench-step-summary.py deleted file mode 100644 index 3a31c89a5..000000000 --- a/.github/scripts/kaibench-step-summary.py +++ /dev/null @@ -1,346 +0,0 @@ -import json, os -from pathlib import Path - -runs = sorted(Path('results').glob('run_*'), key=lambda p: p.stat().st_mtime) -if not runs: - raise SystemExit(0) -latest = runs[-1] -if not (latest / 'summary.json').exists(): - raise SystemExit(0) -s = json.loads((latest / 'summary.json').read_text()) -m = s['metrics'] -mcp_sha = os.environ.get('GITHUB_SHA', 'N/A')[:12] - -# Load per-question results for tool usage stats -results = [] -results_file = latest / 'results.jsonl' -if results_file.exists(): - for line in results_file.read_text().splitlines(): - if line.strip(): - try: - results.append(json.loads(line)) - except json.JSONDecodeError: - continue - -evaluated = [r for r in results if r.get('status') != 'skipped'] - -# Load previous run for regression comparison (downloaded into prev-results/) -prev_by_qid = {} -prev_summary = None -prev_runs = sorted(Path('prev-results').glob('run_*'), key=lambda p: p.stat().st_mtime) if Path('prev-results').exists() else [] -if prev_runs: - prev_run = prev_runs[-1] - prev_file = prev_run / 'results.jsonl' - prev_summary_file = prev_run / 'summary.json' - if prev_file.exists(): - for line in prev_file.read_text().splitlines(): - if line.strip(): - try: - pr = json.loads(line) - except json.JSONDecodeError: - continue - prev_by_qid[str(pr.get('question_id', ''))] = pr - if prev_summary_file.exists(): - prev_summary = json.loads(prev_summary_file.read_text()) - -# Aggregate tool stats -all_tool_calls = [] -for r in evaluated: - all_tool_calls.extend(r.get('trace', {}).get('tool_calls', [])) -total_tools = len(all_tool_calls) -avg_tools = total_tools / len(evaluated) if evaluated else 0 -avg_duration = sum(r.get('duration_ms', 0) for r in evaluated) / len(evaluated) / 1000 if evaluated else 0 -total_tokens = sum(r.get('trace', {}).get('total_tokens', 0) or 0 for r in evaluated) -avg_tokens = total_tokens / len(evaluated) if evaluated else 0 - -# Tool call metrics -tool_errors = [tc for tc in all_tool_calls if tc.get('is_error')] -tool_approvals = [tc for tc in all_tool_calls if tc.get('approval_required')] -tool_denied = [tc for tc in all_tool_calls if tc.get('approval_required') and not tc.get('was_approved')] -tool_success_rate = (total_tools - len(tool_errors)) / total_tools if total_tools else 0 - -# Tool name frequency -from collections import Counter -tool_freq = Counter(tc.get('tool_name', 'unknown') for tc in all_tool_calls) - -# Stream health -stream_errors = [r for r in evaluated if r.get('trace', {}).get('stream_error')] -early_terminations = [r for r in evaluated if r.get('trace', {}).get('stream_terminated_early')] -orphaned_questions = [r for r in evaluated if r.get('trace', {}).get('orphaned_tool_calls')] - -print('## KaiBench Evaluation Results') -print() -print(f'MCP server commit: `{mcp_sha}`') -print() -print('| Metric | Value |') -print('|--------|-------|') -print(f'| Run ID | `{s["run_id"]}` |') -print(f'| Duration | {m["duration_seconds"]:.0f}s |') -print(f'| **Total** | **{m["total_questions"]}** |') -print(f'| Passed | {m["passed"]} |') -print(f'| Failed | {m["failed"]} |') -print(f'| Skipped | {m.get("skipped", 0)} |') -print(f'| Errors | {m.get("errors", 0)} |') -print(f'| **Pass Rate** | **{m["overall_pass_rate"]:.1%}** |') -avg_score = m.get('average_score') -if avg_score is not None: - print(f'| Avg Score | {avg_score:.2f} |') -partial_count = sum(1 for r in evaluated if r.get('status') == 'partial') -if partial_count: - print(f'| Partial | {partial_count} |') -print(f'| Total Tool Calls | {total_tools} |') -print(f'| Avg Tool Calls/Question | {avg_tools:.1f} |') -print(f'| Avg Duration/Question | {avg_duration:.0f}s |') -print(f'| Total Tokens | {total_tokens:,} |') -print(f'| Avg Tokens/Question | {avg_tokens:,.0f} |') - -# Tool call metrics section -if all_tool_calls: - print() - print('### Tool Call Metrics') - print() - print('| Metric | Value |') - print('|--------|-------|') - print(f'| Tool Success Rate | {tool_success_rate:.1%} ({total_tools - len(tool_errors)}/{total_tools}) |') - if tool_errors: - print(f'| Tool Errors | {len(tool_errors)} |') - if tool_approvals: - print(f'| Approvals Required | {len(tool_approvals)} |') - if tool_denied: - print(f'| Approvals Denied | {len(tool_denied)} |') - print() - print('
Tool usage breakdown') - print() - print('| Tool | Calls | Errors |') - print('|------|-------|--------|') - error_by_tool = Counter(tc.get('tool_name', 'unknown') for tc in tool_errors) - for tool_name, count in tool_freq.most_common(20): - errs = error_by_tool.get(tool_name, 0) - err_str = str(errs) if errs else '-' - print(f'| `{tool_name}` | {count} | {err_str} |') - print() - print('
') - -# Stream health section -if stream_errors or early_terminations or orphaned_questions: - print() - print('### Stream Health') - print() - if stream_errors: - print(f':warning: **{len(stream_errors)} question(s) had stream errors**') - for r in stream_errors: - trace = r.get('trace', {}) - qid = r.get('question_id', '?') - err = trace.get('stream_error', 'unknown') - code = trace.get('stream_error_code', '') - code_str = f' (code: {code})' if code else '' - print(f'- Q{qid}: {err}{code_str}') - print() - if orphaned_questions: - print(f':warning: **{len(orphaned_questions)} question(s) had orphaned tool calls** (started but no output)') - for r in orphaned_questions: - trace = r.get('trace', {}) - qid = r.get('question_id', '?') - orphaned = trace.get('orphaned_tool_calls', []) - print(f'- Q{qid}: {len(orphaned)} orphaned call(s)') - print() - early_only = [r for r in early_terminations if r not in stream_errors and r not in orphaned_questions] - if early_only: - print(f':warning: **{len(early_only)} question(s) had early stream termination**') - for r in early_only: - print(f'- Q{r.get("question_id", "?")}') - print() - -# Regression comparison -if prev_by_qid or prev_summary is not None: - print() - print('### Regression Comparison') - print() - print(f'Previous run: `{prev_runs[-1].name}`') - print() - if prev_summary is not None: - pm = prev_summary['metrics'] - prev_rate = pm.get('overall_pass_rate', 0) - curr_rate = m['overall_pass_rate'] - delta = curr_rate - prev_rate - arrow = ':arrow_up:' if delta > 0 else ':arrow_down:' if delta < 0 else ':left_right_arrow:' - print('| Metric | Previous | Current | Delta |') - print('|--------|----------|---------|-------|') - print(f'| Overall Pass Rate | {prev_rate:.1%} | {curr_rate:.1%} | {arrow} {delta:+.1%} |') - prev_passed = pm.get('passed', 0) - curr_passed = m['passed'] - print(f'| Passed | {prev_passed} | {curr_passed} | {curr_passed - prev_passed:+d} |') - - # Per-type comparison - prev_by_type = {t['question_type']: t for t in prev_summary.get('by_question_type', [])} - for t in s.get('by_question_type', []): - qt = t['question_type'] - if qt in prev_by_type: - pt = prev_by_type[qt] - pr = pt.get('pass_rate', 0) - cr = t.get('pass_rate', 0) - td = cr - pr - ta = ':arrow_up:' if td > 0 else ':arrow_down:' if td < 0 else ':left_right_arrow:' - print(f'| {qt} | {pr:.1%} | {cr:.1%} | {ta} {td:+.1%} |') - else: - print('_Previous run artifacts are incomplete (no `summary.json`) — aggregate comparison unavailable._') - print() - - # Per-question regressions and improvements - regressions = [] - improvements = [] - if prev_by_qid: - for r in evaluated: - qid = str(r.get('question_id', '')) - if qid in prev_by_qid: - prev_st = prev_by_qid[qid].get('status', '?') - curr_st = r.get('status', '?') - if prev_st == 'passed' and curr_st not in ('passed', 'skipped'): - regressions.append((qid, r.get('question_type', ''), prev_st, curr_st)) - elif prev_st != 'passed' and curr_st == 'passed': - improvements.append((qid, r.get('question_type', ''), prev_st, curr_st)) - - if regressions: - print() - print(f':rotating_light: **{len(regressions)} Regression(s)**') - print() - print('| Q | Type | Previous | Current |') - print('|---|------|----------|---------|') - for qid, qt, ps, cs in regressions: - print(f'| {qid} | {qt} | {ps} | {cs} |') - - if improvements: - print() - print(f':tada: **{len(improvements)} Improvement(s)**') - print() - print('| Q | Type | Previous | Current |') - print('|---|------|----------|---------|') - for qid, qt, ps, cs in improvements: - print(f'| {qid} | {qt} | {ps} | {cs} |') - -# Errors & failures detail -errors_and_failures = [r for r in evaluated if r.get('status') in ('error', 'failed', 'partial')] -if errors_and_failures: - print() - print('### Errors & Failures Detail') - print() - by_type = {} - for r in errors_and_failures: - qt = r.get('question_type', 'Unknown') - by_type.setdefault(qt, []).append(r) - for qt, items in sorted(by_type.items()): - print(f'**{qt}**') - print() - for r in items: - qid = r.get('question_id', '?') - status = r.get('status', '?') - emoji = {'failed': ':x:', 'error': ':warning:', 'partial': ':large_orange_diamond:'}.get(status, status) - print(f'
{emoji} Q{qid} ({status})') - print() - if r.get('error_message'): - print(f'**Error:** `{r["error_message"][:200]}`') - print() - expected = str(r.get('expected_answer') or '-')[:200] - extracted = str(r.get('extracted_answer') or '-')[:200] - print(f'**Expected:** {expected}') - print() - print(f'**Extracted:** {extracted}') - notes = r.get('verification', {}).get('notes', '') - if notes: - print() - print(f'**Notes:** {notes[:300]}') - score = r.get('verification', {}).get('score', r.get('score')) - if score is not None: - print() - print(f'**Score:** {score}') - print() - print('
') - print() - -# Per-type breakdown -for t in s.get('by_question_type', []): - print() - print(f'### {t["question_type"]}') - print(f'{t["passed_count"]}/{t["total_count"]} passed ({t.get("pass_rate", 0):.1%})') - if t.get('skipped_count', 0): - print(f'_{t["skipped_count"]} skipped_') - -# Per-question table (only evaluated questions) -if evaluated: - print() - print('### Per-Question Results') - print() - print('| Q | Type | Status | Tools | Tokens | Duration | Expected | Extracted | Notes |') - print('|---|------|--------|-------|--------|----------|----------|-----------|-------|') - def sort_key(x): - qid = str(x.get('question_id', '0')) - try: - return (0, int(qid), '') - except ValueError: - last = qid.split('-')[-1] - if last.isdigit(): - return (0, int(last), '') - return (1, 0, qid) - def _cell(s: str) -> str: - return s.replace('|', r'\|').replace('\n', ' ').replace('\r', ' ') - for r in sorted(evaluated, key=sort_key): - qid = r.get('question_id', '?') - qtype = (r.get('question_type') or '')[:12] - status = r.get('status', '?') - emoji = {'passed': ':white_check_mark:', 'failed': ':x:', 'error': ':warning:', 'partial': ':large_orange_diamond:'}.get(status, status) - tools = len(r.get('trace', {}).get('tool_calls', [])) - tokens = r.get('trace', {}).get('total_tokens', 0) or 0 - tokens_str = f'{tokens:,}' if tokens else '-' - dur = f'{r.get("duration_ms", 0)/1000:.0f}s' - expected = _cell(str(r.get('expected_answer') or '-')[:25]) - extracted = _cell(str(r.get('extracted_answer') or '-')[:25]) - notes = _cell((r.get('verification', {}).get('notes') or '')[:40]) - health = '' - trace = r.get('trace', {}) - if trace.get('stream_error'): - health = ' :boom:' - elif trace.get('orphaned_tool_calls'): - health = ' :grey_question:' - print(f'| {qid} | {qtype} | {emoji}{health} | {tools} | {tokens_str} | {dur} | {expected} | {extracted} | {notes} |') - -# MCP Tool Validation detail section -mcp_results = [r for r in evaluated if r.get('question_type') == 'MCP Tool Validation'] -if mcp_results: - print() - print('### MCP Tool Validation Detail') - print() - for r in sorted(mcp_results, key=lambda x: ( - (0, int(str(x.get('question_id', '')).split('-')[-1])) - if str(x.get('question_id', '')).split('-')[-1].isdigit() - else (1, str(x.get('question_id', ''))) - )): - phase = r.get('question_id', '?') - phase_status = r.get('status', '?') - phase_emoji = {'passed': ':white_check_mark:', 'failed': ':x:', 'error': ':warning:', 'partial': ':large_orange_diamond:'}.get(phase_status, phase_status) - extracted = r.get('extracted_answer') - if isinstance(extracted, dict) and extracted: - pass_ct = sum(1 for t in extracted.values() if t.get('status') == 'PASS') - fail_ct = sum(1 for t in extracted.values() if t.get('status') == 'FAIL') - warn_ct = sum(1 for t in extracted.values() if t.get('status') == 'WARN') - skip_ct = sum(1 for t in extracted.values() if t.get('status') == 'SKIP') - total_ct = len(extracted) - print(f'
{phase_emoji} {phase} — {pass_ct}/{total_ct} passed' - + (f', {warn_ct} warn' if warn_ct else '') - + (f', {fail_ct} fail' if fail_ct else '') - + (f', {skip_ct} skip' if skip_ct else '') - + '') - print() - print('| Test | Tool | Status | Trace | Description |') - print('|------|------|--------|-------|-------------|') - for tid in sorted(extracted.keys(), key=lambda x: (0, int(x.split('-')[1])) if '-' in x and x.split('-')[1].isdigit() else (1, x)): - t = extracted[tid] - st = t.get('status', '?') - st_emoji = {'PASS': ':white_check_mark:', 'FAIL': ':x:', 'WARN': ':warning:', 'SKIP': ':fast_forward:'}.get(st, st) - traced = ':white_check_mark:' if t.get('trace_verified') else ':x:' - desc = _cell((t.get('description') or '')[:60]) - print(f'| {tid} | `{t.get("tool_name", "?")}` | {st_emoji} | {traced} | {desc} |') - print() - print('
') - else: - notes = (r.get('verification', {}).get('notes') or r.get('error_message') or '')[:80] - print(f'- {phase_emoji} **{phase}**: {notes}') diff --git a/.github/scripts/kaibench-verbose-results.py b/.github/scripts/kaibench-verbose-results.py deleted file mode 100644 index cd91abe79..000000000 --- a/.github/scripts/kaibench-verbose-results.py +++ /dev/null @@ -1,329 +0,0 @@ -import json -from pathlib import Path -from collections import Counter - -runs = sorted(Path('results').glob('run_*'), key=lambda p: p.stat().st_mtime) -if not runs: - print("No runs found in results/") - exit(0) -latest = runs[-1] -results_file = latest / 'results.jsonl' -if not results_file.exists(): - print("No results.jsonl found") - exit(0) - -results = [] -for _l in results_file.read_text().splitlines(): - if _l.strip(): - try: - results.append(json.loads(_l)) - except json.JSONDecodeError: - continue -evaluated = [r for r in results if r.get('status') != 'skipped'] -summary_file = latest / 'summary.json' -if not summary_file.exists(): - print("No summary.json found (run may have been interrupted)") - exit(0) -summary = json.loads(summary_file.read_text()) -m = summary['metrics'] - -# Load previous run (downloaded into prev-results/) -prev_by_qid = {} -prev_summary = None -prev_runs = sorted(Path('prev-results').glob('run_*'), key=lambda p: p.stat().st_mtime) if Path('prev-results').exists() else [] -if prev_runs: - prev_run = prev_runs[-1] - prev_file = prev_run / 'results.jsonl' - prev_sum_file = prev_run / 'summary.json' - if prev_file.exists(): - for line in prev_file.read_text().splitlines(): - if line.strip(): - try: - pr = json.loads(line) - except json.JSONDecodeError: - continue - prev_by_qid[str(pr.get('question_id', ''))] = pr - if prev_sum_file.exists(): - prev_summary = json.loads(prev_sum_file.read_text()) - -W = 72 # display width - -# ── Overall Summary ── -print('=' * W) -print(f' KAIBENCH VERBOSE RESULTS — {latest.name}') -print('=' * W) -print() - -passed = m['passed'] -failed = m['failed'] -errors = m.get('errors', 0) -partial = sum(1 for r in evaluated if r.get('status') == 'partial') -total = m['total_questions'] -skipped = m.get('skipped', 0) -avg_score = m.get('average_score', 0) -dur = m.get('duration_seconds', 0) - -# Bar chart for pass/partial/fail/error -bar_w = 40 -if total > 0: - p_w = round(passed / total * bar_w) - pt_w = round(partial / total * bar_w) - f_w = round(failed / total * bar_w) - e_w = bar_w - p_w - pt_w - f_w - bar = '\u2588' * p_w + '\u2593' * pt_w + '\u2591' * f_w + '\u00b7' * max(0, e_w) -else: - bar = '\u00b7' * bar_w -print(f' [{bar}] {passed}/{total} passed ({m["overall_pass_rate"]:.1%})') -legend_parts = [f'\u2588 passed={passed}'] -if partial: legend_parts.append(f'\u2593 partial={partial}') -if failed: legend_parts.append(f'\u2591 failed={failed}') -if errors or skipped: - dot_parts = [] - if errors: dot_parts.append(f'errors={errors}') - if skipped: dot_parts.append(f'skipped={skipped}') - legend_parts.append('\u00b7 ' + ', '.join(dot_parts)) -print(f' {" | ".join(legend_parts)}') -print(f' Avg Score: {avg_score:.2f} | Duration: {dur:.0f}s') -print() - -# ── Aggregate Tool & Token Stats ── -all_tool_calls = [] -for r in evaluated: - all_tool_calls.extend(r.get('trace', {}).get('tool_calls', [])) -total_tools = len(all_tool_calls) -total_tokens = sum(r.get('trace', {}).get('total_tokens', 0) or 0 for r in evaluated) -durations = [r.get('duration_ms', 0) / 1000 for r in evaluated] -tool_counts = [len(r.get('trace', {}).get('tool_calls', [])) for r in evaluated] -token_counts = [r.get('trace', {}).get('total_tokens', 0) or 0 for r in evaluated] -n = len(evaluated) or 1 - -tool_errors = [tc for tc in all_tool_calls if tc.get('is_error')] -tool_approvals = [tc for tc in all_tool_calls if tc.get('approval_required')] -tool_denied = [tc for tc in all_tool_calls if tc.get('approval_required') and not tc.get('was_approved')] - -print('-' * W) -print(' AGGREGATE METRICS') -print('-' * W) -print(f' {"Tool Calls":<24} total={total_tools:<8} avg={total_tools/n:<8.1f} ' - f'min={min(tool_counts) if tool_counts else 0:<6} max={max(tool_counts) if tool_counts else 0}') -print(f' {"Tokens":<24} total={total_tokens:<8,} avg={total_tokens/n:<8,.0f} ' - f'min={min(token_counts) if token_counts else 0:<6,} max={max(token_counts) if token_counts else 0:,}') -print(f' {"Duration (s)":<24} total={sum(durations):<8.0f} avg={sum(durations)/n:<8.0f} ' - f'min={min(durations) if durations else 0:<6.0f} max={max(durations) if durations else 0:.0f}') -if total_tools: - err_rate = len(tool_errors) / total_tools - print(f' {"Tool Success Rate":<24} {(1 - err_rate):.1%} ({total_tools - len(tool_errors)}/{total_tools})' - + (f' | errors={len(tool_errors)}' if tool_errors else '')) -if tool_approvals: - print(f' {"Approvals":<24} required={len(tool_approvals)} denied={len(tool_denied)}') -print() - -# ── Tool Usage Breakdown (top 15) ── -if all_tool_calls: - tool_freq = Counter(tc.get('tool_name', '?') for tc in all_tool_calls) - error_by_tool = Counter(tc.get('tool_name', '?') for tc in tool_errors) - print('-' * W) - print(' TOOL USAGE (top 15)') - print('-' * W) - print(f' {"Tool":<36} {"Calls":>6} {"Errors":>7} {"Err%":>6}') - print(f' {"─" * 36} {"─" * 6} {"─" * 7} {"─" * 6}') - for name, count in tool_freq.most_common(15): - errs = error_by_tool.get(name, 0) - err_pct = f'{errs/count:.0%}' if errs else '-' - print(f' {name:<36} {count:>6} {errs:>7} {err_pct:>6}') - print() - -# ── Stream Health ── -stream_errors = [r for r in evaluated if r.get('trace', {}).get('stream_error')] -early_terms = [r for r in evaluated if r.get('trace', {}).get('stream_terminated_early')] -orphaned = [r for r in evaluated if r.get('trace', {}).get('orphaned_tool_calls')] -if stream_errors or early_terms or orphaned: - print('-' * W) - print(' STREAM HEALTH ISSUES') - print('-' * W) - for r in stream_errors: - trace = r.get('trace', {}) - qid = r.get('question_id', '?') - err = trace.get('stream_error', '?') - code = trace.get('stream_error_code', '') - code_s = f' (code: {code})' if code else '' - print(f' [STREAM_ERR] Q{qid}: {err}{code_s}') - for r in orphaned: - qid = r.get('question_id', '?') - orph = r.get('trace', {}).get('orphaned_tool_calls', []) - print(f' [ORPHANED] Q{qid}: {len(orph)} orphaned tool call(s)') - for r in early_terms: - if r not in stream_errors and r not in orphaned: - print(f' [EARLY_TERM] Q{r.get("question_id", "?")}: stream terminated early') - print() - -# ── By Question Type (with partial, avg score, tool/token stats) ── -print('-' * W) -print(' BY QUESTION TYPE') -print('-' * W) -by_type = {} -for r in evaluated: - qt = r.get('question_type', 'Unknown') - by_type.setdefault(qt, []).append(r) -for qt in sorted(by_type): - items = by_type[qt] - t_passed = sum(1 for r in items if r.get('status') == 'passed') - t_failed = sum(1 for r in items if r.get('status') == 'failed') - t_partial = sum(1 for r in items if r.get('status') == 'partial') - t_error = sum(1 for r in items if r.get('status') == 'error') - t_total = len(items) - t_rate = t_passed / t_total if t_total else 0 - t_scores = [r.get('verification', {}).get('score', r.get('score')) for r in items] - t_scores = [s for s in t_scores if s is not None and isinstance(s, (int, float))] - t_avg_score = sum(t_scores) / len(t_scores) if t_scores else 0 - t_tools = sum(len(r.get('trace', {}).get('tool_calls', [])) for r in items) - t_tokens = sum(r.get('trace', {}).get('total_tokens', 0) or 0 for r in items) - t_dur = sum(r.get('duration_ms', 0) for r in items) / 1000 - print() - bw = 20 - pw = round(t_passed / t_total * bw) if t_total else 0 - ptw = round(t_partial / t_total * bw) if t_total else 0 - fw = bw - pw - ptw - type_bar = '\u2588' * pw + '\u2593' * ptw + '\u2591' * fw - print(f' {qt}') - print(f' [{type_bar}] {t_passed}/{t_total} passed ({t_rate:.0%})' - + (f' + {t_partial} partial' if t_partial else '') - + (f' + {t_error} errors' if t_error else '')) - print(f' avg_score={t_avg_score:.2f} tools={t_tools} ' - f'tokens={t_tokens:,} duration={t_dur:.0f}s') - -print() - -# ── Regression Comparison ── -regressions = [] -improvements = [] -if prev_by_qid or prev_summary: - for r in evaluated: - qid = str(r.get('question_id', '')) - if qid in prev_by_qid: - ps = prev_by_qid[qid].get('status', '?') - cs = r.get('status', '?') - if ps == 'passed' and cs != 'passed': - regressions.append((qid, r.get('question_type', ''), ps, cs)) - elif ps != 'passed' and cs == 'passed': - improvements.append((qid, r.get('question_type', ''), ps, cs)) - if prev_summary or regressions or improvements: - print('-' * W) - print(' REGRESSION COMPARISON') - print('-' * W) - if prev_summary: - pm = prev_summary['metrics'] - prev_rate = pm.get('overall_pass_rate', 0) - curr_rate = m['overall_pass_rate'] - delta = curr_rate - prev_rate - arrow = '\u25b2' if delta > 0 else '\u25bc' if delta < 0 else '=' - print(f' Pass rate: {prev_rate:.1%} -> {curr_rate:.1%} ({arrow} {delta:+.1%})') - if regressions: - print(f' REGRESSIONS ({len(regressions)}):') - for qid, qt, ps, cs in regressions: - print(f' Q{qid} ({qt}): {ps} -> {cs}') - if improvements: - print(f' IMPROVEMENTS ({len(improvements)}):') - for qid, qt, ps, cs in improvements: - print(f' Q{qid} ({qt}): {ps} -> {cs}') - if regressions or improvements: - print() - -# ── Per-Question Detail ── -print('-' * W) -print(' PER-QUESTION DETAIL') -print('-' * W) - -def sort_key(x): - qid = str(x.get('question_id', '0')) - try: - return (0, int(qid), '') - except ValueError: - last = qid.split('-')[-1] - if last.isdigit(): - return (0, int(last), '') - return (1, 0, qid) - -for r in sorted(evaluated, key=sort_key): - qid = str(r.get('question_id', '?')) - status = r.get('status', '?') - qtype = r.get('question_type', '') - score = r.get('verification', {}).get('score', r.get('score', '-')) - trace = r.get('trace', {}) - tc_list = trace.get('tool_calls', []) - tools = len(tc_list) - tokens = trace.get('total_tokens', 0) or 0 - input_tok = trace.get('input_tokens', 0) or 0 - output_tok = trace.get('output_tokens', 0) or 0 - dur_s = r.get('duration_ms', 0) / 1000 - - icon = {'passed': '\u2714', 'failed': '\u2718', 'error': '\u26a0', 'partial': '\u25d2'}.get(status, '?') - - # Regression indicator - reg = '' - if qid in prev_by_qid: - ps = prev_by_qid[qid].get('status', '?') - if ps == 'passed' and status != 'passed': - reg = ' <<< REGRESSED' - elif ps != 'passed' and status == 'passed': - reg = ' >>> IMPROVED' - - print() - print(f' {icon} Q{qid} [{status.upper()}] — {qtype}{reg}') - print(f' score={score} tools={tools} tokens={tokens:,}' - + (f' (in={input_tok:,} out={output_tok:,})' if input_tok or output_tok else '') - + f' duration={dur_s:.0f}s') - - # Stream health - if trace.get('stream_error'): - err = trace.get('stream_error', '?') - code = trace.get('stream_error_code', '') - print(f' STREAM ERROR: {err}' + (f' (code: {code})' if code else '')) - if trace.get('orphaned_tool_calls'): - print(f' ORPHANED TOOL CALLS: {len(trace["orphaned_tool_calls"])}') - - # Tool call summary for this question - if tc_list: - tc_freq = Counter(tc.get('tool_name', '?') for tc in tc_list) - tc_errs = [tc for tc in tc_list if tc.get('is_error')] - tool_summary = ', '.join(f'{n}x{c}' if c > 1 else n for n, c in tc_freq.most_common(8)) - if len(tc_freq) > 8: - tool_summary += f' (+{len(tc_freq) - 8} more)' - print(f' tools: {tool_summary}') - if tc_errs: - err_names = Counter(tc.get('tool_name', '?') for tc in tc_errs) - print(f' tool errors: {", ".join(f"{n}({c})" for n, c in err_names.most_common())}') - - # Answer detail for non-passed - if status == 'error': - err = r.get('error_message', '') - if err: - print(f' ERROR: {err[:300]}') - elif status in ('failed', 'partial'): - expected = str(r.get('expected_answer', ''))[:150] - extracted = str(r.get('extracted_answer', ''))[:150] - notes = r.get('verification', {}).get('notes', '')[:200] - print(f' expected: {expected}') - print(f' extracted: {extracted}') - if notes: - print(f' notes: {notes}') - - # MCP Tool Validation sub-tests - if qtype == 'MCP Tool Validation': - extracted = r.get('extracted_answer') - if isinstance(extracted, dict) and extracted: - sub_pass = sum(1 for t in extracted.values() if t.get('status') == 'PASS') - sub_fail = sum(1 for t in extracted.values() if t.get('status') == 'FAIL') - sub_warn = sum(1 for t in extracted.values() if t.get('status') == 'WARN') - sub_skip = sum(1 for t in extracted.values() if t.get('status') == 'SKIP') - print(f' sub-tests: {sub_pass} pass, {sub_fail} fail, ' - f'{sub_warn} warn, {sub_skip} skip / {len(extracted)} total') - for tid in sorted(extracted.keys(), key=lambda x: (0, int(x.split('-')[1])) if '-' in x and x.split('-')[1].isdigit() else (1, x)): - t = extracted[tid] - st = t.get('status', '?') - if st != 'PASS': - desc = (t.get('description') or '')[:80] - print(f' [{st}] {tid} ({t.get("tool_name", "?")}) — {desc}') - -print() -print('=' * W) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5c166510..30d3dc307 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,183 +2,161 @@ name: CI # How the CI Pipeline Works # - The CI workflow runs on all push events and pull requests. -# - For pull requests from forks (coming from a different repository), we skip jobs/actions that need secrets (e.g., -# publishing results, integration tests) to avoid exposing them. Only safe checks like local tests and flake8 are run. -# - For pull requests from branches in this repository, the workflow is skipped to avoid running it twice (once for the -# push and once for the PR). -# - The full workflow (including jobs requiring secrets) runs only on pushes events to branches in this repository -on: [ push, pull_request ] +# - For pull requests from forks (coming from a different repository), we still run the full +# build/test/lint matrix — none of those steps need secrets. +# - For pull requests from branches in this repository, the workflow is skipped to avoid running +# it twice (once for the push and once for the PR). +# - npm publish + Anthropic registration run only on semantic version tags (vX.Y.Z) pushed to +# this repository. +on: [push, pull_request] concurrency: ci-${{ github.ref }} permissions: contents: read - checks: write jobs: build: name: Build, test and package - # run this job only for push (avoiding pull requests from the same repo) or for pull requests from different repo + # run this job only for push (avoiding pull requests from the same repo) or for pull requests from a different repo if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + node-version: ['22', '24'] outputs: is_semantic_tag: ${{ steps.get_package_version.outputs.is_semantic_tag }} tag: ${{ steps.get_package_version.outputs.tag }} version: ${{ steps.get_package_version.outputs.version }} - wheel_artifact_id: ${{ steps.wheel_artifact_upload.outputs.artifact-id }} steps: - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + - name: Set up Node ${{ matrix.node-version }} + uses: actions/setup-node@v4 with: - python-version: ${{ matrix.python-version }} + node-version: ${{ matrix.node-version }} + cache: npm - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install uv - uv sync --frozen --no-editable --extra dev - - # Runs tests with coverage and generates coverage.xml file and test-results.xml file - # and checks flake8 code-style formatting that is compatible with isort & black - # and verifies TOOLS.md is up-to-date with tool definitions - # see setup in pyproject.toml - - name: Unit tests, code style and documentation check - run: | - uv run tox - - - name: Publish test results - uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0 - # don't fail the job if the Checks API call flakes — the tests themselves already ran - continue-on-error: true - # run this step even if a previous step failed, but only for push events or pull requests from the same repo - if: (success() || failure()) && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) - with: - name: Test results (${{ matrix.python-version }}) - path: ./test-results.xml - reporter: 'java-junit' - - - name: Upload coverage to Codecov - # run this step only for push events or pull requests from the same repo - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository - uses: codecov/codecov-action@v7.0.0 - with: - file: ./coverage.xml - fail_ci_if_error: false # Codecov upload is best-effort — transient service outages should not block CI - token: ${{ secrets.CODECOV_TOKEN }} + run: npm ci - - name: Build wheels package - run: | - uv build --wheel --no-sources + - name: Type-check + run: npm run type-check + + - name: Lint and format check + run: npm run lint + + - name: Unit tests + run: npm test + + - name: Verify TOOLS.md is up to date + run: npm run check:tools-docs + + - name: Build package + run: npm run build + env: + SKIP_ENV_VALIDATION: '1' - name: Get package version id: get_package_version - if: matrix.python-version == '3.10' + if: matrix.node-version == '22' run: | - VERSION=`uv run python3 -c 'import importlib.metadata; print(importlib.metadata.version("keboola_mcp_server"))'` + VERSION=$(node -p "require('./package.json').version") TAG="${GITHUB_REF##*/}" IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false) echo "Version = '$VERSION', Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'" - echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT - echo "tag=$TAG" >> $GITHUB_OUTPUT - echo "version=${VERSION}" >> $GITHUB_OUTPUT - - - name: Upload wheel package - id: wheel_artifact_upload - # run this step only for push events or pull requests from the same repo - if: matrix.python-version == '3.10' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) - uses: actions/upload-artifact@v7 - with: - name: keboola_mcp_server-${{ steps.get_package_version.outputs.version }}-py3-none-any.whl - path: dist/keboola_mcp_server-${{ steps.get_package_version.outputs.version }}-py3-none-any.whl - if-no-files-found: error - compression-level: 0 # wheels are ZIP archives - retention-days: 7 + echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> "$GITHUB_OUTPUT" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" integration_tests: - name: Integration Tests - needs: build - # run this job only for push events (not pull requests) + name: Integration tests + # Runs in parallel with `build` (independent job) so the fast unit lane isn't blocked. + # Push-only: PRs from forks have no access to the project-pool secrets. if: github.event_name == 'push' runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.11", "3.12"] - # This ensures tests run sequentially, not in parallel - max-parallel: 1 steps: - uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + - name: Set up Node + uses: actions/setup-node@v4 with: - python-version: ${{ matrix.python-version }} + node-version: '22' + cache: npm - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install uv - uv sync --frozen --no-editable --extra dev + run: npm ci - - name: Integration tests - # run actual tests only for pushes to the same repository (not forks) - if: github.repository == github.event.repository.full_name - env: - INTEGTEST_STORAGE_TOKENS: ${{ secrets.INTEGTEST_STORAGE_TOKENS }} - INTEGTEST_POOL_STORAGE_API_URL: ${{ vars.INTEGTEST_POOL_STORAGE_API_URL }} - INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES: ${{ secrets.INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES }} - run: | - uv run tox -e integtests + # Postgres + pgvector backs the docs-search index (docs_query / find_component_id). + # The docs integ test seeds it with the deterministic stub embedder and reads it back. + - name: Start pgvector with docker compose + run: docker compose up -d --wait pgvector - - name: Skip integration tests for forks - # show a message when tests are skipped for forks - if: github.repository != github.event.repository.full_name - run: | - echo "Integration tests skipped for fork repository" - echo "Dependencies installed successfully - setup is working correctly" - - - name: Publish integration test results - uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0 - # don't fail the job if the Checks API call flakes — the tests themselves already ran - continue-on-error: true - # publish results only when actual tests were run (same repository) - if: (always()) && github.repository == github.event.repository.full_name + # Unwrap the project pool: substitute $TEST_KBC_PROJECT_*_TOKEN secrets into + # .github/ci/projects.json -> ./projects.json (mirrors keboola/go-monorepo). + - name: Export Keboola test projects + uses: ./.github/actions/export-kbc-projects with: - name: Integration test results (${{ matrix.python-version }}) - path: ./integtest-results.xml - reporter: 'java-junit' + secrets: ${{ toJSON(secrets) }} - deploy_to_pypi: - name: Deploy to pypi.org + - name: Integration tests + env: + # Cross-runner project lease via the shared redis (same key scheme as go-monorepo, + # so leases coordinate with other consumers of the pool). + TEST_MCP_PROJECTS_LOCK_HOST: ${{ vars.TEST_MCP_PROJECTS_LOCK_HOST }} + TEST_MCP_PROJECTS_LOCK_PASSWORD: ${{ secrets.TEST_MCP_PROJECTS_LOCK_PASSWORD }} + # Path to the pool file, taken from the repo/org variable you maintain centrally + # (same convention as keboola/go-monorepo). The export step above writes the file + # of this name at the workspace root. + TEST_KBC_PROJECTS_FILE: ${{ github.workspace }}/${{ vars.TEST_KBC_PROJECTS_FILE }} + # Docs-search index: the docker-compose pgvector above + the deterministic offline + # embedder. Enables docs_query / find_component_id for the docs integ test; the + # rest of the suite is unaffected. + DATABASE_URL: postgres://mcp:mcp@localhost:5432/docs + DOCS_EMBEDDER_MODEL: stub + DOCS_EMBEDDER_DIM: '3072' + run: npm run test:integ + + publish_npm: + name: Publish to npm needs: - build - - integration_tests runs-on: ubuntu-latest + # Publish only on a semantic vX.Y.Z tag that matches package.json's version. if: | startsWith(github.ref, 'refs/tags/') && needs.build.outputs.is_semantic_tag == 'true' && needs.build.outputs.tag == format('v{0}', needs.build.outputs.version) + permissions: + contents: read + id-token: write # npm provenance steps: - - name: Download wheel package - uses: actions/download-artifact@v8 - with: - name: keboola_mcp_server-${{ needs.build.outputs.version }}-py3-none-any.whl - path: dist/ + - uses: actions/checkout@v6 - - name: Publish package - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b + - name: Set up Node + uses: actions/setup-node@v4 with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + node-version: '22' + cache: npm + registry-url: 'https://registry.npmjs.org' + + - name: Install dependencies + run: npm ci + + - name: Build package + run: npm run build + env: + SKIP_ENV_VALIDATION: '1' + + - name: Publish + run: npm publish --provenance --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} register-with-anthropic: name: Register with Anthropic needs: - build - - deploy_to_pypi + - publish_npm runs-on: ubuntu-latest if: | startsWith(github.ref, 'refs/tags/') && diff --git a/.github/workflows/openwiki-update.yml b/.github/workflows/openwiki-update.yml new file mode 100644 index 000000000..cc7139368 --- /dev/null +++ b/.github/workflows/openwiki-update.yml @@ -0,0 +1,50 @@ +name: OpenWiki Update + +on: + workflow_dispatch: + schedule: + - cron: "0 8 * * *" + +permissions: + contents: write + pull-requests: write + +jobs: + update: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Install OpenWiki + run: npm install --global openwiki + + - name: Run OpenWiki + run: openwiki code --update --print + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENWIKI_MODEL_ID: z-ai/glm-5.2 + LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} + LANGCHAIN_PROJECT: openwiki + LANGCHAIN_TRACING_V2: "true" + + - name: Create OpenWiki update pull request + uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7 + with: + add-paths: | + openwiki + AGENTS.md + CLAUDE.md + .github/workflows/openwiki-update.yml + branch: openwiki/update + commit-message: "docs: update OpenWiki" + title: "docs: update OpenWiki" + body: | + Automated OpenWiki documentation update. + + This PR was generated by the scheduled OpenWiki workflow. diff --git a/.gitignore b/.gitignore index d0145cacd..17a92cdbc 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,12 @@ venv # Environment variables .env +# Local integration-test project pool (contains real Storage API tokens) — never commit. +# The committed template with $TEST_KBC_PROJECT_*_TOKEN placeholders lives at +# .github/ci/projects.json (anchored ignores below don't match that path). +/projects.json +/projects.local.json + # Test related files .coverage .coverage.* @@ -32,4 +38,9 @@ integtest-results.xml test-results.xml .tox/ local_testing -.mcp.json \ No newline at end of file +.mcp.json +# Node / TypeScript (MCP server rewrite) +node_modules/ +dist/ +*.tsbuildinfo +.DS_Store diff --git a/.oxfmtrc.json b/.oxfmtrc.json new file mode 100644 index 000000000..97799c1f3 --- /dev/null +++ b/.oxfmtrc.json @@ -0,0 +1,15 @@ +{ + "printWidth": 100, + "tabWidth": 2, + "singleQuote": true, + "arrowParens": "always", + "jsxSingleQuote": false, + "bracketSameLine": false, + "sortPackageJson": false, + "ignorePatterns": [ + "dist", + "**/CHANGELOG.md", + "src/resources", + "src/tools/storage-schema.json" + ] +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..8c941b6d2 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,9 @@ + + +## OpenWiki + +This repository uses OpenWiki for recurring code documentation. Start with `openwiki/quickstart.md`, then follow its links to architecture, workflows, domain concepts, operations, integrations, testing guidance, and source maps. + +The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. + + diff --git a/CLAUDE.md b/CLAUDE.md index 5485b8277..ef0f77247 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -155,3 +155,13 @@ server always reflecting your latest code changes: ## Security Considerations - When whitelisting domains in OAuth, prefer **explicit domain lists over regex patterns** - Regex could unintentionally allow future domains that weren't reviewed (principle of least privilege) + + + +## OpenWiki + +This repository uses OpenWiki for recurring code documentation. Start with `openwiki/quickstart.md`, then follow its links to architecture, workflows, domain concepts, operations, integrations, testing guidance, and source maps. + +The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. + + diff --git a/Dockerfile b/Dockerfile index 38fb9f01b..b363bc14a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,67 @@ -# Use a Python image with uv pre-installed -FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS uv +# syntax=docker/dockerfile:1 +# +# Node multi-stage build for @keboola/mcp-server (replaces the Python/uv image). +# Build-time inputs are ARGs; runtime configuration is ENV — kept strictly +# separate so a build never needs runtime secrets and the running container is +# configured only at `docker run` / k8s. -# Install the project into /app +# Pin a specific Node 22 patch rather than a floating tag (matches kai-agent's +# stance) so a bad upstream release can't silently change the runtime. +FROM node:22.20.0-slim AS base WORKDIR /app +ENV NODE_ENV=production -# Enable bytecode compilation -ENV UV_COMPILE_BYTECODE=1 +# ---------------------------------------------------------------------------- +# Stage: deps — install ALL deps (incl. dev) from the lockfile for the build. +# ---------------------------------------------------------------------------- +FROM base AS deps +COPY package.json package-lock.json ./ +RUN --mount=type=cache,target=/root/.npm npm ci --include=dev -# Copy from the cache instead of linking since it's a mounted volume -ENV UV_LINK_MODE=copy +# ---------------------------------------------------------------------------- +# Stage: builder — compile TypeScript to dist/. SKIP_ENV_VALIDATION keeps the +# build from requiring any runtime env var (build-time vs run-time separation). +# ---------------------------------------------------------------------------- +FROM base AS builder +# Build-time-only metadata (consumed by the build, not the runtime contract). +ARG APP_VERSION=DEV +ENV SKIP_ENV_VALIDATION=1 +COPY --from=deps /app/node_modules ./node_modules +COPY . . +RUN npm run build -# Install the project's dependencies using the lockfile and settings -COPY uv.lock pyproject.toml README.md /app/ -RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-install-project --no-dev --no-editable - -# Then, add the rest of the project source code and install it -# Installing separately from its dependencies allows optimal layer caching -ADD src /app/src -RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-editable -RUN --mount=type=cache,target=/root/.cache/uv uv pip install ddtrace~=3.0 - -FROM python:3.12-slim-bookworm - -ARG APP_USER_NAME=app -ARG APP_USER_UID=1000 -ARG APP_USER_GID=1000 - -RUN groupadd --gid ${APP_USER_GID} ${APP_USER_NAME} \ - && useradd --uid ${APP_USER_UID} --gid ${APP_USER_GID} ${APP_USER_NAME} +# Reduce node_modules to production-only for the runtime image. +RUN --mount=type=cache,target=/root/.npm npm ci --omit=dev +# ---------------------------------------------------------------------------- +# Stage: runner — slim runtime image. +# ---------------------------------------------------------------------------- +FROM node:22.20.0-slim AS runner +# Tell the DD agent to aggregate multiline logs (stack traces). +LABEL com.datadoghq.ad.logs='[{"auto_multi_line_detection": true}]' WORKDIR /app -ENV LOG_CONFIG=/app/logging-json.conf -COPY --from=uv --chown=${APP_USER_UID}:${APP_USER_GID} /app/.venv /app/.venv -COPY logging-json.conf /app/logging-json.conf +COPY --from=builder /app/dist ./dist +COPY --from=builder /app/node_modules ./node_modules +COPY --from=builder /app/package.json ./package.json -# Place executables in the environment at the front of the path -ENV PATH="/app/.venv/bin:$PATH" +# ---- Runtime configuration (provided at `docker run` / k8s) ---- +ENV NODE_ENV=production +# Bind to all interfaces inside the container; map a port on the host. +ENV HOST=0.0.0.0 +ENV PORT=8000 +ENV LOG_LEVEL=INFO +# Datadog APM: load the tracer before app code. dd-trace reads DD_* at runtime. +ENV NODE_OPTIONS="--import dd-trace/initialize.mjs" +ENV DD_LOGS_INJECTION=true +# Runtime config NOT baked into the image (set per deployment): +# HOSTNAME_SUFFIX, KBC_STORAGE_API_URL, KBC_STORAGE_TOKEN, KBC_BRANCH_ID, +# KBC_WORKSPACE_SCHEMA, KBC_OAUTH_CLIENT_ID/SECRET, KBC_JWT_SECRET, DD_SERVICE/ENV/VERSION. -USER $APP_USER_NAME +# Non-root (uid 1000), matching the previous image. +USER 1000 +EXPOSE 8000 -# when running the container, add KBC_STORAGE_API_URL environment variable and a bind mount to the host's db file -ENTRYPOINT ["python", "-m", "keboola_mcp_server", "--log-level", "DEBUG"] +# Default to the streamable-HTTP server (the deployed mode); stdio is for local +# CLI/npx use. Override args in k8s if needed. +CMD ["node", "dist/index.js", "--transport", "streamable-http"] diff --git a/TOOLS.md b/TOOLS.md index 337f826ea..b45962bad 100644 --- a/TOOLS.md +++ b/TOOLS.md @@ -4,19 +4,16 @@ This document provides details about the tools available in the Keboola MCP serv ## Index ### Component Tools -- [add_config_row](#add_config_row): Creates a component configuration row in the specified configuration_id, using the specified name, -component ID, configuration JSON, and description. +- [add_config_row](#add_config_row): Creates a component configuration row in the specified configuration_id, using the specified name, component ID, configuration JSON, and description. - [create_config](#create_config): Creates a root component configuration using the specified name, component ID, configuration JSON, and description. -- [create_sql_transformation](#create_sql_transformation): Creates an SQL transformation using the specified name, SQL query following the current SQL dialect, a detailed -description, and a list of created table names. +- [create_sql_transformation](#create_sql_transformation): Creates an SQL transformation using the specified name, SQL query following the current SQL dialect, a detailed description, and a list of created table names. - [get_components](#get_components): Retrieves detailed information about one or more components by their IDs. - [get_config_examples](#get_config_examples): Retrieves sample configuration examples for a specific component. - [get_configs](#get_configs): Retrieves component configurations in the project with optional filtering. - [run_sync_action](#run_sync_action): Executes a synchronous action for a component configuration or a component row configuration. - [update_config](#update_config): Updates an existing root component configuration by modifying its parameters, storage mappings, name or description. - [update_config_row](#update_config_row): Updates an existing component configuration row by modifying its parameters, storage mappings, name, or description. -- [update_sql_transformation](#update_sql_transformation): Updates an existing SQL transformation configuration by modifying its SQL code, storage mappings, -name or description. +- [update_sql_transformation](#update_sql_transformation): Updates an existing SQL transformation configuration by modifying its SQL code, storage mappings, name or description. ### Documentation Tools - [docs_query](#docs_query): Answers a question using the Keboola documentation as a source. @@ -38,28 +35,22 @@ name or description. - [create_oauth_url](#create_oauth_url): Generates an OAuth authorization URL for a Keboola component configuration. ### Other Tools -- [create_python_js_data_app_git_credential](#create_python_js_data_app_git_credential): Mints a one-time HTTPS token on a python-js **prod** data app so the caller can clone, pull, -and push to the app's managed git repo over HTTPS. -- [delete_python_js_data_app_draft](#delete_python_js_data_app_draft): Deletes a python-js DRAFT data app — both the data-app instance (DSAPI) and its Storage -configuration. -- [deploy_data_app](#deploy_data_app): Deploys/redeploys a data app or stops a running data app in the Keboola environment asynchronously, given the -action and the configuration ID. -- [get_data_apps](#get_data_apps): Lists summaries of data apps in the project given the limit and offset or gets details of a data apps by -providing their configuration IDs. +- [create_python_js_data_app_git_credential](#create_python_js_data_app_git_credential): Mints a one-time HTTPS token on a python-js **prod** data app so the caller can clone, pull, and push to the app's managed git repo over HTTPS. +- [delete_python_js_data_app_draft](#delete_python_js_data_app_draft): Deletes a python-js DRAFT data app — both the data-app instance (DSAPI) and its Storage configuration. +- [deploy_data_app](#deploy_data_app): Deploys/redeploys a data app or stops a running data app in the Keboola environment asynchronously, given the action and the configuration ID. +- [get_data_apps](#get_data_apps): Lists summaries of data apps in the project given the limit and offset or gets details of a data apps by providing their configuration IDs. - [modify_python_js_data_app](#modify_python_js_data_app): Creates or updates a python-js data app. - [modify_streamlit_data_app](#modify_streamlit_data_app): Creates or updates a Streamlit data app. ### Project Tools -- [get_project_info](#get_project_info): Retrieves structured information about the current project, -including essential context and base instructions for working with it -(e. +- [get_project_info](#get_project_info): Retrieves structured information about the current project, including essential context and base instructions for working with it (e. - [update_project_description](#update_project_description): Updates the description of the current Keboola project. ### SQL Tools - [query_data](#query_data): Executes an SQL SELECT query to get the data from the underlying database. ### Search Tools -- [find_component_id](#find_component_id): Returns list of component IDs that match the given query. +- [find_component_id](#find_component_id): Returns a list of component IDs that match the given natural-language query. - [search](#search): Searches for Keboola items (tables, buckets, components, configurations, transformations, flows, data-apps, etc. ### Semantic Tools @@ -75,7 +66,7 @@ the expected semantic objects provided. lineage references (created/updated by), and links. - [get_tables](#get_tables): Lists tables in buckets or retrieves full details of specific tables, including fully qualified database name, column definitions, lineage references (created/updated by) and links. -- [update_descriptions](#update_descriptions): Updates the description for a Keboola storage item. +- [update_descriptions](#update_descriptions): Updates the description for Keboola storage items (buckets, tables, or columns). --- @@ -88,79 +79,88 @@ column definitions, lineage references (created/updated by) and links. **Description**: -Creates a component configuration row in the specified configuration_id, using the specified name, -component ID, configuration JSON, and description. - -BEFORE CALLING - REQUIRED STEPS: -1. Call `get_components([component_id])` to retrieve the component's `configuration_row_schema`. -2. Read `configuration_row_schema.required` to find ALL mandatory top-level fields. -3. Call `get_config_examples(component_id)` to see real-world row parameter examples. -4. Populate `parameters` with every required field before calling this tool. -Skipping these steps will cause a schema validation error. - -USAGE: -- Use when you want to create a new row configuration for a specific component configuration. - -WHEN NOT TO USE: -- `keboola.orchestrator` / `keboola.flow` → use flows tools -- `keboola.data-apps` → use data applications tools -- `keboola.snowflake-transformation` / `keboola.google-bigquery-transformation` → use SQL transformation tools - -EXAMPLES: -- user_input: `Create a new configuration row for component X with these settings` - - set the component_id, configuration_id and configuration parameters accordingly - - returns the created component configuration if successful. +Creates a component configuration row in the specified configuration_id, using the specified name, component ID, configuration JSON, and description. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "name": { - "description": "A short, descriptive name summarizing the purpose of the component configuration.", - "type": "string" + "type": "string", + "description": "A short, descriptive name summarizing the purpose of the component configuration." }, "description": { - "description": "The detailed description of the component configuration explaining its purpose and functionality.", - "type": "string" + "type": "string", + "description": "The detailed description of the component configuration explaining its purpose and functionality." }, "component_id": { - "description": "The ID of the component for which to create the configuration.", - "type": "string" + "type": "string", + "description": "The ID of the component for which to create the configuration." }, "configuration_id": { - "description": "The ID of the configuration for which to create the configuration row.", - "type": "string" + "type": "string", + "description": "The ID of the configuration for which to create the configuration row." }, "parameters": { - "additionalProperties": true, - "description": "The component row configuration parameters, adhering to the configuration_row_schema", - "type": "object" + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {}, + "description": "The component row configuration parameters, adhering to the configuration_row_schema" }, "storage": { - "additionalProperties": true, - "default": null, "description": "The table and/or file input / output mapping of the component configuration. It is present only for components that have tables or file input mapping defined", - "type": "object" + "anyOf": [ + { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + }, + { + "type": "null" + } + ] }, "processors_before": { - "default": null, - "description": "The list of processors that will run before the configured component row runs.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" + "description": "The list of processors that will run before the configured component runs.", + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + { + "type": "null" + } + ] }, "processors_after": { - "default": null, - "description": "The list of processors that will run after the configured component row runs.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" + "description": "The list of processors that will run after the configured component runs.", + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + { + "type": "null" + } + ] } }, "required": [ @@ -170,7 +170,7 @@ EXAMPLES: "configuration_id", "parameters" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -185,123 +185,125 @@ EXAMPLES: Creates a root component configuration using the specified name, component ID, configuration JSON, and description. -BEFORE CALLING - REQUIRED STEPS: -1. Call `get_components([component_id])` to retrieve the component's `configuration_schema`. -2. Read `configuration_schema.required` to find ALL mandatory top-level fields. -3. Call `get_config_examples(component_id)` to see real-world parameter examples. -4. Populate `parameters` with every required field before calling this tool. -Skipping these steps will cause a schema validation error. - -USAGE: -- Use when you want to create a new root configuration for a specific component. - -WHEN NOT TO USE: -- `keboola.orchestrator` / `keboola.flow` → use flows tools -- `keboola.data-apps` → use data applications tools -- `keboola.snowflake-transformation` / `keboola.google-bigquery-transformation` → use SQL transformation tools - -EXAMPLES: -- user_input: `Create a new configuration for component X with these settings` - - set the component_id and configuration parameters accordingly - - returns the created component configuration if successful. - **Input JSON Schema**: ```json { - "$defs": { - "VariableDefinition": { - "description": "A single variable definition to attach to a configuration.", - "properties": { - "name": { - "description": "Variable name.", - "type": "string" - }, - "type": { - "default": "string", - "description": "Variable type: \"string\" or \"vault\".", - "enum": [ - "string", - "vault" - ], - "type": "string" - }, - "default_value": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional default value bound at creation time." - } - }, - "required": [ - "name" - ], - "type": "object" - } - }, - "additionalProperties": false, + "type": "object", "properties": { "name": { - "description": "A short, descriptive name summarizing the purpose of the component configuration.", - "type": "string" + "type": "string", + "description": "A short, descriptive name summarizing the purpose of the component configuration." }, "description": { - "description": "The detailed description of the component configuration explaining its purpose and functionality.", - "type": "string" + "type": "string", + "description": "The detailed description of the component configuration explaining its purpose and functionality." }, "component_id": { - "description": "The ID of the component for which to create the configuration.", - "type": "string" + "type": "string", + "description": "The ID of the component for which to create the configuration." }, "parameters": { - "additionalProperties": true, - "description": "The component configuration parameters, adhering to the configuration_schema", - "type": "object" + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {}, + "description": "The component configuration parameters, adhering to the configuration_schema" }, "storage": { - "additionalProperties": true, - "default": null, "description": "The table and/or file input / output mapping of the component configuration. It is present only for components that have tables or file input mapping defined", - "type": "object" + "anyOf": [ + { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + }, + { + "type": "null" + } + ] }, "processors_before": { - "default": null, "description": "The list of processors that will run before the configured component runs.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + { + "type": "null" + } + ] }, "processors_after": { - "default": null, "description": "The list of processors that will run after the configured component runs.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + { + "type": "null" + } + ] }, "variables": { + "description": "Variable definitions to attach to this configuration. Each entry specifies a name, type (\"string\" or \"vault\"), and an optional default value. On creation, both `None` (omitted) and `[]` (empty list) mean \"do not attach variables\" — no `keboola.variables` config is created. To remove variables from an existing configuration, use `update_config` with `variables=[]`.", "anyOf": [ { + "type": "array", "items": { - "$ref": "#/$defs/VariableDefinition" - }, - "type": "array" + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Variable name." + }, + "type": { + "default": "string", + "description": "Variable type: \"string\" or \"vault\".", + "type": "string", + "enum": [ + "string", + "vault" + ] + }, + "default_value": { + "description": "Optional default value bound at creation time.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "name" + ] + } }, { "type": "null" } - ], - "default": null, - "description": "Variable definitions to attach to this configuration. Each entry specifies a name, type (\"string\" or \"vault\"), and an optional default value. On creation, both `None` (omitted) and `[]` (empty list) mean \"do not attach variables\" \u2014 no `keboola.variables` config is created. To remove variables from an existing configuration, use `update_config` with `variables=[]`." + ] } }, "required": [ @@ -310,7 +312,7 @@ EXAMPLES: "component_id", "parameters" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -323,119 +325,50 @@ EXAMPLES: **Description**: -Creates an SQL transformation using the specified name, SQL query following the current SQL dialect, a detailed -description, and a list of created table names. - -CONSIDERATIONS: -- By default, SQL transformation must create at least one table to produce a result; omit only if the user - explicitly indicates that no table creation is needed. -- Each SQL code block must include descriptive name that reflects its purpose and group one or more executable - semantically related SQL statements. -- Each SQL query statement within a code block must be executable and follow the current SQL dialect. -- Use delimited identifiers for the current SQL dialect for all identifiers and FQN references. -- When referring to the input tables within the SQL query, use fully qualified table names, which can be - retrieved using appropriate tools. -- When creating a new table within the SQL query (e.g. CREATE TABLE ...): use only the table name with - delimited identifiers, without the fully qualified path; add the plain table name without delimiters - to the `created_table_names` list. -- Unless otherwise specified by user, transformation name and description are generated based on the SQL query - and user intent. -- If there are 20 or more SQL transformations in the project, consider organizing them with a folder: existing - folder names are surfaced in the response's change_summary — use one of them or create a new one. - -USAGE: -- Use when you want to create a new SQL transformation. - -EXAMPLES: -- user_input: `Can you create a new transformation out of this sql query?` - - set the sql_code_blocks to the query, and set other parameters accordingly. - - returns the created SQL transformation configuration if successful. -- user_input: `Generate me an SQL transformation which [USER INTENT]` - - set the sql_code_blocks to the query based on the [USER INTENT], and set other parameters accordingly. - - returns the created SQL transformation configuration if successful. +Creates an SQL transformation using the specified name, SQL query following the current SQL dialect, a detailed description, and a list of created table names. **Input JSON Schema**: ```json { - "$defs": { - "Code": { - "description": "The code block for the transformation block.", - "properties": { - "name": { - "description": "A descriptive name for the code block", - "type": "string" - }, - "script": { - "description": "The SQL script of the code block", - "type": "string" - } - }, - "required": [ - "name", - "script" - ], - "type": "object" - }, - "VariableDefinition": { - "description": "A single variable definition to attach to a configuration.", - "properties": { - "name": { - "description": "Variable name.", - "type": "string" - }, - "type": { - "default": "string", - "description": "Variable type: \"string\" or \"vault\".", - "enum": [ - "string", - "vault" - ], - "type": "string" - }, - "default_value": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional default value bound at creation time." - } - }, - "required": [ - "name" - ], - "type": "object" - } - }, - "additionalProperties": false, + "type": "object", "properties": { "name": { - "description": "A short, descriptive name summarizing the purpose of the SQL transformation.", - "type": "string" + "type": "string", + "description": "A short, descriptive name summarizing the purpose of the SQL transformation." }, "description": { - "description": "The detailed description of the SQL transformation capturing the user intent, explaining the SQL query, and the expected output.", - "type": "string" + "type": "string", + "description": "The detailed description of the SQL transformation capturing the user intent, explaining the SQL query, and the expected output." }, "sql_code_blocks": { - "description": "The SQL query code blocks, each containing a descriptive name and an executable SQL script written in the current SQL dialect. The query will be automatically reformatted to be more readable.", + "type": "array", "items": { - "$ref": "#/$defs/Code" + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "A descriptive name for the code block" + }, + "script": { + "type": "string", + "description": "The SQL script of the code block" + } + }, + "required": [ + "name", + "script" + ] }, - "type": "array" + "description": "The SQL query code blocks, each containing a descriptive name and an executable SQL script written in the current SQL dialect. The query will be automatically reformatted to be more readable." }, "created_table_names": { "default": [], "description": "A list of created table names if they are generated within the SQL query statements (e.g., using `CREATE TABLE ...`).", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } }, "folder": { "default": "", @@ -443,19 +376,47 @@ EXAMPLES: "type": "string" }, "variables": { + "description": "Variable definitions to attach to this transformation. Each entry specifies a name, type (\"string\" or \"vault\"), and an optional default value. On creation, both `None` (omitted) and `[]` (empty list) mean \"do not attach variables\" — no `keboola.variables` config is created. To remove variables from an existing transformation, use `update_sql_transformation` with `variables=[]`.", "anyOf": [ { + "type": "array", "items": { - "$ref": "#/$defs/VariableDefinition" - }, - "type": "array" + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Variable name." + }, + "type": { + "default": "string", + "description": "Variable type: \"string\" or \"vault\".", + "type": "string", + "enum": [ + "string", + "vault" + ] + }, + "default_value": { + "description": "Optional default value bound at creation time.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "name" + ] + } }, { "type": "null" } - ], - "default": null, - "description": "Variable definitions to attach to this transformation. Each entry specifies a name, type (\"string\" or \"vault\"), and an optional default value. On creation, both `None` (omitted) and `[]` (empty list) mean \"do not attach variables\" \u2014 no `keboola.variables` config is created. To remove variables from an existing transformation, use `update_sql_transformation` with `variables=[]`." + ] } }, "required": [ @@ -463,7 +424,7 @@ EXAMPLES: "description", "sql_code_blocks" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -478,44 +439,24 @@ EXAMPLES: Retrieves detailed information about one or more components by their IDs. -RETURNS FOR EACH COMPONENT: -- Component metadata (name, type, description) -- Documentation and usage instructions -- Configuration JSON schema (required for creating/updating configurations) -- Links to component dashboard in Keboola UI - -WHEN TO USE: -- Before creating a new configuration: fetch the component to get its configuration schema -- Before updating a configuration: fetch the component to understand valid configuration options -- When user asks about component capabilities or documentation - -PREREQUISITES: -- You must know the component_id(s). If unknown, first use `find_component_id` or `docs` tool to discover them. - -EXAMPLES: -- User: "Create a generic extractor configuration" - → First call `find_component_id` to get the component_id, then call this tool to get the schema -- User: "What options does the Snowflake writer support?" - → Call this tool with the Snowflake writer component_id to retrieve its documentation and schema - **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "component_ids": { - "description": "IDs of the components", + "type": "array", "items": { "type": "string" }, - "type": "array" + "description": "IDs of the components to retrieve." } }, "required": [ "component_ids" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -530,30 +471,21 @@ EXAMPLES: Retrieves sample configuration examples for a specific component. -USAGE: -- Use before calling `create_config` or `add_config_row` to understand the expected parameters structure. -- Use when you want to see example configurations for a specific component. - -EXAMPLES: -- user_input: `Show me example configurations for component X` - - set the component_id parameter accordingly - - returns a markdown formatted string with configuration examples - **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "component_id": { - "description": "The ID of the component to get configuration examples for.", - "type": "string" + "type": "string", + "description": "The ID of the component to get configuration examples for." } }, "required": [ "component_id" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -568,100 +500,63 @@ EXAMPLES: Retrieves component configurations in the project with optional filtering. -Can list summaries of multiple configurations (grouped by component) or retrieve full details -for specific configurations. - -Returns a list of components, each containing: -- Component metadata (ID, name, type, description) -- Configurations for that component (summaries by default, full details if requested) -- Links to the Keboola UI - -PARAMETER BEHAVIOR: -- If configs is provided (non-empty): Returns FULL details ONLY for those configs. -- Else if component_ids is provided (non-empty): Lists config summaries for those components. -- Else: Lists configs based on component_types (all types if empty). - -WHEN TO USE: -- For listing: Use component_types/component_ids. -- For details: Use configs (can handle multiple). - -WHEN NOT TO USE: -- Do NOT list all configs just to find a configuration by name. Use `search` with - item_types=["configuration", "transformation"] instead. -- Only use broad listing (empty component_types and component_ids) when you need - a complete inventory of all configurations in the project. - -EXAMPLES: -- List all configs (summaries): component_types=[], component_ids=[] -- List extractors (summaries): component_types=["extractor"] -- Get details for specific configs: - configs=[{"component_id": "keboola.ex-db-mysql", "configuration_id": "12345"}] - **Input JSON Schema**: ```json { - "$defs": { - "FullConfigId": { - "description": "Composite configuration ID (component ID + configuration ID).", - "properties": { - "component_id": { - "description": "ID of the component", - "type": "string" - }, - "configuration_id": { - "description": "ID of the configuration", - "type": "string" - } - }, - "required": [ - "component_id", - "configuration_id" - ], - "type": "object" - } - }, - "additionalProperties": false, + "type": "object", "properties": { "component_types": { "default": [], - "description": "Filter by component types. Options: \"application\", \"extractor\", \"transformation\", \"writer\". Empty list [] means ALL component types will be returned. This parameter is IGNORED when configs is provided (non-empty) or component_ids is non-empty.", + "description": "Filter by component types; empty = all. Ignored when configs/component_ids given.", + "type": "array", "items": { + "type": "string", "enum": [ "application", "extractor", "transformation", "writer" - ], - "type": "string" - }, - "type": "array" + ] + } }, "component_ids": { "default": [], - "description": "Filter by specific component IDs (e.g., [\"keboola.ex-db-mysql\", \"keboola.wr-google-sheets\"]). Empty list [] uses component_types filtering instead. When provided (non-empty) and configs is empty, lists summaries for these components. Ignored if configs is provided.", + "description": "Filter by specific component IDs. Ignored when configs is given.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } }, "configs": { "default": [], - "description": "List of specific configurations to retrieve full details for. Each dict must have \"component_id\" (str) and \"configuration_id\" (str). Example: [{\"component_id\": \"keboola.ex-db-mysql\", \"configuration_id\": \"12345\"}]. If provided (non-empty), ignores other filters and returns full details only for these configs, grouped by component. Use this for detailed retrieval.", + "description": "Specific configs to retrieve full details for (grouped by component).", + "type": "array", "items": { - "$ref": "#/$defs/FullConfigId" - }, - "type": "array" + "type": "object", + "properties": { + "component_id": { + "type": "string" + }, + "configuration_id": { + "type": "string" + } + }, + "required": [ + "component_id", + "configuration_id" + ] + } } }, - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` --- ## run_sync_action -**Annotations**: `read-only` +**Annotations**: **Tags**: `components` @@ -669,30 +564,26 @@ EXAMPLES: Executes a synchronous action for a component configuration or a component row configuration. -WHEN TO USE: -- For finding available values of a configuration field -- For validating already configured values (e.g. testing a database connection) -- For listing remote resources such as endpoints, schemas or tables - **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "action_name": { - "description": "The sync action to execute (e.g., \"testConnection\", \"getTables\").", - "type": "string" + "type": "string", + "description": "The sync action to execute (e.g., \"testConnection\", \"getTables\")." }, "component_id": { - "description": "The ID of the component (e.g., \"keboola.ex-db-mysql\").", - "type": "string" + "type": "string", + "description": "The ID of the component (e.g., \"keboola.ex-db-mysql\")." }, "configuration_id": { - "description": "The ID of the configuration to use for the sync action.", - "type": "string" + "type": "string", + "description": "The ID of the configuration to use for the sync action." }, "configuration_row_id": { + "description": "Optional row ID; row parameters/storage are shallow-merged on top of root config.", "anyOf": [ { "type": "string" @@ -700,9 +591,7 @@ WHEN TO USE: { "type": "null" } - ], - "default": null, - "description": "Optional row ID for row-level actions. When provided, the row parameters and storage are shallow-merged on top of root config." + ] } }, "required": [ @@ -710,7 +599,7 @@ WHEN TO USE: "component_id", "configuration_id" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -723,185 +612,25 @@ WHEN TO USE: **Description**: -Updates an existing root component configuration by modifying its parameters, storage mappings, name or description. - -This tool allows PARTIAL parameter updates - you only need to provide the fields you want to change. -All other fields will remain unchanged. -Use this tool when modifying existing configurations; for configuration rows, use update_config_row instead. - -WHEN TO USE: -- Modifying configuration parameters (credentials, settings, API keys, etc.) -- Updating storage mappings (input/output tables or files) -- Changing configuration name or description -- Any combination of the above - -WHEN NOT TO USE: -- `keboola.orchestrator` / `keboola.flow` → use flows tools -- `keboola.data-apps` → use data applications tools -- `keboola.snowflake-transformation` / `keboola.google-bigquery-transformation` → use SQL transformation tools - -PREREQUISITES: -- Configuration must already exist (use create_config for new configurations) -- You must know both component_id and configuration_id -- For parameter updates: Review the component's root_configuration_schema using get_components. -- For storage updates: Ensure mappings are valid for the component type - -IMPORTANT CONSIDERATIONS: -- Parameter updates are PARTIAL - only specify fields you want to change -- parameter_updates supports granular operations: set keys, replace strings, remove keys, or append to lists -- Parameters must conform to the component's root_configuration_schema -- Validate schemas before calling: use get_components to retrieve root_configuration_schema -- For row-based components, this updates the ROOT only (use update_config_row for individual rows) - -WORKFLOW: -1. Retrieve current configuration using get_configs (to understand current state) -2. Identify specific parameters/storage mappings to modify -3. Prepare parameter_updates list with targeted operations -4. Call update_config with only the fields to change +Updates an existing root component configuration by modifying its parameters, storage mappings, name or description. Updates are PARTIAL — only provide the fields you want to change; parameter_updates apply granular diff operations to the existing parameters. **Input JSON Schema**: ```json { - "$defs": { - "ConfigParamListAppend": { - "description": "Append a value to a list parameter.", - "properties": { - "op": { - "const": "list_append", - "type": "string" - }, - "path": { - "description": "JSONPath to the list parameter", - "type": "string" - }, - "value": { - "description": "Value to append to the list" - } - }, - "required": [ - "op", - "path", - "value" - ], - "type": "object" - }, - "ConfigParamRemove": { - "description": "Remove a parameter key.", - "properties": { - "op": { - "const": "remove", - "type": "string" - }, - "path": { - "description": "JSONPath to the parameter key to remove", - "type": "string" - } - }, - "required": [ - "op", - "path" - ], - "type": "object" - }, - "ConfigParamReplace": { - "description": "Replace a substring in a string parameter.", - "properties": { - "op": { - "const": "str_replace", - "type": "string" - }, - "path": { - "description": "JSONPath to the parameter key to modify", - "type": "string" - }, - "search_for": { - "description": "Substring to search for (non-empty)", - "type": "string" - }, - "replace_with": { - "description": "Replacement string (can be empty for deletion)", - "type": "string" - } - }, - "required": [ - "op", - "path", - "search_for", - "replace_with" - ], - "type": "object" - }, - "ConfigParamSet": { - "description": "Set or create a parameter value at the specified path.\n\nUse this operation to:\n- Update an existing parameter value\n- Create a new parameter key\n- Replace a nested parameter value", - "properties": { - "op": { - "const": "set", - "type": "string" - }, - "path": { - "description": "JSONPath to the parameter key to set (e.g., \"api_key\", \"database.host\")", - "type": "string" - }, - "value": { - "description": "New value to set" - } - }, - "required": [ - "op", - "path", - "value" - ], - "type": "object" - }, - "VariableDefinition": { - "description": "A single variable definition to attach to a configuration.", - "properties": { - "name": { - "description": "Variable name.", - "type": "string" - }, - "type": { - "default": "string", - "description": "Variable type: \"string\" or \"vault\".", - "enum": [ - "string", - "vault" - ], - "type": "string" - }, - "default_value": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional default value bound at creation time." - } - }, - "required": [ - "name" - ], - "type": "object" - } - }, - "additionalProperties": false, + "type": "object", "properties": { "change_description": { - "description": "A clear, human-readable summary of what changed in this update. Be specific: e.g., \"Updated API key\", \"Added customers table to input mapping\".", - "type": "string" + "type": "string", + "description": "A clear, human-readable summary of what changed in this update. Be specific: e.g., \"Updated API key\", \"Added customers table to input mapping\"." }, "component_id": { - "description": "The ID of the component the configuration belongs to.", - "type": "string" + "type": "string", + "description": "The ID of the component the configuration belongs to." }, "configuration_id": { - "description": "The ID of the configuration to update.", - "type": "string" + "type": "string", + "description": "The ID of the configuration to update." }, "name": { "default": "", @@ -914,891 +643,462 @@ WORKFLOW: "type": "string" }, "parameter_updates": { - "default": null, "description": "List of granular parameter update operations to apply. Each operation (set, str_replace, remove, list_append) modifies a specific value using JSONPath notation. Only provide if updating parameters - do not use for changing description, storage or processors. Paths are relative to the `parameters` object, not the configuration root (e.g. use `tables`, not `parameters.tables`). Prefer simple JSONPaths (e.g., \"array_param[1]\", \"object_param.key\") and make the smallest possible updates - only change what needs changing. In case you need to replace the whole parameters section, you can use the `set` operation with `$` as path.", - "items": { - "discriminator": { - "mapping": { - "list_append": "#/$defs/ConfigParamListAppend", - "remove": "#/$defs/ConfigParamRemove", - "set": "#/$defs/ConfigParamSet", - "str_replace": "#/$defs/ConfigParamReplace" - }, - "propertyName": "op" - }, - "oneOf": [ - { - "$ref": "#/$defs/ConfigParamSet" - }, - { - "$ref": "#/$defs/ConfigParamReplace" - }, - { - "$ref": "#/$defs/ConfigParamRemove" - }, - { - "$ref": "#/$defs/ConfigParamListAppend" - } - ] - }, - "type": "array" - }, - "storage": { - "additionalProperties": true, - "default": null, - "description": "Complete storage configuration containing input/output table and file mappings. Only provide if updating storage mappings - this replaces the ENTIRE storage configuration. \n\nWhen to use:\n- Adding/removing input or output tables\n- Modifying table/file mappings\n- Updating table destinations or sources\n\nImportant:\n- Not applicable for row-based components (they use row-level storage)\n- Must conform to the Keboola storage schema\n- Replaces ALL existing storage config - include all mappings you want to keep\n- Use get_configs first to see current storage configuration\n- Leave unfilled to preserve existing storage configuration", - "type": "object" - }, - "processors_before": { - "default": null, - "description": "The list of processors that will run before the configured component row runs.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - "processors_after": { - "default": null, - "description": "The list of processors that will run after the configured component row runs.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - "folder": { "anyOf": [ { - "type": "string" + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "set" + }, + "path": { + "type": "string", + "description": "JSONPath to the parameter key to set (e.g., \"api_key\", \"database.host\")" + }, + "value": { + "description": "New value to set" + } + }, + "required": [ + "op", + "path", + "value" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "str_replace" + }, + "path": { + "type": "string", + "description": "JSONPath to the parameter key to modify" + }, + "search_for": { + "type": "string", + "description": "Substring to search for (non-empty)" + }, + "replace_with": { + "type": "string", + "description": "Replacement string (can be empty for deletion)" + } + }, + "required": [ + "op", + "path", + "search_for", + "replace_with" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "remove" + }, + "path": { + "type": "string", + "description": "JSONPath to the parameter key to remove" + } + }, + "required": [ + "op", + "path" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "list_append" + }, + "path": { + "type": "string", + "description": "JSONPath to the list parameter" + }, + "value": { + "description": "Value to append to the list" + } + }, + "required": [ + "op", + "path", + "value" + ] + } + ] + } }, { "type": "null" } - ], - "default": null, - "description": "Folder name to organize this configuration in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more configurations in the project. If there are 20 or more configurations, you should assign one of the existing folders or create a new one that clearly reflects the configuration purpose." + ] }, - "variables": { + "storage": { + "description": "Complete storage configuration containing input/output table and file mappings. Only provide if updating storage mappings - this replaces the ENTIRE storage configuration.", "anyOf": [ { - "items": { - "$ref": "#/$defs/VariableDefinition" + "type": "object", + "propertyNames": { + "type": "string" }, - "type": "array" + "additionalProperties": {} }, { "type": "null" } - ], - "default": null, - "description": "Variable definitions for this configuration. Provide a non-empty list to create or replace all variable definitions. Provide an empty list ([]) to remove all variables. Omit (None) to leave existing variables unchanged." - } - }, - "required": [ - "change_description", - "component_id", - "configuration_id" - ], - "type": "object" -} -``` - ---- - -## update_config_row -**Annotations**: `destructive` - -**Tags**: `components, config-diff-preview` - -**Description**: - -Updates an existing component configuration row by modifying its parameters, storage mappings, name, or description. - -This tool allows PARTIAL parameter updates - you only need to provide the fields you want to change. -All other fields will remain unchanged. -Configuration rows are individual items within a configuration, often representing separate data sources, -tables, or endpoints that share the same component type and parent configuration settings. - -WHEN TO USE: -- Modifying row-specific parameters (table sources, filters, credentials, etc.) -- Updating storage mappings for a specific row (input/output tables or files) -- Changing row name or description -- Any combination of the above - -WHEN NOT TO USE: -- `keboola.orchestrator` / `keboola.flow` → use flows tools -- `keboola.data-apps` → use data applications tools -- `keboola.snowflake-transformation` / `keboola.google-bigquery-transformation` → use SQL transformation tools - -PREREQUISITES: -- The configuration row must already exist (use add_config_row for new rows) -- You must know component_id, configuration_id, and configuration_row_id -- For parameter updates: Review the component's row_configuration_schema using get_components -- For storage updates: Ensure mappings are valid for row-level storage - -IMPORTANT CONSIDERATIONS: -- Parameter updates are PARTIAL - only specify fields you want to change -- parameter_updates supports granular operations: set individual keys, replace strings, or remove keys -- Parameters must conform to the component's row_configuration_schema (not root schema) -- Validate schemas before calling: use get_components to retrieve row_configuration_schema -- Each row operates independently - changes to one row don't affect others -- Row-level storage is separate from root-level storage configuration - -WORKFLOW: -1. Retrieve current configuration using get_configs to see existing rows -2. Identify the specific row to modify by its configuration_row_id -3. Prepare parameter_updates list with targeted operations for this row -4. Call update_config_row with only the fields to change - - -**Input JSON Schema**: -```json -{ - "$defs": { - "ConfigParamListAppend": { - "description": "Append a value to a list parameter.", - "properties": { - "op": { - "const": "list_append", - "type": "string" - }, - "path": { - "description": "JSONPath to the list parameter", - "type": "string" - }, - "value": { - "description": "Value to append to the list" - } - }, - "required": [ - "op", - "path", - "value" - ], - "type": "object" + ] }, - "ConfigParamRemove": { - "description": "Remove a parameter key.", - "properties": { - "op": { - "const": "remove", - "type": "string" + "processors_before": { + "description": "The list of processors that will run before the configured component runs.", + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } }, - "path": { - "description": "JSONPath to the parameter key to remove", - "type": "string" + { + "type": "null" } - }, - "required": [ - "op", - "path" - ], - "type": "object" + ] }, - "ConfigParamReplace": { - "description": "Replace a substring in a string parameter.", - "properties": { - "op": { - "const": "str_replace", - "type": "string" - }, - "path": { - "description": "JSONPath to the parameter key to modify", - "type": "string" - }, - "search_for": { - "description": "Substring to search for (non-empty)", - "type": "string" + "processors_after": { + "description": "The list of processors that will run after the configured component runs.", + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } }, - "replace_with": { - "description": "Replacement string (can be empty for deletion)", - "type": "string" + { + "type": "null" } - }, - "required": [ - "op", - "path", - "search_for", - "replace_with" - ], - "type": "object" + ] }, - "ConfigParamSet": { - "description": "Set or create a parameter value at the specified path.\n\nUse this operation to:\n- Update an existing parameter value\n- Create a new parameter key\n- Replace a nested parameter value", - "properties": { - "op": { - "const": "set", - "type": "string" - }, - "path": { - "description": "JSONPath to the parameter key to set (e.g., \"api_key\", \"database.host\")", + "folder": { + "description": "Folder name to organize this configuration in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more configurations in the project. If there are 20 or more configurations, you should assign one of the existing folders or create a new one that clearly reflects the configuration purpose.", + "anyOf": [ + { "type": "string" }, - "value": { - "description": "New value to set" + { + "type": "null" } - }, - "required": [ - "op", - "path", - "value" - ], - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "change_description": { - "description": "A clear, human-readable summary of what changed in this row update. Be specific.", - "type": "string" - }, - "component_id": { - "description": "The ID of the component the configuration belongs to.", - "type": "string" - }, - "configuration_id": { - "description": "The ID of the parent configuration containing the row to update.", - "type": "string" - }, - "configuration_row_id": { - "description": "The ID of the specific configuration row to update.", - "type": "string" - }, - "name": { - "default": "", - "description": "New name for the configuration row. Only provide if changing the name. Name should be short (typically under 50 characters) and descriptive of this specific row.", - "type": "string" - }, - "description": { - "default": "", - "description": "New detailed description for the configuration row. Only provide if changing the description. Should explain the specific purpose and behavior of this individual row.", - "type": "string" - }, - "parameter_updates": { - "default": null, - "description": "List of granular parameter update operations to apply to this row. Each operation (set, str_replace, remove, list_append) modifies a specific parameter using JSONPath notation. Only provide if updating parameters - do not use for changing description or storage. Paths are relative to the row's `parameters` object, not the row root (e.g. use `tables`, not `parameters.tables`). Prefer simple dot-delimited JSONPaths and make the smallest possible updates - only change what needs changing. In case you need to replace the whole parameters, you can use the `set` operation with `$` as path.", - "items": { - "discriminator": { - "mapping": { - "list_append": "#/$defs/ConfigParamListAppend", - "remove": "#/$defs/ConfigParamRemove", - "set": "#/$defs/ConfigParamSet", - "str_replace": "#/$defs/ConfigParamReplace" - }, - "propertyName": "op" - }, - "oneOf": [ - { - "$ref": "#/$defs/ConfigParamSet" - }, - { - "$ref": "#/$defs/ConfigParamReplace" - }, - { - "$ref": "#/$defs/ConfigParamRemove" - }, - { - "$ref": "#/$defs/ConfigParamListAppend" - } - ] - }, - "type": "array" - }, - "storage": { - "additionalProperties": true, - "default": null, - "description": "Complete storage configuration for this row containing input/output table and file mappings. Only provide if updating storage mappings - this replaces the ENTIRE storage configuration for this row. \n\nWhen to use:\n- Adding/removing input or output tables for this specific row\n- Modifying table/file mappings for this row\n- Updating table destinations or sources for this row\n\nImportant:\n- Must conform to the component's row storage schema\n- Replaces ALL existing storage config for this row - include all mappings you want to keep\n- Use get_configs first to see current row storage configuration\n- Leave unfilled to preserve existing storage configuration", - "type": "object" - }, - "processors_before": { - "default": null, - "description": "The list of processors that will run before the configured component row runs.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - "processors_after": { - "default": null, - "description": "The list of processors that will run after the configured component row runs.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" + ] }, - "is_disabled": { + "variables": { + "description": "Variable definitions for this configuration. Provide a non-empty list to create or replace all variable definitions. Provide an empty list ([]) to remove all variables. Omit (None) to leave existing variables unchanged.", "anyOf": [ { - "type": "boolean" + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Variable name." + }, + "type": { + "default": "string", + "description": "Variable type: \"string\" or \"vault\".", + "type": "string", + "enum": [ + "string", + "vault" + ] + }, + "default_value": { + "description": "Optional default value bound at creation time.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "name" + ] + } }, { "type": "null" } - ], - "default": null, - "description": "Enable or disable the configuration row. Set to True to disable execution (config row won't run), False to enable execution (config row will run). Only provide if changing the status, leave as null to preserve current state." + ] } }, "required": [ "change_description", "component_id", - "configuration_id", - "configuration_row_id" + "configuration_id" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` --- - -## update_sql_transformation + +## update_config_row **Annotations**: `destructive` **Tags**: `components, config-diff-preview` **Description**: -Updates an existing SQL transformation configuration by modifying its SQL code, storage mappings, -name or description. - -This tool allows PARTIAL parameter updates for transformation SQL blocks and code - you only need to provide -the operations you want to perform. All other fields will remain unchanged. -Use this for modifying SQL transformations created with create_sql_transformation. - -WHEN TO USE: -- SQL transformations only (Snowflake/BigQuery); use update_config for Python/R transformations -- Modifying SQL queries in transformation (add/edit/remove SQL statements) -- Updating transformation block or code block names -- Changing input/output table mappings for the transformation -- Updating the transformation name or description -- Any combination of the above - -PREREQUISITES: -- Transformation must already exist (use create_sql_transformation for new transformations) -- You must know the configuration_id of the transformation -- SQL dialect is determined automatically from the workspace -- CRITICAL: Use get_configs first to see the current transformation structure and get block_id/code_id values - -TRANSFORMATION STRUCTURE: -A transformation has this hierarchy: - transformation - └─ blocks[] - List of transformation blocks (each has a unique block_id) - └─ block.name - Descriptive name for the block - └─ block.codes[] - List of code blocks within the block (each has a unique code_id) - └─ code.name - Descriptive name for the code block - └─ code.script - SQL script (string with SQL statements) - -Example structure from get_configs: -{ - "blocks": [ - { - "id": "b0", ← block_id needed for operations (format: b{index}) - "name": "Data Preparation", - "codes": [ - { - "id": "b0.c0", ← code_id needed for operations (format: b{block_index}.c{code_index}) - "name": "Load customers", - "script": "SELECT * FROM customers WHERE status = 'active';" - } - ] - } - ] -} - -PARAMETER UPDATE OPERATIONS: -All operations use block_id and code_id to identify elements (get these from get_configs first). - -ID Format: -- block_id: "b0", "b1", "b2", etc. (format: b{index}) -- code_id: "b0.c0", "b0.c1", "b1.c0", etc. (format: b{block_index}.c{code_index}) - -1. BLOCK OPERATIONS: - - add_block: Create a new block in the transformation - {"op": "add_block", "block": {"name": "New Block", "codes": []}, "position": "end"} - - - remove_block: Delete an entire block - {"op": "remove_block", "block_id": "b0"} - - - rename_block: Change a block's name - {"op": "rename_block", "block_id": "b2", "block_name": "Updated Name"} - -2. CODE BLOCK OPERATIONS: - - add_code: Create a new code block within an existing block - {"op": "add_code", "block_id": "b1", "code": {"name": "New Code", "script": "SELECT 1;"}, "position": "end"} - - - remove_code: Delete a code block - {"op": "remove_code", "block_id": "b0", "code_id": "b0.c0"} - - - rename_code: Change a code block's name - {"op": "rename_code", "block_id": "b1", "code_id": "b1.c2", "code_name": "Updated Name"} - -3. SQL SCRIPT OPERATIONS: - - set_code: Replace the entire SQL script (overwrites existing) - {"op": "set_code", "block_id": "b0", "code_id": "b0.c0", "script": "SELECT * FROM new_table;"} - - - add_script: Append or prepend SQL to existing script (preserves existing) - {"op": "add_script", "block_id": "b2", "code_id": "b2.c1", "script": "WHERE date > '2024-01-01'", - "position": "end"} - - - str_replace: Find and replace text in SQL scripts - {"op": "str_replace", "search_for": "old_table", "replace_with": "new_table", "block_id": "b0",' - "code_id": "b0.c0"} - - Omit code_id to replace in all codes of a block - - Omit both block_id and code_id to replace everywhere - -IMPORTANT CONSIDERATIONS: -- Parameter updates are PARTIAL - only the operations you specify are applied -- All other parts of the transformation remain unchanged -- Each SQL script must be executable and follow the current SQL dialect: - - Use delimited identifiers for the current SQL dialect. - - Never mix delimiter styles within a single query. -- Storage configuration is COMPLETE REPLACEMENT - include ALL mappings you want to keep -- Leave updated_description empty to preserve the original description -- SCHEMA CHANGES: Destructive schema changes (removing columns, changing types, renaming columns) require - manually deleting the output table before running the updated transformation to avoid schema mismatch errors. - Non-destructive changes (adding columns) typically do not require table deletion. - -WORKFLOW: -1. Call get_configs to retrieve current transformation structure and identify block_id/code_id values -2. Identify what needs to change (SQL code, storage, description) -3. For SQL changes: Prepare parameter_updates list with targeted operations -4. For storage changes: Build complete storage configuration (include all mappings) -5. Call update_sql_transformation with change_description and only the fields to change - -EXAMPLE WORKFLOWS: - -Example 1 - Update SQL script in existing code block: -Step 1: Get current config - result = get_configs(component_id="keboola.snowflake-transformation", configuration_id="12345") - # Note the block_id (e.g., "b0") and code_id (e.g., "b0.c1") from result - -Step 2: Update the SQL - update_sql_transformation( - configuration_id="12345", - change_description="Updated WHERE clause to filter active customers only", - parameter_updates=[ - { - "op": "set_code", - "block_id": "b0", # from step 1 - "code_id": "b0.c0", # from step 1 - "script": "SELECT * FROM customers WHERE status = 'active' AND region = 'US';" - } - ] - ) - -Example 2 - Append a new code block to the second block of an existing transformation: - update_sql_transformation( - configuration_id="12345", - change_description="Added aggregation step", - parameter_updates=[ - { - "op": "add_code", - "block_id": "b1", # second block - "code": { - "name": "Aggregate Sales", - "script": "SELECT customer_id, SUM(amount) as total FROM orders GROUP BY customer_id;" - }, - "position": "end" - } - ] - ) - -Example 3 - Replace table name across all SQL scripts: - update_sql_transformation( - configuration_id="12345", - change_description="Renamed source table from old_customers to customers", - parameter_updates=[ - { - "op": "str_replace", - "search_for": "old_customers", - "replace_with": "customers" - # No block_id or code_id = applies to all scripts - } - ] - ) - -Example 4 - Update storage mappings: - update_sql_transformation( - configuration_id="12345", - change_description="Added new input table", - storage={ - "input": { - "tables": [ - { - "source": "in.c-main.customers", - "destination": "customers" - }, - { - "source": "in.c-main.orders", - "destination": "orders" - } - ] - }, - "output": { - "tables": [ - { - "source": "result", - "destination": "out.c-main.customer_summary" - } - ] - } - } - ) +Updates an existing component configuration row by modifying its parameters, storage mappings, name, or description. Updates are PARTIAL — only provide the fields you want to change; parameter_updates apply granular diff operations to the existing row parameters. **Input JSON Schema**: ```json { - "$defs": { - "Block": { - "description": "The transformation block.", - "properties": { - "name": { - "description": "A descriptive name for the code block", - "type": "string" - }, - "codes": { - "description": "SQL code sub-blocks", - "items": { - "$ref": "#/$defs/Code" - }, - "type": "array" - } - }, - "required": [ - "name", - "codes" - ], - "type": "object" - }, - "Code": { - "description": "The code block for the transformation block.", - "properties": { - "name": { - "description": "A descriptive name for the code block", - "type": "string" - }, - "script": { - "description": "The SQL script of the code block", - "type": "string" - } - }, - "required": [ - "name", - "script" - ], - "type": "object" - }, - "TfAddBlock": { - "description": "Add a new block to the transformation.", - "properties": { - "op": { - "const": "add_block", - "type": "string" - }, - "block": { - "$ref": "#/$defs/Block", - "description": "The block to add" - }, - "position": { - "default": "end", - "description": "The position of the block to add", - "enum": [ - "start", - "end" - ], - "type": "string" - } - }, - "required": [ - "op", - "block" - ], - "type": "object" - }, - "TfAddCode": { - "description": "Add a new code to an existing block in the transformation.", - "properties": { - "op": { - "const": "add_code", - "type": "string" - }, - "block_id": { - "description": "The ID of the block to add the code to", - "type": "string" - }, - "code": { - "$ref": "#/$defs/Code", - "description": "The code to add" - }, - "position": { - "default": "end", - "description": "The position of the code to add", - "enum": [ - "start", - "end" - ], - "type": "string" - } - }, - "required": [ - "op", - "block_id", - "code" - ], - "type": "object" - }, - "TfAddScript": { - "description": "Append or prepend SQL script text to an existing code in an existing block in the transformation.", - "properties": { - "op": { - "const": "add_script", - "type": "string" - }, - "block_id": { - "description": "The ID of the block to add the script to", - "type": "string" - }, - "code_id": { - "description": "The ID of the code to add the script to", - "type": "string" - }, - "script": { - "description": "The SQL script to add", - "type": "string" - }, - "position": { - "default": "end", - "description": "The position of the script to add", - "enum": [ - "start", - "end" - ], - "type": "string" - } - }, - "required": [ - "op", - "block_id", - "code_id", - "script" - ], - "type": "object" + "type": "object", + "properties": { + "change_description": { + "type": "string", + "description": "A clear, human-readable summary of what changed in this row update. Be specific." }, - "TfRemoveBlock": { - "description": "Remove an existing block from the transformation.", - "properties": { - "op": { - "const": "remove_block", - "type": "string" - }, - "block_id": { - "description": "The ID of the block to remove", - "type": "string" - } - }, - "required": [ - "op", - "block_id" - ], - "type": "object" + "component_id": { + "type": "string", + "description": "The ID of the component the configuration belongs to." }, - "TfRemoveCode": { - "description": "Remove an existing code from an existing block in the transformation.", - "properties": { - "op": { - "const": "remove_code", - "type": "string" - }, - "block_id": { - "description": "The ID of the block to remove the code from", - "type": "string" - }, - "code_id": { - "description": "The ID of the code to remove", - "type": "string" - } - }, - "required": [ - "op", - "block_id", - "code_id" - ], - "type": "object" + "configuration_id": { + "type": "string", + "description": "The ID of the parent configuration containing the row to update." }, - "TfRenameBlock": { - "description": "Rename an existing block in the transformation.", - "properties": { - "op": { - "const": "rename_block", - "type": "string" - }, - "block_id": { - "description": "The ID of the block to rename", - "type": "string" - }, - "block_name": { - "description": "The new name of the block", - "type": "string" - } - }, - "required": [ - "op", - "block_id", - "block_name" - ], - "type": "object" + "configuration_row_id": { + "type": "string", + "description": "The ID of the specific configuration row to update." }, - "TfRenameCode": { - "description": "Rename an existing code in an existing block in the transformation.", - "properties": { - "op": { - "const": "rename_code", - "type": "string" - }, - "block_id": { - "description": "The ID of the block to rename the code in", - "type": "string" - }, - "code_id": { - "description": "The ID of the code to rename", - "type": "string" + "name": { + "default": "", + "description": "New name for the configuration row. Only provide if changing the name. Name should be short (typically under 50 characters) and descriptive of this specific row.", + "type": "string" + }, + "description": { + "default": "", + "description": "New detailed description for the configuration row. Only provide if changing the description. Should explain the specific purpose and behavior of this individual row.", + "type": "string" + }, + "parameter_updates": { + "description": "List of granular parameter update operations to apply to this row. Each operation (set, str_replace, remove, list_append) modifies a specific parameter using JSONPath notation. Only provide if updating parameters - do not use for changing description or storage. Paths are relative to the row's `parameters` object, not the row root (e.g. use `tables`, not `parameters.tables`). Prefer simple dot-delimited JSONPaths and make the smallest possible updates - only change what needs changing. In case you need to replace the whole parameters, you can use the `set` operation with `$` as path.", + "anyOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "set" + }, + "path": { + "type": "string", + "description": "JSONPath to the parameter key to set (e.g., \"api_key\", \"database.host\")" + }, + "value": { + "description": "New value to set" + } + }, + "required": [ + "op", + "path", + "value" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "str_replace" + }, + "path": { + "type": "string", + "description": "JSONPath to the parameter key to modify" + }, + "search_for": { + "type": "string", + "description": "Substring to search for (non-empty)" + }, + "replace_with": { + "type": "string", + "description": "Replacement string (can be empty for deletion)" + } + }, + "required": [ + "op", + "path", + "search_for", + "replace_with" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "remove" + }, + "path": { + "type": "string", + "description": "JSONPath to the parameter key to remove" + } + }, + "required": [ + "op", + "path" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "list_append" + }, + "path": { + "type": "string", + "description": "JSONPath to the list parameter" + }, + "value": { + "description": "Value to append to the list" + } + }, + "required": [ + "op", + "path", + "value" + ] + } + ] + } }, - "code_name": { - "description": "The new name of the code", - "type": "string" + { + "type": "null" } - }, - "required": [ - "op", - "block_id", - "code_id", - "code_name" - ], - "type": "object" + ] }, - "TfSetCode": { - "description": "Set the SQL script of an existing code in an existing block in the transformation.", - "properties": { - "op": { - "const": "set_code", - "type": "string" - }, - "block_id": { - "description": "The ID of the block to set the code in", - "type": "string" - }, - "code_id": { - "description": "The ID of the code to set", - "type": "string" + "storage": { + "description": "Complete storage configuration for this row containing input/output table and file mappings. Only provide if updating storage mappings - this replaces the ENTIRE storage configuration for this row.", + "anyOf": [ + { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} }, - "script": { - "description": "The SQL script of the code to set", - "type": "string" + { + "type": "null" } - }, - "required": [ - "op", - "block_id", - "code_id", - "script" - ], - "type": "object" + ] }, - "TfStrReplace": { - "description": "Replace a substring in SQL statements in the transformation.", - "properties": { - "op": { - "const": "str_replace", - "type": "string" - }, - "block_id": { - "anyOf": [ - { + "processors_before": { + "description": "The list of processors that will run before the configured component runs.", + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { "type": "string" }, - { - "type": "null" - } - ], - "default": null, - "description": "The ID of the block to replace substrings in. If not provided, all blocks will be updated." + "additionalProperties": {} + } }, - "code_id": { - "anyOf": [ - { + { + "type": "null" + } + ] + }, + "processors_after": { + "description": "The list of processors that will run after the configured component runs.", + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { "type": "string" }, - { - "type": "null" - } - ], - "default": null, - "description": "The ID of the code to replace substrings in. If not provided, all codes in the block will be updated." - }, - "search_for": { - "description": "Substring to search for (non-empty)", - "type": "string" + "additionalProperties": {} + } }, - "replace_with": { - "description": "Replacement string (can be empty for deletion)", - "type": "string" + { + "type": "null" } - }, - "required": [ - "op", - "search_for", - "replace_with" - ], - "type": "object" + ] }, - "VariableDefinition": { - "description": "A single variable definition to attach to a configuration.", - "properties": { - "name": { - "description": "Variable name.", - "type": "string" - }, - "type": { - "default": "string", - "description": "Variable type: \"string\" or \"vault\".", - "enum": [ - "string", - "vault" - ], - "type": "string" + "is_disabled": { + "description": "Enable or disable the configuration row. Set to True to disable execution (config row won't run), False to enable execution (config row will run). Only provide if changing the status, leave as null to preserve current state.", + "anyOf": [ + { + "type": "boolean" }, - "default_value": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional default value bound at creation time." + { + "type": "null" } - }, - "required": [ - "name" - ], - "type": "object" + ] } }, - "additionalProperties": false, + "required": [ + "change_description", + "component_id", + "configuration_id", + "configuration_row_id" + ], + "$schema": "http://json-schema.org/draft-07/schema#" +} +``` + +--- + +## update_sql_transformation +**Annotations**: `destructive` + +**Tags**: `components, config-diff-preview` + +**Description**: + +Updates an existing SQL transformation configuration by modifying its SQL code, storage mappings, name or description. parameter_updates apply PARTIAL, granular diff operations to the transformation blocks/codes; storage is a complete replacement. + + +**Input JSON Schema**: +```json +{ + "type": "object", "properties": { "change_description": { - "description": "A clear, human-readable summary of what changed in this transformation update. Be specific: e.g., \"Added JOIN with customers table\", \"Updated WHERE clause to filter active records\".", - "type": "string" + "type": "string", + "description": "A clear, human-readable summary of what changed in this transformation update. Be specific: e.g., \"Added JOIN with customers table\", \"Updated WHERE clause to filter active records\"." }, "configuration_id": { - "description": "The ID of the transformation configuration to update.", - "type": "string" + "type": "string", + "description": "The ID of the transformation configuration to update." }, "name": { "default": "", @@ -1811,62 +1111,319 @@ Example 4 - Update storage mappings: "type": "string" }, "parameter_updates": { - "default": null, - "description": "List of operations to apply to the transformation structure (blocks, codes, SQL scripts). Each operation modifies specific elements using block_id and code_id identifiers. Only provide if updating SQL code or block structure - do not use for description or storage changes. \n\nIMPORTANT: Use get_configs first to retrieve the current transformation structure and identify the block_id and code_id values needed for your operations. IDs are automatically assigned.\n\nAvailable operations:\n1. add_block: Add a new block to the transformation\n - Fields: op=\"add_block\", block={name, codes}, position=\"start\"|\"end\"\n2. remove_block: Remove an existing block\n - Fields: op=\"remove_block\", block_id (e.g., \"b0\")\n3. rename_block: Rename an existing block\n - Fields: op=\"rename_block\", block_id (e.g., \"b0\"), block_name\n4. add_code: Add a new code block to an existing block\n - Fields: op=\"add_code\", block_id (e.g., \"b0\"), code={name, script}, position=\"start\"|\"end\"\n5. remove_code: Remove an existing code block\n - Fields: op=\"remove_code\", block_id (e.g., \"b0\"), code_id (e.g., \"b0.c0\")\n6. rename_code: Rename an existing code block\n - Fields: op=\"rename_code\", block_id (e.g., \"b0\"), code_id (e.g., \"b0.c0\"), code_name\n7. set_code: Replace the entire SQL script of a code block\n - Fields: op=\"set_code\", block_id (e.g., \"b0\"), code_id (e.g., \"b0.c0\"), script\n8. add_script: Append or prepend SQL to a code block\n - Fields: op=\"add_script\", block_id (e.g., \"b0\"), code_id (e.g., \"b0.c0\"), script, position=\"start\"|\"end\"\n9. str_replace: Replace substring in SQL scripts\n - Fields: op=\"str_replace\", search_for, replace_with, block_id (optional), code_id (optional)\n - If block_id omitted: replaces in all blocks\n - If code_id omitted: replaces in all codes of the specified block\n", - "items": { - "discriminator": { - "mapping": { - "add_block": "#/$defs/TfAddBlock", - "add_code": "#/$defs/TfAddCode", - "add_script": "#/$defs/TfAddScript", - "remove_block": "#/$defs/TfRemoveBlock", - "remove_code": "#/$defs/TfRemoveCode", - "rename_block": "#/$defs/TfRenameBlock", - "rename_code": "#/$defs/TfRenameCode", - "set_code": "#/$defs/TfSetCode", - "str_replace": "#/$defs/TfStrReplace" - }, - "propertyName": "op" - }, - "oneOf": [ - { - "$ref": "#/$defs/TfAddBlock" - }, - { - "$ref": "#/$defs/TfRemoveBlock" - }, - { - "$ref": "#/$defs/TfRenameBlock" - }, - { - "$ref": "#/$defs/TfAddCode" - }, - { - "$ref": "#/$defs/TfRemoveCode" - }, - { - "$ref": "#/$defs/TfRenameCode" - }, - { - "$ref": "#/$defs/TfSetCode" - }, - { - "$ref": "#/$defs/TfAddScript" - }, - { - "$ref": "#/$defs/TfStrReplace" + "description": "List of operations to apply to the transformation structure (blocks, codes, SQL scripts). Each operation modifies specific elements using block_id and code_id identifiers. Only provide if updating SQL code or block structure - do not use for description or storage changes. Use get_configs first to retrieve the current transformation structure and identify the block_id and code_id values needed for your operations. IDs are automatically assigned. Available operations: add_block, remove_block, rename_block, add_code, remove_code, rename_code, set_code, add_script, str_replace.", + "anyOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "add_block" + }, + "block": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "A descriptive name for the code block" + }, + "codes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "A descriptive name for the code block" + }, + "script": { + "type": "string", + "description": "The SQL script of the code block" + } + }, + "required": [ + "name", + "script" + ] + }, + "description": "SQL code sub-blocks" + } + }, + "required": [ + "name", + "codes" + ], + "description": "The block to add" + }, + "position": { + "default": "end", + "type": "string", + "enum": [ + "start", + "end" + ] + } + }, + "required": [ + "op", + "block" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "remove_block" + }, + "block_id": { + "type": "string", + "description": "The ID of the block to remove" + } + }, + "required": [ + "op", + "block_id" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "rename_block" + }, + "block_id": { + "type": "string" + }, + "block_name": { + "type": "string", + "description": "The new name of the block" + } + }, + "required": [ + "op", + "block_id", + "block_name" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "add_code" + }, + "block_id": { + "type": "string" + }, + "code": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "A descriptive name for the code block" + }, + "script": { + "type": "string", + "description": "The SQL script of the code block" + } + }, + "required": [ + "name", + "script" + ], + "description": "The code to add" + }, + "position": { + "default": "end", + "type": "string", + "enum": [ + "start", + "end" + ] + } + }, + "required": [ + "op", + "block_id", + "code" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "remove_code" + }, + "block_id": { + "type": "string" + }, + "code_id": { + "type": "string" + } + }, + "required": [ + "op", + "block_id", + "code_id" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "rename_code" + }, + "block_id": { + "type": "string" + }, + "code_id": { + "type": "string" + }, + "code_name": { + "type": "string", + "description": "The new name of the code" + } + }, + "required": [ + "op", + "block_id", + "code_id", + "code_name" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "set_code" + }, + "block_id": { + "type": "string" + }, + "code_id": { + "type": "string" + }, + "script": { + "type": "string", + "description": "The SQL script of the code to set" + } + }, + "required": [ + "op", + "block_id", + "code_id", + "script" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "add_script" + }, + "block_id": { + "type": "string" + }, + "code_id": { + "type": "string" + }, + "script": { + "type": "string", + "description": "The SQL script to add" + }, + "position": { + "default": "end", + "type": "string", + "enum": [ + "start", + "end" + ] + } + }, + "required": [ + "op", + "block_id", + "code_id", + "script" + ] + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "str_replace" + }, + "block_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "code_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "search_for": { + "type": "string", + "description": "Substring to search for (non-empty)" + }, + "replace_with": { + "type": "string", + "description": "Replacement string (can be empty for deletion)" + } + }, + "required": [ + "op", + "search_for", + "replace_with" + ] + } + ] } - ] - }, - "type": "array" + }, + { + "type": "null" + } + ] }, "storage": { - "additionalProperties": true, - "default": null, - "description": "Complete storage configuration for transformation input/output table mappings. Only provide if updating storage mappings - this replaces the ENTIRE storage configuration. \n\nWhen to use:\n- Adding/removing input tables for the transformation\n- Modifying output table mappings and destinations\n- Changing table aliases used in SQL\n\nImportant:\n- Must conform to transformation storage schema (input/output tables)\n- Replaces ALL existing storage config - include all mappings you want to keep\n- Use get_configs first to see current storage configuration\n- Leave unfilled to preserve existing storage configuration", - "type": "object" + "description": "Complete storage configuration for transformation input/output table mappings. Only provide if updating storage mappings - this replaces the ENTIRE storage configuration.", + "anyOf": [ + { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + }, + { + "type": "null" + } + ] }, "folder": { + "description": "Folder name to organize this transformation in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more transformations in the project. If there are 20 or more transformations, you should assign one of the existing folders or create a new one that clearly reflects the transformation purpose.", "anyOf": [ { "type": "string" @@ -1874,31 +1431,57 @@ Example 4 - Update storage mappings: { "type": "null" } - ], - "default": null, - "description": "Folder name to organize this transformation in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more transformations in the project. If there are 20 or more transformations, you should assign one of the existing folders or create a new one that clearly reflects the transformation purpose." + ] }, "variables": { + "description": "Variable definitions for this transformation. Provide a non-empty list to create or replace all variable definitions. Provide an empty list ([]) to remove all variables. Omit (None) to leave existing variables unchanged.", "anyOf": [ { + "type": "array", "items": { - "$ref": "#/$defs/VariableDefinition" - }, - "type": "array" + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Variable name." + }, + "type": { + "default": "string", + "description": "Variable type: \"string\" or \"vault\".", + "type": "string", + "enum": [ + "string", + "vault" + ] + }, + "default_value": { + "description": "Optional default value bound at creation time.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "name" + ] + } }, { "type": "null" } - ], - "default": null, - "description": "Variable definitions for this transformation. Provide a non-empty list to create or replace all variable definitions. Provide an empty list ([]) to remove all variables. Omit (None) to leave existing variables unchanged." + ] } }, "required": [ "change_description", "configuration_id" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -1913,60 +1496,33 @@ Example 4 - Update storage mappings: **Description**: -Mints a one-time HTTPS token on a python-js **prod** data app so the caller can clone, pull, -and push to the app's managed git repo over HTTPS. - -**Always call against the prod app's configuration_id** — drafts have no managed repo of their -own, so calling this on a draft fails. The prod app is the canonical repo owner; drafts -iterate against branches of that same repo. - -**MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, -branch-delete — is yours. This tool only mints credentials. +Mints a one-time HTTPS token on a python-js **prod** data app so the caller can clone, pull, and push to the app's managed git repo over HTTPS. -Returns a ready-to-use `git_clone_url` of the form `https://kai:@/.git` -plus the raw `secret`. The token is returned **only** at creation — the platform cannot return -it again on any subsequent read. Stash the URL (or the secret) somewhere the LLM can reuse for -the rest of the session. +**Always call against the prod app's configuration_id** — drafts have no managed repo of their own, so calling this on a draft fails. The prod app is the canonical repo owner; drafts iterate against branches of that same repo. -The data-science API accepts multiple credentials per app, so calling this again mints an -additional token without invalidating any tokens already held by other clients. +**MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, branch-delete — is yours. This tool only mints credentials. -## When to call - -1. **Right after `modify_python_js_data_app` create of a prod app** — the new prod has a - managed repo but no credentials yet. Call this tool with the new app's `configuration_id` - to enable git access. (Note: when creating a **draft**, the prod-side token is minted and - embedded into the returned `git_clone_url` automatically — no separate call needed.) - -2. **Recovery when the cached token is gone / continuing an unfinished draft** — e.g., a fresh - sandbox continuing yesterday's work, with the previous sandbox's filesystem wiped. The - cached `git_clone_url` is lost; the configuration ID for the prod app is all you have. - Call this tool with the **prod app's** `configuration_id` to mint a fresh token (drafts - have no managed repo, so always mint against prod). Existing credentials remain valid, so - other clients are not disrupted. +Returns a ready-to-use `git_clone_url` of the form `https://kai:@/.git` plus the raw `secret`. The token is returned **only** at creation — the platform cannot return it again on any subsequent read. Stash the URL (or the secret) somewhere the LLM can reuse for the rest of the session. ## Constraints -- Only python-js prod data apps have a managed git repo. Streamlit apps reject the call with - a clear error. -- Permissions are always `readWrite` — the LLM virtually always needs push access. The - data-science API supports read-only credentials, but the tool does not expose that knob; - revisit once a real use case appears. +- Only python-js prod data apps have a managed git repo. Streamlit apps reject the call with a clear error. +- Permissions are always `readWrite`. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "configuration_id": { - "description": "Storage configuration ID of the python-js data app.", - "type": "string" + "type": "string", + "description": "Storage configuration ID of the python-js data app." } }, "required": [ "configuration_id" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -1979,16 +1535,11 @@ additional token without invalidating any tokens already held by other clients. **Description**: -Deletes a python-js DRAFT data app — both the data-app instance (DSAPI) and its Storage -configuration. +Deletes a python-js DRAFT data app — both the data-app instance (DSAPI) and its Storage configuration. -**MCP never runs git on your behalf.** Deleting the feature branch on the remote is your job; -this tool only tears down the draft config and its data-app instance. +**MCP never runs git on your behalf.** Deleting the feature branch on the remote is your job; this tool only tears down the draft config and its data-app instance. -WHEN TO CALL: at the end of a promote-to-prod sequence, after you have merged the draft's -branch into `main`, pushed, deleted the feature branch from the remote, and redeployed the -prod app. The Keboola UI lists drafts under their parent prod app; once you call this tool, -the draft disappears from that list. +WHEN TO CALL: at the end of a promote-to-prod sequence, after you have merged the draft's branch into `main`, pushed, deleted the feature branch from the remote, and redeployed the prod app. The Keboola UI lists drafts under their parent prod app; once you call this tool, the draft disappears from that list. WHAT THIS TOOL REFUSES: - prod apps (no `isDraft` flag) — protects against accidental prod deletion; @@ -1996,27 +1547,25 @@ WHAT THIS TOOL REFUSES: WHAT THIS TOOL DOES NOT DO: - Run git. Deleting the feature branch on the remote is your job. - - Revoke the prod-side git credential minted when the draft was created. Credential - rotation is the user's job via the Keboola UI. + - Revoke the prod-side git credential minted when the draft was created. -After a successful call, pivot back to the parent prod app (its configuration_id is returned -in the response) or to `get_data_apps` for further work. +After a successful call, pivot back to the parent prod app (its configuration_id is returned in the response) or to `get_data_apps` for further work. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "configuration_id": { - "description": "Storage configuration ID of the python-js draft data app to delete.", - "type": "string" + "type": "string", + "description": "Storage configuration ID of the python-js draft data app to delete." } }, "required": [ "configuration_id" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2029,74 +1578,62 @@ in the response) or to `get_data_apps` for further work. **Description**: -Deploys/redeploys a data app or stops a running data app in the Keboola environment asynchronously, given the -action and the configuration ID. +Deploys/redeploys a data app or stops a running data app in the Keboola environment asynchronously, given the action and the configuration ID. -**MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, -branch-delete — is yours. This tool only triggers deploys against existing git state. +**MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, branch-delete — is yours. This tool only triggers deploys against existing git state. ## Mode (python-js apps) -- `mode='dev'` deploys the target as a **dev version of the data app** — the runtime uses a - development `setup.sh` (hot reload) and the data-app proxy enables an auto-auth path so an - iframe preview can render without a manual login. Only meaningful on **draft** configs - (python-js apps with `isDraft=true`). -- For prod redeploys (including after merging a draft's branch into `main`), use no `mode` — - the prod app picks up the current `main`. -- The branch a draft deploys from is pinned in `parameters.dataApp.git.branch` at create time; - there is no deploy-time override. -- python-js apps do NOT fetch a Storage `configVersion` for deployment (their source lives in - git, not in the Storage configuration); this is handled automatically. +- `mode='dev'` deploys the target as a **dev version of the data app** — the runtime uses a development `setup.sh` (hot reload) and the data-app proxy enables an auto-auth path so an iframe preview can render without a manual login. Only meaningful on **draft** configs (python-js apps with `isDraft=true`). +- For prod redeploys (including after merging a draft's branch into `main`), use no `mode` — the prod app picks up the current `main`. +- The branch a draft deploys from is pinned in `parameters.dataApp.git.branch` at create time; there is no deploy-time override. +- python-js apps do NOT fetch a Storage `configVersion` for deployment (their source lives in git, not in the Storage configuration); this is handled automatically. ## Streamlit apps -Streamlit apps have no managed git repo, so `mode` has no effect on the deployed app. -`mode=None` is the expected call shape. +Streamlit apps have no managed git repo, so `mode` has no effect on the deployed app. `mode=None` is the expected call shape. ## General considerations -- Redeploying a data app takes some time, and the app may temporarily report status "stopped" during the - restart. -- After deployment, the deployment info includes the app URL and the latest logs to help diagnose in-app - errors. +- Redeploying a data app takes some time, and the app may temporarily report status "stopped" during the restart. +- After deployment, the deployment info includes the app URL and the latest logs to help diagnose in-app errors. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "action": { - "description": "The action to perform.", + "type": "string", "enum": [ "deploy", "stop" ], - "type": "string" + "description": "The action to perform." }, "configuration_id": { - "description": "The ID of the data app configuration.", - "type": "string" + "type": "string", + "description": "The ID of the data app configuration." }, "mode": { + "description": "Deployment mode. Set to \"dev\" to deploy a python-js draft as a **dev version of the data app** — the runtime uses a development `setup.sh` (hot reload), and the data-app proxy enables an auto-auth path so an iframe preview can render without a manual login. Only meaningful on **draft** configs (python-js apps with `isDraft=true`). Leave None (default) for prod redeploys and for Streamlit apps.", "anyOf": [ { + "type": "string", "enum": [ "dev", "production" - ], - "type": "string" + ] }, { "type": "null" } - ], - "default": null, - "description": "Deployment mode. Set to \"dev\" to deploy a python-js draft as a **dev version of the data app** \u2014 the runtime uses a development `setup.sh` (hot reload), and the data-app proxy enables an auto-auth path so an iframe preview can render without a manual login. Only meaningful on **draft** configs (python-js apps with `isDraft=true`). Leave None (default) for prod redeploys and for Streamlit apps." + ] } }, "required": [ "action", "configuration_id" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2109,60 +1646,50 @@ Streamlit apps have no managed git repo, so `mode` has no effect on the deployed **Description**: -Lists summaries of data apps in the project given the limit and offset or gets details of a data apps by -providing their configuration IDs. +Lists summaries of data apps in the project given the limit and offset or gets details of a data apps by providing their configuration IDs. WHEN NOT TO USE: -- Do NOT list all data apps just to find one by name. Use `search` with - item_types=["data-app"] instead. +- Do NOT list all data apps just to find one by name. Use `search` with item_types=["data-app"] instead. - Only list all data apps when you need a complete inventory. Considerations: - If configuration_ids are provided, the tool will return details of the data apps by their configuration IDs. - If no configuration_ids are provided, the tool will list all data apps in the project given the limit and offset. -- Data App detail contains configuration, metadata, source code, links, and deployment info along with the latest -data app logs to investigate in-app errors. The logs may be updated after opening the data app URL. -- `deployment_info.last_run` carries the outcome of the most recent deployment attempt. For an app - that fails to start, check its `failure_reason`/`failure_message` FIRST — they cover setup-phase - failures (e.g. invalid secrets, git clone errors, failing setup scripts) that happen before the - container starts and therefore never appear in the regular logs. -- `repo_url` (managed git repo URL for python-js apps) is ONLY populated on the detail path - (when `configuration_ids` is provided). The inventory list always returns `repo_url=None`, - even for python-js apps with a managed repo — to retrieve the URL, call this tool again - with the target `configuration_ids`. -- When called with `configuration_ids=[]` for a python-js **prod** app, the response - includes a `drafts: [...]` array of every draft (configs with `isDraft=true` and - `parentConfigurationId == `) currently in the project. Drafts in trash are not - included. Use this to discover existing drafts when continuing a previously abandoned - iteration (Scenario C in `modify_python_js_data_app`). The array is empty for drafts - themselves and for Streamlit apps. +- Data App detail contains configuration, metadata, source code, links, and deployment info along with the latest data app logs to investigate in-app errors. The logs may be updated after opening the data app URL. +- `deployment_info.last_run` carries the outcome of the most recent deployment attempt. For an app that fails to start, check its `failure_reason`/`failure_message` FIRST — they cover setup-phase failures (e.g. invalid secrets, git clone errors, failing setup scripts) that happen before the container starts and therefore never appear in the regular logs. +- `repo_url` (managed git repo URL for python-js apps) is ONLY populated on the detail path (when `configuration_ids` is provided). The inventory list always returns `repo_url=None`, even for python-js apps with a managed repo — to retrieve the URL, call this tool again with the target `configuration_ids`. +- When called with `configuration_ids=[]` for a python-js **prod** app, the response includes a `drafts: [...]` array of every draft (configs with `isDraft=true` and `parentConfigurationId == `) currently in the project. Drafts in trash are not included. The array is empty for drafts themselves and for Streamlit apps. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "configuration_ids": { "default": [], "description": "The IDs of the data app configurations.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } }, "limit": { "default": 100, "description": "The limit of the data apps to fetch.", - "type": "integer" + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 }, "offset": { "default": 0, "description": "The offset of the data apps to fetch.", - "type": "integer" + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 } }, - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2177,110 +1704,37 @@ data app logs to investigate in-app errors. The logs may be updated after openin Creates or updates a python-js data app. -Two-app project model. Every python-js project has a persistent **prod app** that owns the -only managed git repository for the project, and zero or more **drafts** parented to that -prod app. A draft is a Storage configuration with `parameters.dataApp.isDraft=true` and -`parameters.dataApp.parentConfigurationId=`; it's an *external-git* app that -clones the parent prod's repo at a pinned branch on every deploy. Drafts are surfaced in the -Keboola UI under their parent prod app. Use `deploy_data_app(mode='dev')` to deploy a draft -as a dev version of the data app (hot reload + auto-auth for iframe preview); use -`delete_python_js_data_app_draft` to tear a draft down after its branch has been promoted. - -**MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, -branch-delete — is yours. MCP gives you authenticated clone URLs and manages configs/deploys; -it never invokes git. - -**The draft flow below is mandatory — never edit prod source directly.** Every source-code -change goes through a draft branch that the user previews and explicitly approves first. NEVER -push directly to `main`: `main` only ever advances by merging an approved draft branch, and -only after the user has approved that draft's preview. - -Three scenarios the agent has to distinguish: - -## Scenario A — Create a brand-new data app - -1. `modify_python_js_data_app(slug='demo')` → `(configuration_id=PROD, repo_url=R)`. - PROD owns the only managed repo for this app. -2. `modify_python_js_data_app(slug='demo-draft', parent_configuration_id=PROD)` - → `(configuration_id=DRAFT, repo_url=R, git_clone_url=U, branch='init')`. - Default draft branch is `'init'`. Override with `branch=` for a descriptive name. -3. YOU: `git clone U`; `git checkout init` (creating it if the repo is empty); write source; - `git push origin init`. -4. `deploy_data_app(action='deploy', configuration_id=DRAFT, mode='dev')` - → preview URL serving the `init` branch as a dev version. Iterate with the user. -5. Once approved — YOU: `git checkout main`; `git merge init`; `git push origin main`; - `git push origin --delete init`. -6. `deploy_data_app(action='deploy', configuration_id=PROD)` - → prod URL now serves the merged `main`. -7. `delete_python_js_data_app_draft(configuration_id=DRAFT)` - → tears down the draft's config + data-app instance. Always run this once promoted. - -## Scenario B — Edit an existing data app - -You already have PROD's `configuration_id` (from `get_data_apps` or earlier conversation). - -1. `create_python_js_data_app_git_credential(configuration_id=PROD)` - → fresh `git_clone_url U` with an embedded one-time token. -2. `modify_python_js_data_app( - slug='demo-draft-', - parent_configuration_id=PROD, - branch='', # e.g. 'add-revenue-filter' - )` → `(DRAFT, R, U2, branch)`. Use U2 (it has its own fresh token). -3. YOU: `git clone U2`; `git checkout ` (creating it from `main`); edit source; - `git push origin `. -4–7. Same as Scenario A steps 4–7. - -## Scenario C — Continue an unfinished draft - -The previous sandbox is gone. You have PROD's `configuration_id` but no working clone and no -draft handle. - -1. `get_data_apps(configuration_ids=[PROD])` → returns PROD's detail including `drafts: [...]`. - Pick the draft the user means (ask if multiple and unclear). Each entry exposes its - `configuration_id`, slug, and pinned branch. -2. `create_python_js_data_app_git_credential(configuration_id=PROD)` - → fresh `git_clone_url U` (the previous one was minted in a wiped sandbox and is lost). - Drafts have no managed repo of their own — always mint against PROD. -3. YOU: `git clone U`; `git checkout `; resume work; `git push`. -4. `deploy_data_app(action='deploy', configuration_id=, mode='dev')` → preview URL. - The draft's branch is already pinned in its config. -5–7. Same promote/cleanup sequence as Scenario A steps 5–7. +Two-app project model. Every python-js project has a persistent **prod app** that owns the only managed git repository for the project, and zero or more **drafts** parented to that prod app. A draft is a Storage configuration with `parameters.dataApp.isDraft=true` and `parameters.dataApp.parentConfigurationId=`; it's an *external-git* app that clones the parent prod's repo at a pinned branch on every deploy. Drafts are surfaced in the Keboola UI under their parent prod app. Use `deploy_data_app(mode='dev')` to deploy a draft as a dev version of the data app (hot reload + auto-auth for iframe preview); use `delete_python_js_data_app_draft` to tear a draft down after its branch has been promoted. -## Argument rules +**MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, branch-delete — is yours. MCP gives you authenticated clone URLs and manages configs/deploys; it never invokes git. +**The draft flow is mandatory — never edit prod source directly.** Every source-code change goes through a draft branch that the user previews and explicitly approves first. NEVER push directly to `main`: `main` only ever advances by merging an approved draft branch, and only after the user has approved that draft's preview. + +## Argument rules - `parent_configuration_id` is **create-only**. Rejected on update. -- `branch` is **create-only** and only valid when `parent_configuration_id` is set. - Defaults to `'init'`. Must not be `'main'`. Rejected on prod create and on update. +- `branch` is **create-only** and only valid when `parent_configuration_id` is set. Defaults to `'init'`. Must not be `'main'`. Rejected on prod create and on update. - `slug` is required on create and immutable after. -- The **update path** (passing `configuration_id`) is for changing `name`, `description`, - `authentication_type`, `auto_suspend_after_seconds`, `storage` on either a prod app or - a draft. Source code changes go through the git flow above, not this tool. +- The **update path** (passing `configuration_id`) is for changing `name`, `description`, `authentication_type`, `auto_suspend_after_seconds`, `storage` on either a prod app or a draft. Source code changes go through the git flow above, not this tool. ## Authentication - -New apps default to HTTP basic authentication for safety. Pass `authentication_type='no-auth'` -to expose publicly. On update, `authentication_type='default'` preserves the existing -`authorization` block (including OIDC setups configured outside the MCP); `'basic-auth'` / -`'no-auth'` overwrite it. +New apps default to HTTP basic authentication for safety. Pass `authentication_type='no-auth'` to expose publicly. On update, `authentication_type='default'` preserves the existing `authorization` block (including OIDC setups configured outside the MCP); `'basic-auth'` / `'no-auth'` overwrite it. ## Slug constraint - -Must be DNS-label-safe (lowercase letters, digits, hyphens, ≤63 chars). For drafts, append a -short suffix (e.g. `-draft-abc123`) to keep slugs unique across the prod and its drafts. +Must be DNS-label-safe (lowercase letters, digits, hyphens, ≤63 chars). For drafts, append a short suffix (e.g. `-draft-abc123`) to keep slugs unique across the prod and its drafts. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "name": { - "description": "Name of the data app (max ~50 chars to fit DNS label limit).", - "type": "string" + "type": "string", + "description": "Name of the data app (max ~50 chars to fit DNS label limit)." }, "description": { - "description": "Description of the data app.", - "type": "string" + "type": "string", + "description": "Description of the data app." }, "configuration_id": { "default": "", @@ -2293,6 +1747,7 @@ short suffix (e.g. `-draft-abc123`) to keep slugs unique across the prod and its "type": "string" }, "slug": { + "description": "URL-safe slug for the data app (used as a subdomain). Required when creating; immutable after.", "anyOf": [ { "type": "string" @@ -2300,11 +1755,10 @@ short suffix (e.g. `-draft-abc123`) to keep slugs unique across the prod and its { "type": "null" } - ], - "default": null, - "description": "URL-safe slug for the data app (used as a subdomain). Required when creating; immutable after." + ] }, "parent_configuration_id": { + "description": "Storage configuration ID of the prod python-js data app this draft will iterate against. When set on create, the new app is created as a **draft**: no managed repo is provisioned for it; instead its `parameters.dataApp.git` block is populated to point at the prod app's managed repo, with a freshly-minted prod-app HTTPS token and the chosen draft branch. Leave None on create to make a **prod app** (which gets its own managed repo). Rejected on update.", "anyOf": [ { "type": "string" @@ -2312,11 +1766,10 @@ short suffix (e.g. `-draft-abc123`) to keep slugs unique across the prod and its { "type": "null" } - ], - "default": null, - "description": "Storage configuration ID of the prod python-js data app this draft will iterate against. When set on create, the new app is created as a **draft**: no managed repo is provisioned for it; instead its `parameters.dataApp.git` block is populated to point at the prod app's managed repo, with a freshly-minted prod-app HTTPS token and the chosen draft branch. Leave None on create to make a **prod app** (which gets its own managed repo). Rejected on update." + ] }, "branch": { + "description": "Draft branch to pin the new draft to. Only valid on the draft create path (when `parent_configuration_id` is set). Defaults to `init` when unset. Must not be `main` (reserved for the prod app). Rejected on prod create and on update.", "anyOf": [ { "type": "string" @@ -2324,39 +1777,42 @@ short suffix (e.g. `-draft-abc123`) to keep slugs unique across the prod and its { "type": "null" } - ], - "default": null, - "description": "Draft branch to pin the new draft to. Only valid on the draft create path (when `parent_configuration_id` is set). Defaults to `init` when unset (a sensible name for the first draft of a brand-new prod app). For subsequent edit-existing drafts, pass a descriptive branch name like 'add-revenue-filter'. Must not be `main` (reserved for the prod app). Rejected on prod create and on update." + ] }, "authentication_type": { "default": "default", "description": "Authentication type. \"no-auth\" removes authentication completely, \"basic-auth\" secures the data app via HTTP basic authentication, and \"default\" means: on create, apply basic auth (safe default for new apps); on update, keep the existing authentication configuration (including OIDC setups configured outside the MCP).", + "type": "string", "enum": [ "no-auth", "basic-auth", "default" - ], - "type": "string" + ] }, "auto_suspend_after_seconds": { "default": 900, "description": "Number of seconds after which the running data app is automatically suspended.", - "type": "integer" + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 }, "storage": { + "description": "Complete storage configuration for the data app (input/output table mappings). Replaces the ENTIRE storage block when updating an existing app. Leave unset (None) to preserve the existing storage configuration; pass an empty dict to explicitly clear it.", "anyOf": [ { - "additionalProperties": true, - "type": "object" + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} }, { "type": "null" } - ], - "default": null, - "description": "Complete storage configuration for the data app (input/output table mappings). Validated against the storage JSON schema. Replaces the ENTIRE storage block when updating an existing app. For data apps with Storage Access, declare output tables with `unload_strategy: \"direct-grant\"` (in that case `source` is not required and the workspace is granted direct SELECT/INSERT/UPDATE/DELETE/TRUNCATE on the destination Storage table). Leave unset (None) to preserve the existing storage configuration; pass an empty dict to explicitly clear it." + ] }, "folder": { + "description": "Folder name to organize this data app in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more data apps in the project. If there are 20 or more data apps, you should assign one of the existing folders or create a new one that clearly reflects the data app purpose.", "anyOf": [ { "type": "string" @@ -2364,16 +1820,14 @@ short suffix (e.g. `-draft-abc123`) to keep slugs unique across the prod and its { "type": "null" } - ], - "default": null, - "description": "Folder name to organize this data app in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more data apps in the project. If there are 20 or more data apps, you should assign one of the existing folders or create a new one that clearly reflects the data app purpose." + ] } }, "required": [ "name", "description" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2389,63 +1843,49 @@ short suffix (e.g. `-draft-abc123`) to keep slugs unique across the prod and its Creates or updates a Streamlit data app. Considerations: -- The `source_code` parameter must be a complete and runnable Streamlit app. It must include a placeholder -`{QUERY_DATA_FUNCTION}` where a `query_data` function will be injected. This function queries the workspace to get -data, it accepts a string of SQL query following current sql dialect and returns a pandas DataFrame with the results -from the workspace. -- Write SQL queries so they are compatible with the current workspace backend, you can ensure this by using the -`query_data` tool to inspect the data in the workspace before using it in the data app. -- If you're updating an existing data app, provide the `configuration_id` parameter and the `change_description` -parameter. To keep existing data app values during an update, leave them as empty strings, lists, or None -appropriately based on the parameter type. -- After creating or updating a data app with this tool, ALWAYS call -`deploy_data_app(action="deploy", configuration_id=...)` to start a new app or restart an existing app so -changes take effect. Without this step, a newly created app will not start, and an existing app will keep -running the previous deployment without the latest changes. -- New apps use the HTTP basic authentication by default for security unless explicitly specified otherwise; when -updating, set `authentication_type` to `default` to keep the existing authentication type configuration -(including OIDC setups) unless explicitly specified otherwise. +- The `source_code` parameter must be a complete and runnable Streamlit app. It must include a placeholder `{QUERY_DATA_FUNCTION}` where a `query_data` function will be injected. This function queries the workspace to get data, it accepts a string of SQL query following current sql dialect and returns a pandas DataFrame with the results from the workspace. +- Write SQL queries so they are compatible with the current workspace backend, you can ensure this by using the `query_data` tool to inspect the data in the workspace before using it in the data app. +- If you're updating an existing data app, provide the `configuration_id` parameter and the `change_description` parameter. To keep existing data app values during an update, leave them as empty strings, lists, or None appropriately based on the parameter type. +- After creating or updating a data app with this tool, ALWAYS call `deploy_data_app(action="deploy", configuration_id=...)` to start a new app or restart an existing app so changes take effect. Without this step, a newly created app will not start, and an existing app will keep running the previous deployment without the latest changes. +- New apps use the HTTP basic authentication by default for security unless explicitly specified otherwise; when updating, set `authentication_type` to `default` to keep the existing authentication type configuration (including OIDC setups) unless explicitly specified otherwise. SQL & DATA TYPE RULES: -- Use delimited identifiers for the current SQL dialect for all column names and aliases in SQL. - Match the exact identifier case used in SQL when referencing columns in Python code. -- `query_data` RETURNS ALL COLUMNS AS STRINGS regardless of SQL CAST. Always convert types in Python after loading: -`df["col"] = pd.to_numeric(df["col"], errors="coerce").fillna(0)` and -`df["date"] = pd.to_datetime(df["date"], errors="coerce")`. +- Use delimited identifiers for the current SQL dialect for all column names and aliases in SQL. Match the exact identifier case used in SQL when referencing columns in Python code. +- `query_data` RETURNS ALL COLUMNS AS STRINGS regardless of SQL CAST. Always convert types in Python after loading: `df["col"] = pd.to_numeric(df["col"], errors="coerce").fillna(0)` and `df["date"] = pd.to_datetime(df["date"], errors="coerce")`. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "name": { - "description": "Name of the data app (max ~50 chars to fit DNS label limit).", - "type": "string" + "type": "string", + "description": "Name of the data app (max ~50 chars to fit DNS label limit)." }, "description": { - "description": "Description of the data app.", - "type": "string" + "type": "string", + "description": "Description of the data app." }, "source_code": { - "description": "Complete Python/Streamlit source code for the data app.", - "type": "string" + "type": "string", + "description": "Complete Python/Streamlit source code for the data app." }, "packages": { - "description": "Python packages used in the source code that will be installed by `pip install` into the environment before the code runs. For example: [\"pandas\", \"requests~=2.32\"].", + "type": "array", "items": { "type": "string" }, - "type": "array" + "description": "Python packages used in the source code that will be installed by `pip install` into the environment before the code runs. For example: [\"pandas\", \"requests~=2.32\"]." }, "authentication_type": { - "description": "Authentication type, \"no-auth\" removes authentication completely, \"basic-auth\" sets the data app to be secured using the HTTP basic authentication, and \"default\" keeps the existing authentication type when updating.", + "type": "string", "enum": [ "no-auth", "basic-auth", "default" ], - "type": "string" + "description": "Authentication type, \"no-auth\" removes authentication completely, \"basic-auth\" sets the data app to be secured using the HTTP basic authentication, and \"default\" keeps the existing authentication type when updating." }, "configuration_id": { "default": "", @@ -2458,6 +1898,7 @@ SQL & DATA TYPE RULES: "type": "string" }, "folder": { + "description": "Folder name to organize this data app in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more data apps in the project. If there are 20 or more data apps, you should assign one of the existing folders or create a new one that clearly reflects the data app purpose.", "anyOf": [ { "type": "string" @@ -2465,9 +1906,7 @@ SQL & DATA TYPE RULES: { "type": "null" } - ], - "default": null, - "description": "Folder name to organize this data app in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more data apps in the project. If there are 20 or more data apps, you should assign one of the existing folders or create a new one that clearly reflects the data app purpose." + ] } }, "required": [ @@ -2477,7 +1916,7 @@ SQL & DATA TYPE RULES: "packages", "authentication_type" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2498,17 +1937,17 @@ Answers a question using the Keboola documentation as a source. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "query": { - "description": "Natural language query to search for in the documentation.", - "type": "string" + "type": "string", + "description": "Natural language query to search for in the documentation." } }, "required": [ "query" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2544,31 +1983,37 @@ WHEN TO USE: **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "name": { - "description": "A short, descriptive name for the flow.", - "type": "string" + "type": "string", + "description": "A short, descriptive name for the flow." }, "description": { - "description": "Detailed description of the flow purpose.", - "type": "string" + "type": "string", + "description": "Detailed description of the flow purpose." }, "phases": { - "description": "List of phase definitions for conditional flows.", + "type": "array", "items": { - "additionalProperties": true, - "type": "object" + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} }, - "type": "array" + "description": "List of phase definitions for conditional flows." }, "tasks": { - "description": "List of task definitions for conditional flows.", + "type": "array", "items": { - "additionalProperties": true, - "type": "object" + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} }, - "type": "array" + "description": "List of task definitions for conditional flows." }, "folder": { "default": "", @@ -2582,7 +2027,7 @@ WHEN TO USE: "phases", "tasks" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2615,31 +2060,37 @@ WHEN TO USE: **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "name": { - "description": "A short, descriptive name for the flow.", - "type": "string" + "type": "string", + "description": "A short, descriptive name for the flow." }, "description": { - "description": "Detailed description of the flow purpose.", - "type": "string" + "type": "string", + "description": "Detailed description of the flow purpose." }, "phases": { - "description": "List of phase definitions.", + "type": "array", "items": { - "additionalProperties": true, - "type": "object" + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} }, - "type": "array" + "description": "List of phase definitions." }, "tasks": { - "description": "List of task definitions.", + "type": "array", "items": { - "additionalProperties": true, - "type": "object" + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} }, - "type": "array" + "description": "List of task definitions." }, "folder": { "default": "", @@ -2653,7 +2104,7 @@ WHEN TO USE: "phases", "tasks" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2680,21 +2131,21 @@ RULES: **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "flow_type": { - "description": "The type of the flow to retrieve examples for.", + "type": "string", "enum": [ "keboola.flow", "keboola.orchestrator" ], - "type": "string" + "description": "The type of the flow to retrieve examples for." } }, "required": [ "flow_type" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2721,21 +2172,21 @@ RULES: **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "flow_type": { - "description": "The type of flow for which to fetch schema.", + "type": "string", "enum": [ "keboola.flow", "keboola.orchestrator" ], - "type": "string" + "description": "The type of flow for which to fetch schema." } }, "required": [ "flow_type" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2763,18 +2214,18 @@ OPTIONS: **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "flow_ids": { "default": [], "description": "IDs of flows to retrieve full details for. When provided (non-empty), returns full flow configurations including phases and tasks. When empty [], lists all flows in the project as summaries.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } } }, - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -2814,117 +2265,64 @@ LEGACY FLOWS (`keboola.orchestrator`): WHEN TO USE: - Renaming a flow, updating descriptions, adding/removing phases or tasks, updating schedules, adjusting dependencies, or enabling/disabling flow execution - - -**Input JSON Schema**: -```json -{ - "$defs": { - "ScheduleRequest": { - "properties": { - "action": { - "description": "Action to perform on the schedule.", - "enum": [ - "add", - "update", - "remove" - ], - "type": "string" - }, - "scheduleId": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "ID of the schedule configuration to update. None if creating a new schedule." - }, - "timezone": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Timezone for the schedule. Default UTC if None provided." - }, - "cronTab": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Cron expression for the schedule following the format: `* * * * *`.Where 1. minutes, 2. hours, 3. days of month, 4. months, 5. days of week. Example: `15,45 1,13 * * 0`" - }, - "state": { - "anyOf": [ - { - "enum": [ - "enabled", - "disabled" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Enable or disable the schedule." - } - }, - "required": [ - "action" - ], - "type": "object" - } - }, - "additionalProperties": false, + + +**Input JSON Schema**: +```json +{ + "type": "object", "properties": { "configuration_id": { - "description": "ID of the flow configuration.", - "type": "string" + "type": "string", + "description": "ID of the flow configuration." }, "flow_type": { - "description": "The type of flow to update. Use \"keboola.flow\" for conditional flows or \"keboola.orchestrator\" for legacy flows. This MUST match the existing flow type.", + "type": "string", "enum": [ "keboola.flow", "keboola.orchestrator" ], - "type": "string" + "description": "The type of flow to update. Use \"keboola.flow\" for conditional flows or \"keboola.orchestrator\" for legacy flows. This MUST match the existing flow type." }, "change_description": { - "description": "Description of changes made.", - "type": "string" + "type": "string", + "description": "Description of changes made." }, "phases": { - "default": null, "description": "Updated list of phase definitions.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + { + "type": "null" + } + ] }, "tasks": { - "default": null, "description": "Updated list of task definitions.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + { + "type": "null" + } + ] }, "name": { "default": "", @@ -2939,12 +2337,75 @@ adjusting dependencies, or enabling/disabling flow execution "schedules": { "default": [], "description": "Optional sequence of schedule requests to add/update/remove schedules for this flow. Each request must have \"action\": \"add\"|\"update\"|\"remove\". For add: include \"cron_tab\", \"state\" (\"enabled\"|\"disabled\"), \"timezone\". For update/remove: include \"schedule_id\". Example: [{\"action\": \"add\", \"cron_tab\": \"0 8 * * 1-5\", \"state\": \"enabled\", \"timezone\": \"UTC\"}]", + "type": "array", "items": { - "$ref": "#/$defs/ScheduleRequest" - }, - "type": "array" + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "add", + "update", + "remove" + ], + "description": "Action to perform on the schedule." + }, + "schedule_id": { + "description": "ID of the schedule configuration to update. None if creating a new schedule.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "timezone": { + "description": "Timezone for the schedule. Default UTC if None provided.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "cron_tab": { + "description": "Cron expression for the schedule following the format: `* * * * *`.Where 1. minutes, 2. hours, 3. days of month, 4. months, 5. days of week. Example: `15,45 1,13 * * 0`", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "state": { + "description": "Enable or disable the schedule.", + "anyOf": [ + { + "type": "string", + "enum": [ + "enabled", + "disabled" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "action" + ] + } }, "is_disabled": { + "description": "Enable or disable the flow. Set to True to disable execution (flow won't run), False to enable execution (flow will run). Only provide if changing the status, leave as null to preserve current state.", "anyOf": [ { "type": "boolean" @@ -2952,11 +2413,10 @@ adjusting dependencies, or enabling/disabling flow execution { "type": "null" } - ], - "default": null, - "description": "Enable or disable the flow. Set to True to disable execution (flow won't run), False to enable execution (flow will run). Only provide if changing the status, leave as null to preserve current state." + ] }, "folder": { + "description": "Folder name to organize this flow in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more flows in the project. If there are 20 or more flows, you should assign one of the existing folders or create a new one that clearly reflects the flow purpose.", "anyOf": [ { "type": "string" @@ -2964,9 +2424,7 @@ adjusting dependencies, or enabling/disabling flow execution { "type": "null" } - ], - "default": null, - "description": "Folder name to organize this flow in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more flows in the project. If there are 20 or more flows, you should assign one of the existing folders or create a new one that clearly reflects the flow purpose." + ] } }, "required": [ @@ -2974,7 +2432,7 @@ adjusting dependencies, or enabling/disabling flow execution "flow_type", "change_description" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3016,41 +2474,59 @@ or enabling/disabling flow execution **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "configuration_id": { - "description": "ID of the flow configuration.", - "type": "string" + "type": "string", + "description": "ID of the flow configuration." }, "flow_type": { - "description": "The type of flow to update. Use \"keboola.flow\" for conditional flows or \"keboola.orchestrator\" for legacy flows. This MUST match the existing flow type.", + "type": "string", "enum": [ "keboola.flow", "keboola.orchestrator" ], - "type": "string" + "description": "The type of flow to update. Use \"keboola.flow\" for conditional flows or \"keboola.orchestrator\" for legacy flows. This MUST match the existing flow type." }, "change_description": { - "description": "Description of changes made.", - "type": "string" + "type": "string", + "description": "Description of changes made." }, "phases": { - "default": null, "description": "Updated list of phase definitions.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + { + "type": "null" + } + ] }, "tasks": { - "default": null, "description": "Updated list of task definitions.", - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + { + "type": "null" + } + ] }, "name": { "default": "", @@ -3063,6 +2539,7 @@ or enabling/disabling flow execution "type": "string" }, "is_disabled": { + "description": "Enable or disable the flow. Set to True to disable execution (flow won't run), False to enable execution (flow will run). Only provide if changing the status, leave as null to preserve current state.", "anyOf": [ { "type": "boolean" @@ -3070,11 +2547,10 @@ or enabling/disabling flow execution { "type": "null" } - ], - "default": null, - "description": "Enable or disable the flow. Set to True to disable execution (flow won't run), False to enable execution (flow will run). Only provide if changing the status, leave as null to preserve current state." + ] }, "folder": { + "description": "Folder name to organize this flow in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more flows in the project. If there are 20 or more flows, you should assign one of the existing folders or create a new one that clearly reflects the flow purpose.", "anyOf": [ { "type": "string" @@ -3082,9 +2558,7 @@ or enabling/disabling flow execution { "type": "null" } - ], - "default": null, - "description": "Folder name to organize this flow in the Keboola UI. Pass an empty string to remove an existing folder assignment. Existing folder names are returned in the response change_summary when no folder is provided and there are 20 or more flows in the project. If there are 20 or more flows, you should assign one of the existing folders or create a new one that clearly reflects the flow purpose." + ] } }, "required": [ @@ -3092,7 +2566,7 @@ or enabling/disabling flow execution "flow_type", "change_description" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3109,85 +2583,23 @@ or enabling/disabling flow execution Retrieves job execution information from the Keboola project. -CONTEXT: -Jobs in Keboola are execution records of components (extractors, transformations, writers, flows). -Each job represents a single run with its status, timing, configuration, and results. - -TWO MODES OF OPERATION (controlled by job_ids parameter): - -MODE 1: GET DETAILS FOR SPECIFIC JOBS (job_ids is non-empty) -- Provide one or more job IDs: job_ids=["12345", "67890"] -- Returns: FULL details for each job including status, config_data, results, timing, and metadata -- Ignores: All filtering/sorting parameters (status, component_id, config_id, limit, offset, sort_by, sort_order) -- Use when: You know specific job IDs and need complete information about them - -MODE 2: LIST/SEARCH JOBS (job_ids is empty) -- Leave job_ids empty: job_ids=[] -- Returns: SUMMARY list of jobs (id, status, component_id, config_id, timing only - no config_data or results) -- Supports: Filtering by status/component_id/config_id, pagination with limit/offset, sorting -- Use when: You need to find jobs, see recent executions, or monitor job history - -DECISION GUIDE: -- Start with MODE 2 (list) to find jobs → then use MODE 1 (details) if you need full information -- If you already know job IDs → use MODE 1 directly -- For monitoring/browsing → use MODE 2 with filters - -NOTE: Jobs cannot be found by name using the `search` tool. However, always use the filtering -parameters (status, component_id, config_id) to narrow results rather than listing all jobs -with no filters. If you need to find jobs for a specific configuration but only know its name, -first use `search` to find the configuration ID, then filter jobs by that config_id. - -COMMON WORKFLOWS: -1. Find failed jobs: job_ids=[], status="error" → identify problematic job IDs → get details with MODE 1 -2. Check recent runs: job_ids=[], component_id="...", limit=10 → see latest executions -3. Monitor specific job: job_ids=["123"] → poll for status and results -4. Troubleshoot config: job_ids=[], component_id="...", config_id="...", status="error" → find which runs failed - -EXAMPLES: - -MODE 1 - Get full details: -- job_ids=["12345"] → detailed info for job 12345 -- job_ids=["12345", "67890"] → detailed info for multiple jobs - -MODE 2 - List/search jobs: -- job_ids=[] → list latest 100 jobs (default) -- job_ids=[], status="error" → list only failed jobs -- job_ids=[], status="processing" → list currently running jobs -- job_ids=[], component_id="keboola.ex-aws-s3" → list jobs for S3 extractor -- job_ids=[], component_id="keboola.ex-aws-s3", config_id="12345" → list jobs for specific configuration -- job_ids=[], limit=50, offset=100 → pagination (skip first 100, get next 50) -- job_ids=[], sort_by="endTime", sort_order="asc" → oldest completed first -- job_ids=[], sort_by="durationSeconds", sort_order="desc" → longest running first - -LOG RETRIEVAL (only in MODE 1): -- Set include_logs=True to fetch execution logs from the Storage API events -- Logs are fetched using the job's runId and returned in chronological order -- Use log_tail_lines to control how many recent log events to return (default 50, max 500) -- Use log_event_types to filter by event type: ["error"] for just errors, ["error", "warn"] for errors and warnings -- If a job has no runId (e.g., not yet started), logs will be None - -EXAMPLES WITH LOGS: -- job_ids=["12345"], include_logs=True → job details + last 50 log events -- job_ids=["12345"], include_logs=True, log_event_types=["error"] → job details + only error events -- job_ids=["12345"], include_logs=True, log_tail_lines=200 → job details + last 200 log events - **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "job_ids": { "default": [], - "description": "IDs of jobs to retrieve full details for. When provided (non-empty), returns full job details including status, parameters, results, and metadata. When empty [], lists jobs in the project as summaries with optional filtering.", + "description": "IDs of jobs to retrieve full details for; empty lists jobs as summaries.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } }, "status": { - "default": null, - "description": "The optional status of the jobs to filter by when listing (ignored if job_ids is provided). If None then all statuses are included.", + "description": "Filter listed jobs by status (ignored if job_ids given).", + "type": "string", "enum": [ "waiting", "processing", @@ -3198,88 +2610,78 @@ EXAMPLES WITH LOGS: "terminating", "cancelled", "terminated" - ], - "type": "string" + ] }, "component_id": { - "default": null, - "description": "The optional ID of the component whose jobs you want to list (ignored if job_ids is provided). Default = None.", + "description": "Filter listed jobs by component id (ignored if job_ids given).", "type": "string" }, "config_id": { - "default": null, - "description": "The optional ID of the component configuration whose jobs you want to list (ignored if job_ids is provided). Default = None.", + "description": "Filter listed jobs by configuration id (ignored if job_ids given).", "type": "string" }, "limit": { "default": 100, - "description": "The number of jobs to list when listing (ignored if job_ids is provided), default = 100, max = 500.", - "maximum": 500, + "description": "Number of jobs to list (max 500).", + "type": "integer", "minimum": 1, - "type": "integer" + "maximum": 500 }, "offset": { "default": 0, - "description": "The offset of the jobs to list when listing (ignored if job_ids is provided), default = 0.", + "description": "Offset of jobs to list.", + "type": "integer", "minimum": 0, - "type": "integer" + "maximum": 9007199254740991 }, "sort_by": { "default": "startTime", - "description": "The field to sort the jobs by when listing (ignored if job_ids is provided), default = \"startTime\".", + "description": "Field to sort listed jobs by.", + "type": "string", "enum": [ "startTime", "endTime", "createdTime", "durationSeconds", "id" - ], - "type": "string" + ] }, "sort_order": { "default": "desc", - "description": "The order to sort the jobs by when listing (ignored if job_ids is provided), default = \"desc\".", + "description": "Sort order for listed jobs.", + "type": "string", "enum": [ "asc", "desc" - ], - "type": "string" + ] }, "include_logs": { "default": false, - "description": "Whether to include execution logs for each job. Only used when job_ids is provided (MODE 1). Logs are fetched from the Storage API events endpoint using the job's runId. Default is False.", + "description": "Include execution logs (only when job_ids given).", "type": "boolean" }, "log_tail_lines": { "default": 50, - "description": "Maximum number of log events to return per job (most recent first). Only used when include_logs=True. Default = 50, max = 500.", - "maximum": 500, + "description": "Max log events per job (most recent).", + "type": "integer", "minimum": 1, - "type": "integer" + "maximum": 500 }, "log_event_types": { - "anyOf": [ - { - "items": { - "enum": [ - "info", - "warn", - "error", - "success" - ], - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Filter log events by type. Only used when include_logs=True. If None, all event types are included. Example: [\"error\"] to only show errors, [\"error\", \"warn\"] for errors and warnings." + "description": "Filter log events by type (only when include_logs=true).", + "type": "array", + "items": { + "type": "string", + "enum": [ + "info", + "warn", + "error", + "success" + ] + } } }, - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3298,37 +2700,29 @@ Starts a new job for a given component or transformation. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "component_id": { - "description": "The ID of the component or transformation for which to start a job.", - "type": "string" + "type": "string", + "description": "The ID of the component or transformation to start a job for." }, "configuration_id": { - "description": "The ID of the configuration for which to start a job.", - "type": "string" + "type": "string", + "description": "The ID of the configuration to start a job for." }, "configuration_row_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional list of configuration row IDs to run. If not provided, all rows are executed." + "description": "Optional configuration row IDs to run; if omitted, all rows are executed.", + "type": "array", + "items": { + "type": "string" + } } }, "required": [ "component_id", "configuration_id" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3337,7 +2731,7 @@ Starts a new job for a given component or transformation. # OAuth Tools ## create_oauth_url -**Annotations**: `destructive` +**Annotations**: **Tags**: `oauth` @@ -3345,32 +2739,26 @@ Starts a new job for a given component or transformation. Generates an OAuth authorization URL for a Keboola component configuration. -When using this tool, be very concise in your response. Just guide the user to click the -authorization link. - -Note that this tool should be called specifically for the OAuth-requiring components after their -configuration is created e.g. keboola.ex-google-analytics-v4 and keboola.ex-gmail. - **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "component_id": { - "description": "The component ID to grant access to (e.g., \"keboola.ex-google-analytics-v4\").", - "type": "string" + "type": "string", + "description": "The component ID to grant access to (e.g., \"keboola.ex-google-analytics-v4\")." }, "config_id": { - "description": "The configuration ID for the component.", - "type": "string" + "type": "string", + "description": "The configuration ID for the component." } }, "required": [ "component_id", "config_id" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3385,20 +2773,17 @@ configuration is created e.g. keboola.ex-google-analytics-v4 and keboola.ex-gmai **Description**: -Retrieves structured information about the current project, -including essential context and base instructions for working with it -(e.g., transformations, components, workflows, and dependencies). +Retrieves structured information about the current project, including essential context and base instructions for working with it (e.g., transformations, components, workflows, and dependencies). -Always call this tool at least once at the start of a conversation -to establish the project context before using other tools. +Always call this tool at least once at the start of a conversation to establish the project context before using other tools. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": {}, - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3417,17 +2802,17 @@ Updates the description of the current Keboola project. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "description": { - "description": "The new project description text.", - "type": "string" + "type": "string", + "description": "The new project description text." } }, "required": [ "description" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3442,30 +2827,23 @@ Updates the description of the current Keboola project. **Description**: -Returns list of component IDs that match the given query. - -WHEN TO USE: -- Use when you want to find the component for a specific purpose. - -USAGE EXAMPLES: -- user_input: "I am looking for a salesforce extractor component" - → Returns a list of component IDs that match the query, ordered by relevance/best match. +Returns a list of component IDs that match the given natural-language query. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "query": { - "description": "Natural language query to find the requested component.", - "type": "string" + "type": "string", + "description": "Natural language query to find the requested component." } }, "required": [ "query" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3478,147 +2856,27 @@ USAGE EXAMPLES: **Description**: -Searches for Keboola items (tables, buckets, components, configurations, transformations, flows, data-apps, etc.) -in the current project and returns matching ID + metadata. - -This tool supports two complementary search types: - -1) textual -- Searches items by name, server-side (fast, independent of project size). -- Tokenized full-text name matching, case- and diacritics-insensitive. Pass the plain name; do NOT build - regex (rejected). It is NOT typo-corrected — misspellings may not match. -- Prefers the current branch context; when nothing is found there, automatically widens the search to all - branches of the project — such hits carry `branch_id`/`branch_name` so you can tell where they live. - -2) config-based -- Searches item configurations (JSON objects) by matching patterns against the configuration values ​​converted - to a string, optionally narrowed by JSON path `scopes`. -- Returns also `match_scopes` with JSON paths and matched patterns per scope. - -THIS IS THE PRIMARY DISCOVERY TOOL. Always use it BEFORE any get_* tool when you need to find items -by name or specific configuration content. Do NOT enumerate items with get_buckets, get_tables, get_configs, -get_flows, or get_data_apps just to locate a specific item — use this tool instead. - -WHEN TO USE: -- User asks to "find", "locate", or "search for" something by name, keyword, text pattern, configuration content or -value -- User mentions a partial name and you need to find the full item (e.g., "find the customer table") -- User asks "what tables/configs/flows do I have with X in the name?" -- You need to discover items before performing operations on them -- User asks to "list all items with [name] or [configuration value/part] in it" -- User asks where a value, table, component, specific configuration ID, or specific settings is used in components, -data-apps, flows, or transformations -- You need to trace lineage by searching for IDs referenced in configurations, or to find flows using a - specific component, or find usage of a bucket/table in transformations or components, or to find items with - specific parameters. -- User asks to "what is the genesis of this item?" or "explain me business logic of this item?" - -HOW IT WORKS: -- Supports two types: - - search_type="textual": tokenized full-text name search, server-side. Names only — descriptions, column - names, IDs and configuration contents are NOT searched (use config-based search for configuration contents, - or get_tables for columns). Matching is case- and diacritics-insensitive but NOT typo-corrected. - - search_type="config-based": matches inside configuration JSON objects, optionally narrowed by JSON path `scopes` -- case-insensitive search -- mode for pattern search: applies to config-based only — `literal` (default) or `regex`. Textual search ignores - `mode` (always full-text) and rejects `regex`. -- Multiple patterns work as OR condition - matches items containing ANY of the patterns -- Each result includes the item's ID, name, creation date, and relevant metadata; the response also carries - `total` and `by_type` counts and the `branch_scope` the hits come from -- textual search prefers the current branch; on zero hits it automatically retries across all branches of the - project and marks the response with branch_scope="all-branches" -- scopes (config-based) narrow matching to specific JSONPath areas within configurations; matching is performed - against the stringified JSON node content in those areas. -- config-based always returns all matched paths per item in `match_scopes` (including matched patterns) - -IMPORTANT: -- Always use this tool when the user mentions a name but you don't have the exact ID -- The search returns IDs that you can use with other tools (e.g., get_tables, get_configs, get_flows) -- Results are ordered by the `updated` field, most recent first. `updated` is the item's last update time - when available, or its creation time otherwise (textual/global-search hits expose only the creation time). -- Textual search matches names only, with tokenized full-text matching (case/diacritics-insensitive; not - typo-corrected; no regex). It may not return every item the legacy enumeration did. To find items by - description or by table column, use get_tables; to find items by configuration content, use config-based search. -- For exact ID lookups, use specific tools like get_tables, get_configs, get_flows instead -- Use specific `scopes` only when you know the config structure (schema or real example); otherwise run config-based - search without scopes. -- Use find_component_id and get_configs tools to find configurations related to a specific component -- If results are too numerous or empty, ask the user to refine their query rather than enumerating all items. - -USAGE EXAMPLES: -1) textual search examples: -- user_input: "Find all tables with 'customer' in the name" - → patterns=["customer"], item_types=["table"] - → Returns all tables whose name matches "customer" - -- user_input: "Search for the sales transformation" - → patterns=["sales"], item_types=["transformation"] - → Returns transformations with "sales" in the name - -- user_input: "Find items named 'daily report' or 'weekly summary'" - → patterns=["daily report", "weekly summary"], item_types=[] - → Returns all items matching any of these patterns - -- user_input: "Show me all configurations related to Google Analytics" - → patterns=["google analytics"], item_types=["configuration"] - → Returns configurations with matching names - -2) config-based search examples: -- user_input: "Find transformations/configs/components referencing table in.c-prod.customers" - -> patterns=["in.c-prod.customers"], item_types=["transformation", "configuration"], - search_type="config-based" - -> No scopes = search whole stringified config; result includes `match_scopes` with exact paths + patterns - -- user_input: "Find configurations/transformations (etc.) using specific setting / id anywhere" - -> patterns=["setting", "id"], item_types=["configuration", "transformations"], search_type="config-based", - -- user_input: "Find configurations/transformations (etc.) using specific setting / id in parameters" --> patterns=["setting", "id"], item_types=["configuration", "transformations"], search_type="config-based", -scopes=["parameters"] - -- user_input: "Find configurations/transformations (etc.) using specific setting / id in storage" --> patterns=["setting", "id"], item_types=["configuration", "transformations"], search_type="config-based", -scopes=["storage"] - -- user_input: "Find configurations/transformations (etc.) using specific setting / id in authorization" - -> patterns=["setting", "id"], item_types=["configuration", "transformations"], search_type="config-based", - scopes=["parameters.authorization", "authorization"] - -- user_input: "Find components/transformations using my_bucket in input or output mappings" - -> patterns=["my_bucket"], item_types=["configuration", "transformation"], search_type="config-based", - scopes=["storage.input", "storage.output"] - -> Returns matches with paths like `storage.input.tables[0].source`, `storage.input.files[0].source`, - or `storage.output.tables[0].destination` - -- user_input: "Find flows using configuration ID 01k9cz233cvd1rga3zzx40g8qj" - -> patterns=["01k9cz233cvd1rga3zzx40g8qj"], item_types=["flow"], search_type="config-based", - scopes=["tasks", "phases"] - -- user_input: "Find transformations using this table / column / specific code in its script" - -> patterns=["element"], item_types=["transformation"], search_type="config-based", - scopes=["parameters", "storage"] - -- user_input: "Find data apps using something in its config / python code / setting" - -> patterns=["something"], item_types=["data-app"], search_type="config-based" - -> Returns data apps where script/config sections contain the keyword and includes `match_scopes` +Searches for Keboola items (tables, buckets, components, configurations, transformations, flows, data-apps, etc.) in the current project and returns matching IDs and metadata. Supports textual search (matches item names, server-side) and config-based search (matches patterns against the configuration JSON content, optionally narrowed by JSONPath scopes). THIS IS THE PRIMARY DISCOVERY TOOL — use it before any get_* tool when you need to find items by name or configuration content. Multiple patterns work as an OR condition. Textual search prefers the current branch and, when nothing is found there, automatically widens to all branches of the project. **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "patterns": { - "description": "One or more search patterns. For textual search they match item names (server-side, tokenized full-text); for config-based search they match the configuration JSON content. Case-insensitive by default. Examples: [\"customer\"], [\"sales\", \"revenue\"], [\"my_bucket\"]. Do not use empty strings or empty lists.", + "type": "array", "items": { "type": "string" }, - "type": "array" + "description": "One or more search patterns. For textual search they match item names (server-side, tokenized full-text); for config-based search they match the configuration JSON content. Case-insensitive by default. Examples: [\"customer\"], [\"sales\", \"revenue\"], [\"my_bucket\"]. Do not use empty strings or empty lists." }, "item_types": { "default": [], "description": "Filter for specific Keboola item types. Common values: \"table\" (data tables), \"bucket\" (table containers), \"transformation\" (SQL/Python transformations), \"component\" (extractor/writer/application components), \"data-app\" (data apps), \"flow\" (orchestration flows). Use when you know what type of item you're looking for or leave empty to search all types.", + "type": "array", "items": { + "type": "string", "enum": [ "bucket", "table", @@ -3632,52 +2890,50 @@ scopes=["storage"] "shared-code", "rows", "state" - ], - "type": "string" - }, - "type": "array" + ] + } }, "search_type": { "default": "textual", "description": "Search mode: \"textual\" (name/id/description) or \"config-based\" (stringified configuration payloads). (default: \"textual\")", + "type": "string", "enum": [ "textual", "config-based" - ], - "type": "string" + ] }, "scopes": { "default": [], "description": "JSONPath expressions to narrow config-based search to specific parts of the configuration. Simple dot-notation (e.g. \"parameters\", \"storage.input\") and full JSONPath (e.g. \"$.tasks[*]\") are both supported (e.g. \"parameters.host\", \"storage.input[0].source\"). Leave empty to search the whole configuration.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } }, "mode": { "default": "literal", "description": "How to interpret patterns. Applies to config-based search only: \"regex\" for regular expressions or \"literal\" for exact text (default: \"literal\"). Ignored by textual search, which is always a tokenized full-text name query (not typo-corrected) and rejects \"regex\".", + "type": "string", "enum": [ "regex", "literal" - ], - "type": "string" + ] }, "limit": { "default": 50, "description": "Maximum number of items to return (default: 50, max: 100).", - "type": "integer" + "type": "number" }, "offset": { "default": 0, "description": "Number of matching items to skip for pagination (default: 0).", - "type": "integer" + "type": "number" } }, "required": [ "patterns" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3723,62 +2979,53 @@ EXAMPLES: **Input JSON Schema**: ```json { - "$defs": { - "SemanticObjectType": { - "enum": [ - "semantic-model", - "semantic-dataset", - "semantic-metric", - "semantic-relationship", - "semantic-glossary", - "semantic-constraint" - ], - "type": "string" - }, - "SemanticObjectTypeSelection": { - "description": "Semantic object type selection used by semantic tools.", - "properties": { - "object_type": { - "$ref": "#/$defs/SemanticObjectType", - "description": "Semantic object type to load." - }, - "ids": { - "default": [], - "description": "Specific object UUIDs to include. Empty list [] means include all objects of this type.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "required": [ - "object_type" - ], - "type": "object" - } - }, - "additionalProperties": false, + "type": "object", "properties": { "semantic_objects": { - "description": "List of semantic object selections to load. Each item contains \"object_type\" and optional \"ids\". If \"ids\" is empty, all objects of that type are returned in compact form. If \"ids\" is non-empty, only those objects are returned with full attributes.", + "type": "array", "items": { - "$ref": "#/$defs/SemanticObjectTypeSelection" + "type": "object", + "properties": { + "object_type": { + "type": "string", + "enum": [ + "semantic-model", + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-glossary", + "semantic-constraint" + ], + "description": "Semantic object type to load." + }, + "ids": { + "default": [], + "description": "Specific object UUIDs to include. Empty list [] means include all objects of this type.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "object_type" + ] }, - "type": "array" + "description": "List of semantic object selections to load. Each item contains \"object_type\" and optional \"ids\". If \"ids\" is empty, all objects of that type are returned in compact form. If \"ids\" is non-empty, only those objects are returned with full attributes." }, "semantic_model_ids": { "default": [], "description": "Optional list of semantic model IDs to restrict loading to specific models. Empty list [] means load across all semantic models.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } } }, "required": [ "semantic_objects" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3800,33 +3047,28 @@ WHEN TO USE: **Input JSON Schema**: ```json { - "$defs": { - "SemanticObjectType": { - "enum": [ - "semantic-model", - "semantic-dataset", - "semantic-metric", - "semantic-relationship", - "semantic-glossary", - "semantic-constraint" - ], - "type": "string" - } - }, - "additionalProperties": false, + "type": "object", "properties": { "semantic_types": { - "description": "List of semantic object types for which JSON schemas should be returned. Each returned item contains the requested semantic type and its metastore schema.", + "type": "array", "items": { - "$ref": "#/$defs/SemanticObjectType" + "type": "string", + "enum": [ + "semantic-model", + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-glossary", + "semantic-constraint" + ] }, - "type": "array" + "description": "List of semantic object types for which JSON schemas should be returned. Each returned item contains the requested semantic type and its metastore schema." } }, "required": [ "semantic_types" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3882,43 +3124,38 @@ EXAMPLES: **Input JSON Schema**: ```json { - "$defs": { - "SemanticObjectType": { - "enum": [ - "semantic-model", - "semantic-dataset", - "semantic-metric", - "semantic-relationship", - "semantic-glossary", - "semantic-constraint" - ], - "type": "string" - } - }, - "additionalProperties": false, + "type": "object", "properties": { "patterns": { - "description": "One or more regex patterns used to search semantic metadata. The search checks semantic model names plus semantic object names and nested attribute values. Use multiple patterns when you need to find objects related to several business terms at once.", + "type": "array", "items": { "type": "string" }, - "type": "array" + "description": "One or more regex patterns used to search semantic metadata. The search checks semantic model names plus semantic object names and nested attribute values. Use multiple patterns when you need to find objects related to several business terms at once." }, "semantic_types": { "default": [], "description": "Optional semantic object types to search. Empty list [] means ALL semantic object types are searched. Use this to narrow the search when you already know whether you want datasets, metrics, relationships, glossary terms, constraints, or models.", + "type": "array", "items": { - "$ref": "#/$defs/SemanticObjectType" - }, - "type": "array" + "type": "string", + "enum": [ + "semantic-model", + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-glossary", + "semantic-constraint" + ] + } }, "semantic_model_ids": { "default": [], "description": "Optional list of semantic model IDs to restrict the search to specific models. Empty list [] means search across all semantic models.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } }, "case_sensitive": { "default": false, @@ -3928,13 +3165,15 @@ EXAMPLES: "max_results": { "default": 100, "description": "Maximum number of matched semantic objects to return. Use a smaller value for quick discovery and a larger value only when you need a broader result set.", - "type": "integer" + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 } }, "required": [ "patterns" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -3990,67 +3229,58 @@ EXAMPLES: **Input JSON Schema**: ```json { - "$defs": { - "SemanticObjectType": { - "enum": [ - "semantic-model", - "semantic-dataset", - "semantic-metric", - "semantic-relationship", - "semantic-glossary", - "semantic-constraint" - ], - "type": "string" - }, - "SemanticObjectTypeSelection": { - "description": "Semantic object type selection used by semantic tools.", - "properties": { - "object_type": { - "$ref": "#/$defs/SemanticObjectType", - "description": "Semantic object type to load." - }, - "ids": { - "default": [], - "description": "Specific object UUIDs to include. Empty list [] means include all objects of this type.", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "required": [ - "object_type" - ], - "type": "object" - } - }, - "additionalProperties": false, + "type": "object", "properties": { "sql_query": { - "description": "SQL query that should be checked against the semantic layer. The query is not executed; the tool performs best-effort semantic detection and rule validation using heuristic string matching, so the detected objects may be incomplete or imperfect.", - "type": "string" + "type": "string", + "description": "SQL query that should be checked against the semantic layer. The query is not executed; the tool performs best-effort semantic detection and rule validation using heuristic string matching, so the detected objects may be incomplete or imperfect." }, "semantic_model_ids": { - "description": "One or more semantic model IDs against which the SQL should be validated. Contexts from all models are merged into a single universe for object detection. Constraint evaluation is performed per model to avoid cross-model rule contamination.", + "type": "array", "items": { "type": "string" }, - "type": "array" + "description": "One or more semantic model IDs against which the SQL should be validated. Contexts from all models are merged into a single universe for object detection. Constraint evaluation is performed per model to avoid cross-model rule contamination." }, "expected_semantic_objects": { "default": [], "description": "Optional semantic object selections that define the expected semantic scope of the query. These expectations are compared with the objects actually detected in the SQL. Use `ids` when you want to assert that specific semantic objects should be present.", + "type": "array", "items": { - "$ref": "#/$defs/SemanticObjectTypeSelection" - }, - "type": "array" + "type": "object", + "properties": { + "object_type": { + "type": "string", + "enum": [ + "semantic-model", + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-glossary", + "semantic-constraint" + ], + "description": "Semantic object type to load." + }, + "ids": { + "default": [], + "description": "Specific object UUIDs to include. Empty list [] means include all objects of this type.", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "object_type" + ] + } } }, "required": [ "sql_query", "semantic_model_ids" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -4065,66 +3295,68 @@ EXAMPLES: **Description**: -Executes an SQL SELECT query to get the data from the underlying database. -BEFORE QUERYING: -* Always verify the table has a non-null fullyQualifiedName from get_tables tool. - If it does not, the table is not SQL-accessible from this workspace — do not attempt the query and inform user. + Executes an SQL SELECT query to get the data from the underlying database. -CRITICAL SQL REQUIREMENTS: + BEFORE QUERYING: + * Always verify the table has a non-null fullyQualifiedName from get_tables tool. + If it does not, the table is not SQL-accessible from this workspace — do not attempt the query and inform user. -* ALWAYS check the SQL dialect before constructing queries. -* Do not include any comments in the SQL code -* Use delimited identifiers and FQN format for the current SQL dialect. + CRITICAL SQL REQUIREMENTS: -TABLE AND COLUMN REFERENCES: -* Always use fully qualified table names in the exact FQN format provided by table information tools -* Follow the identifier structure exactly as shown by table info tools for the current SQL dialect -* Always use delimited identifiers when referring to table columns + * ALWAYS check the SQL dialect before constructing queries. + * Do not include any comments in the SQL code + * Use delimited identifiers and FQN format for the current SQL dialect. -CTE (WITH CLAUSE) RULES: -* ALL column references in main query MUST match exact case used in the CTE -* If you alias a column in a CTE, reference it under the aliased name in the subsequent queries -* Define all column aliases explicitly in CTEs -* Use delimited identifiers in both CTE definition and references to preserve case + TABLE AND COLUMN REFERENCES: + * Always use fully qualified table names in the exact FQN format provided by table information tools + * Follow the identifier structure exactly as shown by table info tools for the current SQL dialect + * Always use delimited identifiers when referring to table columns -FUNCTION COMPATIBILITY: -* Check data types before using date functions (DATE_TRUNC, EXTRACT require proper date/timestamp types) -* Cast VARCHAR columns to appropriate types before using in date/numeric functions + CTE (WITH CLAUSE) RULES: + * ALL column references in main query MUST match exact case used in the CTE + * If you alias a column in a CTE, reference it under the aliased name in the subsequent queries + * Define all column aliases explicitly in CTEs + * Use delimited identifiers in both CTE definition and references to preserve case -ERROR PREVENTION: -* Never pass empty strings ('') where numeric or date values are expected -* Use NULLIF or CASE statements to handle empty values -* Always use TRY_CAST or similar safe casting functions when converting data types -* Check for division by zero using NULLIF(denominator, 0) -* Always use the LIMIT clause in your SELECT statements when fetching data. There are hard limits imposed - by this tool on the maximum number of rows that can be fetched and the maximum number of characters. - The tool will truncate the data if those limits are exceeded. + FUNCTION COMPATIBILITY: + * Check data types before using date functions (DATE_TRUNC, EXTRACT require proper date/timestamp types) + * Cast VARCHAR columns to appropriate types before using in date/numeric functions -DATA VALIDATION: -* When querying columns with categorical values, use query_data tool to inspect distinct values beforehand -* Ensure valid filtering by checking actual data values first + ERROR PREVENTION: + * Never pass empty strings ('') where numeric or date values are expected + * Use NULLIF or CASE statements to handle empty values + * Always use TRY_CAST or similar safe casting functions when converting data types + * Check for division by zero using NULLIF(denominator, 0) + * Always use the LIMIT clause in your SELECT statements when fetching data. There are hard limits imposed + by this tool on the maximum number of rows that can be fetched and the maximum number of characters. + The tool will truncate the data if those limits are exceeded. + + DATA VALIDATION: + * When querying columns with categorical values, use query_data tool to inspect distinct values beforehand + * Ensure valid filtering by checking actual data values first + **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "sql_query": { - "description": "SQL SELECT query to run.", - "type": "string" + "type": "string", + "description": "SQL SELECT query to run." }, "query_name": { - "description": "A concise, human-readable name for this query based on its purpose and what data it retrieves. Use normal words with spaces (e.g., \"Customer Orders Last Month\", \"Top Selling Products\", \"User Activity Summary\").", - "type": "string" + "type": "string", + "description": "A concise, human-readable name for this query based on its purpose and what data it retrieves. Use normal words with spaces (e.g., \"Customer Orders Last Month\", \"Top Selling Products\", \"User Activity Summary\")." } }, "required": [ "sql_query", "query_name" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -4155,18 +3387,18 @@ EXAMPLES: **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "bucket_ids": { "default": [], "description": "Filter by specific bucket IDs.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } } }, - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` @@ -4219,23 +3451,23 @@ EXAMPLES: **Input JSON Schema**: ```json { - "additionalProperties": false, + "type": "object", "properties": { "bucket_ids": { "default": [], "description": "Filter by specific bucket IDs.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } }, "table_ids": { "default": [], "description": "Filter by specific table IDs.", + "type": "array", "items": { "type": "string" - }, - "type": "array" + } }, "include_usage": { "default": false, @@ -4243,73 +3475,53 @@ EXAMPLES: "type": "boolean" } }, - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` --- ## update_descriptions -**Annotations**: `destructive` +**Annotations**: **Tags**: `storage` **Description**: -Updates the description for a Keboola storage item. - -This tool supports three item types, inferred from the provided item_id: - -- bucket: item_id = "in.c-bucket" -- table: item_id = "in.c-bucket.table" -- column: item_id = "in.c-bucket.table.column" - -Usage examples (payload uses a list of DescriptionUpdate objects): -- Update a bucket: - updates=[DescriptionUpdate(item_id="in.c-my-bucket", description="New bucket description")] -- Update a table: - updates=[DescriptionUpdate(item_id="in.c-my-bucket.my-table", description="New table description")] -- Update a column: - updates=[DescriptionUpdate(item_id="in.c-my-bucket.my-table.my_column", description="New column description")] +Updates the description for Keboola storage items (buckets, tables, or columns). **Input JSON Schema**: ```json { - "$defs": { - "DescriptionUpdate": { - "description": "Structured update describing a storage item and its new description.", - "properties": { - "item_id": { - "description": "Storage item name: \"bucket_id\", \"bucket_id.table_id\", \"bucket_id.table_id.column_name\"", - "type": "string" - }, - "description": { - "description": "New description to set for the storage item.", - "type": "string" - } - }, - "required": [ - "item_id", - "description" - ], - "type": "object" - } - }, - "additionalProperties": false, + "type": "object", "properties": { "updates": { - "description": "List of DescriptionUpdate objects with storage item_id and new description. Examples: \"bucket_id\", \"bucket_id.table_id\", \"bucket_id.table_id.column_name\"", + "type": "array", "items": { - "$ref": "#/$defs/DescriptionUpdate" + "type": "object", + "properties": { + "item_id": { + "type": "string", + "description": "Storage item: \"bucket_id\", \"bucket_id.table_id\", or \"bucket_id.table_id.column_name\"." + }, + "description": { + "type": "string", + "description": "New description to set." + } + }, + "required": [ + "item_id", + "description" + ] }, - "type": "array" + "description": "List of description updates to apply." } }, "required": [ "updates" ], - "type": "object" + "$schema": "http://json-schema.org/draft-07/schema#" } ``` diff --git a/__tests__/clients.test.ts b/__tests__/clients.test.ts new file mode 100644 index 000000000..c28ac6610 --- /dev/null +++ b/__tests__/clients.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; + +import { createKeboolaClients } from '@/clients/keboola'; +import { deriveServiceUrls } from '@/clients/urls'; +import { Config } from '@/config'; + +describe('deriveServiceUrls', () => { + it('derives every service URL from the storage hostname suffix', () => { + const urls = deriveServiceUrls('https://connection.eu-central-1.keboola.com'); + expect(urls).toEqual({ + storage: 'https://connection.eu-central-1.keboola.com', + metastore: 'https://metastore.eu-central-1.keboola.com', + queue: 'https://queue.eu-central-1.keboola.com', + ai: 'https://ai.eu-central-1.keboola.com', + dataScience: 'https://data-science.eu-central-1.keboola.com', + encryption: 'https://encryption.eu-central-1.keboola.com', + scheduler: 'https://scheduler.eu-central-1.keboola.com', + syncActions: 'https://sync-actions.eu-central-1.keboola.com', + queryService: 'https://query.eu-central-1.keboola.com', + }); + }); + + it.each(['https://example.com', 'not-a-url', 'https://storage.keboola.com'])( + 'rejects a non-connection Storage API URL %j', + (url) => { + expect(() => deriveServiceUrls(url)).toThrow(/Invalid Keboola Storage API URL/); + }, + ); +}); + +describe('createKeboolaClients', () => { + it('builds clients from a configured token + url', () => { + const config = new Config({ + storageApiUrl: 'https://connection.keboola.com', + storageToken: 'token', + }); + const clients = createKeboolaClients(config); + expect(clients.storage.buckets).toBeDefined(); + expect(clients.queue).toBeDefined(); + }); + + it.each([ + ['missing url', { storageToken: 'token' }, /Storage API URL is not configured/], + [ + 'missing token', + { storageApiUrl: 'https://connection.keboola.com' }, + /token is not configured/, + ], + ])('throws on %s', (_label, fields, pattern) => { + expect(() => createKeboolaClients(new Config(fields))).toThrow(pattern); + }); +}); diff --git a/__tests__/config.test.ts b/__tests__/config.test.ts new file mode 100644 index 000000000..47038a681 --- /dev/null +++ b/__tests__/config.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; + +describe('Config', () => { + it.each([ + ['canonical field name', { storageToken: 'abc' }], + ['KBC_ env-style key', { KBC_STORAGE_TOKEN: 'abc' }], + ['X- header-style key', { 'X-StorageApiToken': 'abc' }], + ['snake_case alias', { storage_api_token: 'abc' }], + ])('resolves storageToken from %s', (_label, map) => { + expect(Config.fromMap(map).storageToken).toBe('abc'); + }); + + it('amends a Storage API URL down to scheme + host', () => { + const config = Config.fromMap({ KBC_STORAGE_API_URL: 'connection.keboola.com/some/path' }); + expect(config.storageApiUrl).toBe('https://connection.keboola.com'); + }); + + it.each(['', 'none', 'null', 'default', 'production', 'PRODUCTION'])( + 'normalizes branch id %j to undefined', + (branch) => { + expect(Config.fromMap({ KBC_BRANCH_ID: branch }).branchId).toBeUndefined(); + }, + ); + + it('keeps a real branch id', () => { + expect(Config.fromMap({ KBC_BRANCH_ID: '12345' }).branchId).toBe('12345'); + }); + + it('redacts secret fields in the string form', () => { + const text = new Config({ storageToken: 'super-secret', branchId: '1' }).toString(); + expect(text).toContain("storageToken='****'"); + expect(text).not.toContain('super-secret'); + expect(text).toContain("branchId='1'"); + }); + + it('layers replaceBy values over the base config', () => { + const base = new Config({ storageToken: 'a', branchId: '1' }); + const next = base.replaceBy({ storageToken: 'b' }); + expect(next.storageToken).toBe('b'); + expect(next.branchId).toBe('1'); + }); +}); diff --git a/__tests__/docsSearch.embedder.test.ts b/__tests__/docsSearch.embedder.test.ts new file mode 100644 index 000000000..57d1e54b8 --- /dev/null +++ b/__tests__/docsSearch.embedder.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from 'vitest'; + +import { migrationSql } from '../scripts/docsIndex'; + +import { + createEmbedderFromEnv, + DEFAULT_LOCAL_MODEL, + LocalEmbedder, + StubEmbedder, +} from '@/clients/docsSearch'; +import { parseEnv } from '@/env'; + +const env = (m: Record) => parseEnv(m); + +describe('createEmbedderFromEnv', () => { + it('selects the stub embedder with a default 3072 dim', () => { + const e = createEmbedderFromEnv(env({ DOCS_EMBEDDER_MODEL: 'stub' })); + expect(e).toBeInstanceOf(StubEmbedder); + expect(e?.dim).toBe(3072); + }); + + it('honors a custom dim for the stub embedder', () => { + const e = createEmbedderFromEnv( + env({ DOCS_EMBEDDER_MODEL: 'stub', DOCS_EMBEDDER_DIM: '1024' }), + ); + expect(e?.dim).toBe(1024); + }); + + it('selects the local embedder with a default HF model + 384 dim', () => { + const e = createEmbedderFromEnv(env({ DOCS_EMBEDDER_MODEL: 'local' })); + expect(e).toBeInstanceOf(LocalEmbedder); + expect(e?.model).toBe(DEFAULT_LOCAL_MODEL); + expect(e?.dim).toBe(384); + }); + + it('honors a custom local HF model + dim (e.g. a 1024 model)', () => { + const e = createEmbedderFromEnv( + env({ + DOCS_EMBEDDER_MODEL: 'local', + DOCS_EMBEDDER_LOCAL_MODEL: 'Xenova/bge-large-en-v1.5', + DOCS_EMBEDDER_DIM: '1024', + }), + ); + expect(e).toBeInstanceOf(LocalEmbedder); + expect(e?.model).toBe('Xenova/bge-large-en-v1.5'); + expect(e?.dim).toBe(1024); + }); + + it('selects a remote embedder when endpoint + key + model are set', () => { + const e = createEmbedderFromEnv( + env({ + DOCS_EMBEDDER_MODEL: 'text-embedding-3-large', + DOCS_EMBEDDER_ENDPOINT: 'https://embed.example/v1/embeddings', + DOCS_EMBEDDER_API_KEY: 'secret', + }), + ); + expect(e).not.toBeInstanceOf(StubEmbedder); + expect(e).not.toBeInstanceOf(LocalEmbedder); + expect(e?.model).toBe('text-embedding-3-large'); + expect(e?.dim).toBe(3072); + }); + + it('returns null when unconfigured or a remote model lacks endpoint/key', () => { + expect(createEmbedderFromEnv(env({}))).toBeNull(); + expect( + createEmbedderFromEnv(env({ DOCS_EMBEDDER_MODEL: 'text-embedding-3-large' })), + ).toBeNull(); + }); +}); + +describe('migrationSql', () => { + it('parametrizes the halfvec column dimension', () => { + expect(migrationSql(1024)).toContain('embedding halfvec(1024) not null'); + expect(migrationSql(384)).toContain('halfvec(384)'); + expect(migrationSql(3072)).toContain('halfvec(3072)'); + }); +}); diff --git a/__tests__/env.test.ts b/__tests__/env.test.ts new file mode 100644 index 000000000..fbd92edf1 --- /dev/null +++ b/__tests__/env.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { applyDeploymentDefaults, parseEnv, redactedEnv } from '@/env'; + +describe('redactedEnv', () => { + it('masks secrets, strips DATABASE_URL credentials, and keeps non-secrets', () => { + const dump = redactedEnv( + parseEnv({ + DATABASE_URL: 'postgres://mcp:s3cret@db.internal:5432/docs', + DOCS_EMBEDDER_MODEL: 'local', + DOCS_EMBEDDER_DIM: '384', + DOCS_EMBEDDER_API_KEY: 'sk-abc', + KBC_JWT_SECRET: 'jwt-xyz', + KBC_OAUTH_CLIENT_ID: 'client-123', + }), + ); + // Credentials stripped, host/db preserved. + expect(dump.DATABASE_URL).toBe('postgres://***:***@db.internal:5432/docs'); + // Secrets masked. + expect(dump.DOCS_EMBEDDER_API_KEY).toBe('***'); + expect(dump.KBC_JWT_SECRET).toBe('***'); + // Non-secrets visible — this is the whole point (see the embedder config at a glance). + expect(dump.DOCS_EMBEDDER_MODEL).toBe('local'); + expect(dump.DOCS_EMBEDDER_DIM).toBe(384); + expect(dump.KBC_OAUTH_CLIENT_ID).toBe('client-123'); + // Unset values shown as null so the dump is complete. + expect(dump.HOSTNAME_SUFFIX).toBeNull(); + }); +}); + +describe('parseEnv', () => { + it('applies defaults and coerces PORT', () => { + const env = parseEnv({}); + expect(env.HOST).toBe('localhost'); + expect(env.PORT).toBe(8000); + expect(env.LOG_LEVEL).toBe('INFO'); + expect(env.APP_ENV).toBe('local'); + expect(env.APP_VERSION).toBe('DEV'); + }); + + it('treats empty strings as unset (falls back to defaults)', () => { + const env = parseEnv({ HOST: '', PORT: '', APP_ENV: '' }); + expect(env.HOST).toBe('localhost'); + expect(env.PORT).toBe(8000); + expect(env.APP_ENV).toBe('local'); + }); + + it('reads provided values and coerces booleans', () => { + const env = parseEnv({ + PORT: '3000', + HOSTNAME_SUFFIX: 'keboola.com', + DD_LOGS_INJECTION: 'true', + }); + expect(env.PORT).toBe(3000); + expect(env.HOSTNAME_SUFFIX).toBe('keboola.com'); + expect(env.DD_LOGS_INJECTION).toBe(true); + }); + + it('throws on an invalid PORT when validation is not skipped', () => { + expect(() => parseEnv({ PORT: 'not-a-number' })).toThrow(/Invalid deployment environment/); + }); + + it('does not throw on a build (SKIP_ENV_VALIDATION)', () => { + expect(() => parseEnv({ SKIP_ENV_VALIDATION: '1', PORT: 'not-a-number' })).not.toThrow(); + }); +}); + +describe('applyDeploymentDefaults (HOSTNAME_SUFFIX derivation)', () => { + it('derives the Storage API URL from HOSTNAME_SUFFIX when unset', () => { + const config = applyDeploymentDefaults( + new Config(), + parseEnv({ HOSTNAME_SUFFIX: 'keboola.com' }), + ); + expect(config.storageApiUrl).toBe('https://connection.keboola.com'); + }); + + it('does not override an explicit Storage API URL', () => { + const base = new Config({ storageApiUrl: 'https://connection.north-europe.azure.keboola.com' }); + const config = applyDeploymentDefaults(base, parseEnv({ HOSTNAME_SUFFIX: 'keboola.com' })); + expect(config.storageApiUrl).toBe('https://connection.north-europe.azure.keboola.com'); + }); + + it('derives OAuth + MCP URLs and default scope only when OAuth is configured', () => { + const base = new Config({ oauthClientId: 'cid', oauthClientSecret: 'sec' }); + const config = applyDeploymentDefaults(base, parseEnv({ HOSTNAME_SUFFIX: 'keboola.com' })); + expect(config.oauthServerUrl).toBe('https://connection.keboola.com'); + expect(config.mcpServerUrl).toBe('https://mcp.keboola.com'); + expect(config.oauthScope).toBe('email'); + }); + + it('leaves OAuth/MCP URLs unset when OAuth is not configured', () => { + const config = applyDeploymentDefaults( + new Config(), + parseEnv({ HOSTNAME_SUFFIX: 'keboola.com' }), + ); + expect(config.oauthServerUrl).toBeUndefined(); + expect(config.mcpServerUrl).toBeUndefined(); + }); + + it('is a no-op when HOSTNAME_SUFFIX is absent', () => { + const config = applyDeploymentDefaults(new Config(), parseEnv({})); + expect(config.storageApiUrl).toBeUndefined(); + }); +}); diff --git a/__tests__/errors.test.ts b/__tests__/errors.test.ts new file mode 100644 index 000000000..bc3a6ee7c --- /dev/null +++ b/__tests__/errors.test.ts @@ -0,0 +1,125 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; + +import { formatValidationErrors, prettifyValidationError } from '@/mcp/errors'; +import { registerTool } from '@/mcp/tool'; + +/** + * Port of `tests/test_errors.py`. + * + * The Python `errors` module bundled three concerns: (1) Pydantic validation-error + * prettifying, (2) the `tool_errors` decorator (recovery hints + logging + SAPI event + * telemetry), and (3) a FastMCP `ValidationErrorMiddleware`. + * + * In the TypeScript port those concerns are split: the recovery-hint / error-result / + * logging behavior lives in `@/mcp/tool` (`registerTool`), so it is tested here against + * that surface; the SAPI-event telemetry and FastMCP middleware have no TS equivalent + * (different runtime) and are not ported. The validation-error formatting helpers were + * ported to `@/mcp/errors` (adapted from Pydantic's `ValidationError` to Zod's + * `ZodError`) and are tested directly. + */ + +const zodError = (shape: z.ZodRawShape, input: unknown): z.ZodError => { + const result = z.object(shape).safeParse(input); + if (result.success) throw new Error('expected a validation failure'); + return result.error; +}; + +describe('formatValidationErrors', () => { + it('extracts field, message and extra (code) from each issue', () => { + const err = zodError({ sql_query: z.string(), query_name: z.string() }, { foo: 'bar' }); + const formatted = formatValidationErrors(err.issues); + + expect(formatted.errors).toHaveLength(2); + const fields = formatted.errors.map((e) => e.field).sort(); + expect(fields).toEqual(['query_name', 'sql_query']); + for (const e of formatted.errors) { + expect(e.message).toBeTruthy(); + // The Zod issue `code` is carried through under `extra`, mirroring Pydantic's `type`. + expect(e.extra.code).toBe('invalid_type'); + } + }); + + it('joins nested paths with dots', () => { + const err = zodError({ outer: z.object({ inner: z.string() }) }, { outer: { inner: 1 } }); + const formatted = formatValidationErrors(err.issues); + expect(formatted.errors[0]!.field).toBe('outer.inner'); + }); +}); + +describe('prettifyValidationError', () => { + it('renders the count header with the model name', () => { + const err = zodError({ a: z.string(), b: z.number() }, {}); + const text = prettifyValidationError(err, 'MyTool'); + const lines = text.split('\n'); + expect(lines[0]).toBe('Found 2 validation error(s) for MyTool'); + // Field locations are surfaced explicitly in the body. + expect(text).toContain('field: a'); + expect(text).toContain('field: b'); + }); + + it('defaults the model name to "unknown"', () => { + const err = zodError({ a: z.string() }, {}); + expect(prettifyValidationError(err).split('\n')[0]).toBe( + 'Found 1 validation error(s) for unknown', + ); + }); +}); + +// --- registerTool error path (the TS home of Python's `tool_errors` recovery/logging) --- + +const callTool = async ( + def: Parameters[1], + args: Record = {}, +) => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + const mcp = new McpServer({ name: 'test', version: '0.0.0' }); + registerTool(mcp, def); + await mcp.connect(serverT); + const client = new Client({ name: 'test', version: '0.0.0' }); + await client.connect(clientT); + return client.callTool({ name: def.name, arguments: args }); +}; + +describe('registerTool error handling', () => { + it('returns an error result carrying the exception message', async () => { + const result = await callTool({ + name: 'boom', + description: 'd', + handler: () => { + throw new Error('Simulated failure'); + }, + }); + expect(result.isError).toBe(true); + expect((result.content as { text: string }[])[0]!.text).toBe('Simulated failure'); + }); + + it('appends the recovery hint when one is configured', async () => { + const result = await callTool({ + name: 'boom_recover', + description: 'd', + recovery: 'Check that data has valid types.', + handler: () => { + throw new Error('Simulated failure'); + }, + }); + expect(result.isError).toBe(true); + const text = (result.content as { text: string }[])[0]!.text; + expect(text).toContain('Simulated failure'); + expect(text).toContain('Recovery: Check that data has valid types.'); + }); + + it('omits the Recovery line when no hint is configured', async () => { + const result = await callTool({ + name: 'boom_plain', + description: 'd', + handler: () => { + throw new Error('Simulated failure'); + }, + }); + expect((result.content as { text: string }[])[0]!.text).not.toContain('Recovery:'); + }); +}); diff --git a/__tests__/http.test.ts b/__tests__/http.test.ts new file mode 100644 index 000000000..56eeccf58 --- /dev/null +++ b/__tests__/http.test.ts @@ -0,0 +1,63 @@ +import { serve } from '@hono/node-server'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import type { AddressInfo } from 'node:net'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { createHttpApp } from '@/transports/http'; + +let httpServer: ReturnType; +let port: number; + +const connect = async (headers?: Record) => { + const url = new URL(`http://localhost:${port}/mcp`); + const client = new Client({ name: 'test-client', version: '0.0.0' }); + await client.connect(new StreamableHTTPClientTransport(url, { requestInit: { headers } })); + return client; +}; + +// Returns the TOON-encoded tool output text. +const callServerInfo = async (client: Awaited>) => { + const result = await client.callTool({ name: 'get_server_info', arguments: {} }); + const content = result.content as { type: string; text: string }[]; + return content[0]!.text; +}; + +describe('streamable-http transport', () => { + beforeAll(async () => { + const app = createHttpApp(new Config({ branchId: '999' })); + port = await new Promise((resolve) => { + httpServer = serve({ fetch: app.fetch, port: 0 }, (info: AddressInfo) => resolve(info.port)); + }); + }); + + afterAll(() => { + httpServer.close(); + }); + + it('serves a health check', async () => { + const res = await fetch(`http://localhost:${port}/health-check`); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ status: 'ok' }); + }); + + it('lists tools over the MCP endpoint', async () => { + const client = await connect(); + const { tools } = await client.listTools(); + expect(tools.map((tool) => tool.name)).toContain('get_server_info'); + await client.close(); + }); + + it('uses the base config when no headers override it', async () => { + const client = await connect(); + expect(await callServerInfo(client)).toContain('branchId: "999"'); + await client.close(); + }); + + it('layers X- headers over the base config per request', async () => { + const client = await connect({ 'X-Branch-Id': '123', 'X-StorageApiToken': 'tok' }); + expect(await callServerInfo(client)).toContain('branchId: "123"'); + await client.close(); + }); +}); diff --git a/__tests__/links.test.ts b/__tests__/links.test.ts new file mode 100644 index 000000000..804e91f8f --- /dev/null +++ b/__tests__/links.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest'; + +import { ProjectLinksManager } from '@/links'; + +const BASE = 'https://connection.keboola.com'; + +const manager = (branchId?: string) => + new ProjectLinksManager({ baseUrl: BASE, projectId: '1234', branchId }); + +describe('ProjectLinksManager', () => { + it('builds production URLs without a branch segment', () => { + expect(manager().getProjectDetailLink()).toEqual({ + type: 'ui-detail', + title: 'Project Dashboard', + url: `${BASE}/admin/projects/1234/`, + }); + }); + + it('inserts the branch segment on a development branch', () => { + expect(manager('567').getBucketDetailLink('in.c-main', 'main').url).toBe( + `${BASE}/admin/projects/1234/branch/567/storage/in.c-main`, + ); + }); + + it.each([ + ['keboola.flow', 'flows-v2'], + ['keboola.orchestrator', 'flows'], + ] as const)('routes %s flows to /%s', (flowType, path) => { + expect(manager().getFlowDetailLink('99', 'My Flow', flowType).url).toBe( + `${BASE}/admin/projects/1234/${path}/99`, + ); + }); + + it('routes transformation components to the transformations path via getComponentConfigLink', () => { + const link = manager().getComponentConfigLink('keboola.snowflake-transformation', 'cfg1', 'T'); + expect(link.url).toBe( + `${BASE}/admin/projects/1234/transformations-v2/keboola.snowflake-transformation/cfg1`, + ); + }); + + it('routes data-app components to the data-apps path', () => { + const link = manager().getComponentConfigLink('keboola.data-apps', 'cfg1', 'App'); + expect(link.url).toBe(`${BASE}/admin/projects/1234/data-apps/cfg1`); + }); + + it('splits a fully-qualified table id into bucket + table', () => { + expect(manager().getTableDetailLinkFromTableId('in.c-main.users').url).toBe( + `${BASE}/admin/projects/1234/storage/in.c-main/table/users`, + ); + }); + + it('getLinks picks the config link when component + configuration are given', () => { + const links = manager().getLinks({ + componentId: 'keboola.ex-aws-s3', + configurationId: 'c1', + name: 'My cfg', + }); + expect(links).toHaveLength(1); + expect(links[0]!.url).toBe(`${BASE}/admin/projects/1234/components/keboola.ex-aws-s3/c1`); + }); +}); diff --git a/__tests__/mcp.authorization.test.ts b/__tests__/mcp.authorization.test.ts new file mode 100644 index 000000000..f1c5ff77b --- /dev/null +++ b/__tests__/mcp.authorization.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from 'vitest'; + +import { + hasAuthorizationFilters, + isToolNameAuthorized, + parseAuthorizationConfig, +} from '@/mcp/authorization'; + +// Port of tests/test_authorization.py. The Python middleware reads HTTP headers; +// here the header values are surfaced onto Config (allowedTools / disallowedTools / +// readOnlyMode), parsed by parseAuthorizationConfig, and applied per tool by +// isToolNameAuthorized. + +// 3 read-only (get_configs, get_buckets, query_data), 2 write (create_config, update_descriptions). +const TOOLS: { name: string; readOnly: boolean }[] = [ + { name: 'get_configs', readOnly: true }, + { name: 'create_config', readOnly: false }, + { name: 'get_buckets', readOnly: true }, + { name: 'update_descriptions', readOnly: false }, + { name: 'query_data', readOnly: true }, +]; +const ALL_TOOLS = new Set(TOOLS.map((t) => t.name)); +const READ_ONLY_TOOLS = new Set(TOOLS.filter((t) => t.readOnly).map((t) => t.name)); + +type Headers = { allowedTools?: string; disallowedTools?: string; readOnlyMode?: string }; + +const filterList = (headers: Headers): Set => { + const config = parseAuthorizationConfig(headers); + if (!hasAuthorizationFilters(config)) return new Set(TOOLS.map((t) => t.name)); + return new Set( + TOOLS.filter((t) => isToolNameAuthorized(t.name, t.readOnly, config)).map((t) => t.name), + ); +}; + +describe('parseAuthorizationConfig + isToolNameAuthorized (list filtering)', () => { + it.each<[string, Headers, Set]>([ + ['no_headers', {}, ALL_TOOLS], + [ + 'allowed_tools_only', + { allowedTools: 'get_configs, get_buckets' }, + new Set(['get_configs', 'get_buckets']), + ], + ['read_only_mode_only', { readOnlyMode: 'true' }, READ_ONLY_TOOLS], + [ + 'disallowed_tools_only', + { disallowedTools: 'create_config, update_descriptions' }, + READ_ONLY_TOOLS, + ], + [ + 'allowed_and_read_only', + { allowedTools: 'get_configs, create_config, get_buckets', readOnlyMode: 'true' }, + new Set(['get_configs', 'get_buckets']), + ], + [ + 'allowed_and_disallowed', + { allowedTools: 'get_configs, create_config, get_buckets', disallowedTools: 'create_config' }, + new Set(['get_configs', 'get_buckets']), + ], + [ + 'read_only_and_disallowed', + { readOnlyMode: 'true', disallowedTools: 'get_configs' }, + new Set(['get_buckets', 'query_data']), + ], + [ + 'all_three_headers', + { + allowedTools: 'get_configs, get_buckets, query_data, create_config', + readOnlyMode: 'true', + disallowedTools: 'query_data', + }, + new Set(['get_configs', 'get_buckets']), + ], + ['empty_allowed_tools', { allowedTools: '' }, ALL_TOOLS], + ['whitespace_only_allowed_tools', { allowedTools: ' , , ' }, ALL_TOOLS], + ['empty_disallowed_tools', { disallowedTools: '' }, ALL_TOOLS], + [ + 'allowed_tools_with_whitespace', + { allowedTools: ' get_configs , get_buckets , ' }, + new Set(['get_configs', 'get_buckets']), + ], + [ + 'disallowed_tools_with_whitespace', + { disallowedTools: ' create_config , update_descriptions , ' }, + READ_ONLY_TOOLS, + ], + ])('%s', (_label, headers, expected) => { + expect(filterList(headers)).toEqual(expected); + }); + + it.each(['true', 'True', 'TRUE', '1', 'yes', 'Yes', 'YES'])( + 'read-only mode enabled for truthy value %j', + (value) => { + expect(filterList({ readOnlyMode: value })).toEqual(READ_ONLY_TOOLS); + }, + ); + + it.each(['false', 'False', '0', 'no', '', 'random'])( + 'read-only mode disabled for falsy value %j', + (value) => { + expect(filterList({ readOnlyMode: value })).toEqual(ALL_TOOLS); + }, + ); +}); + +describe('isToolNameAuthorized (call decision)', () => { + it.each<[string, string, boolean, Headers, boolean]>([ + ['no_headers_write_tool', 'create_config', false, {}, true], + ['no_headers_read_tool', 'get_configs', true, {}, true], + [ + 'allowed_tool_in_list', + 'get_configs', + true, + { allowedTools: 'get_configs, get_buckets' }, + true, + ], + [ + 'allowed_tool_not_in_list', + 'create_config', + false, + { allowedTools: 'get_configs, get_buckets' }, + false, + ], + ['read_only_mode_read_tool', 'get_configs', true, { readOnlyMode: 'true' }, true], + ['read_only_mode_write_tool', 'create_config', false, { readOnlyMode: 'true' }, false], + [ + 'disallowed_tool_in_list', + 'create_config', + false, + { disallowedTools: 'create_config, update_descriptions' }, + false, + ], + [ + 'disallowed_tool_not_in_list', + 'get_configs', + true, + { disallowedTools: 'create_config, update_descriptions' }, + true, + ], + [ + 'allowed_and_read_only_write_tool', + 'create_config', + false, + { allowedTools: 'get_configs, create_config', readOnlyMode: 'true' }, + false, + ], + [ + 'allowed_and_disallowed_same_tool', + 'get_configs', + true, + { allowedTools: 'get_configs, get_buckets', disallowedTools: 'get_configs' }, + false, + ], + ])('%s', (_label, toolName, readOnly, headers, shouldAllow) => { + const config = parseAuthorizationConfig(headers); + expect(isToolNameAuthorized(toolName, readOnly, config)).toBe(shouldAllow); + }); +}); diff --git a/__tests__/mcp.filtering.test.ts b/__tests__/mcp.filtering.test.ts new file mode 100644 index 000000000..942b7cbf9 --- /dev/null +++ b/__tests__/mcp.filtering.test.ts @@ -0,0 +1,367 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { + authorizeToolCall, + filterToolsList, + type GatedTool, + type GatingContext, + getProjectFeatures, + getTokenRole, +} from '@/mcp/filtering'; +import { createServer } from '@/server'; + +// --- Pure gating functions (port of tests/test_mcp.py TestToolsFilteringMiddleware) --- + +const tool = (name: string, readOnly = false): GatedTool => ({ name, readOnly }); + +const ctx = (over: Partial = {}): GatingContext => ({ + tokenRole: '', + features: new Set(), + isOauth: false, + isMainBranch: true, + docsIndexAvailable: true, + ...over, +}); + +describe('getProjectFeatures / getTokenRole', () => { + it('reads features and role from token info', () => { + const info = { owner: { features: ['a', '', 'b'] }, admin: { role: 'admin' } }; + expect(getProjectFeatures(info)).toEqual(new Set(['a', 'b'])); + expect(getTokenRole(info)).toBe('admin'); + }); + + it('defaults to empty when missing', () => { + expect(getProjectFeatures({})).toEqual(new Set()); + expect(getTokenRole({})).toBe(''); + }); +}); + +describe('filterToolsList — data app tools by branch', () => { + const dataAppTools = [ + 'modify_streamlit_data_app', + 'get_data_apps', + 'deploy_data_app', + 'delete_python_js_data_app_draft', + ]; + const tools = [...dataAppTools.map((n) => tool(n)), tool('other_tool')]; + + it.each<[boolean, boolean]>([ + [false, true], // non-main branch -> filtered out + [true, false], // main branch -> kept + ])('isMainBranch=%s filters=%s', (isMainBranch, expectFiltered) => { + const names = new Set(filterToolsList(tools, ctx({ isMainBranch })).map((t) => t.name)); + for (const n of dataAppTools) { + expect(names.has(n)).toBe(!expectFiltered); + } + expect(names.has('other_tool')).toBe(true); + }); +}); + +describe('filterToolsList — flow tools by role / oauth', () => { + const tools = [ + tool('modify_flow'), + tool('update_flow'), + tool('other_tool'), + tool('read_only_tool', true), + ]; + + it.each<[string, boolean, string[], string[]]>([ + ['admin', false, ['update_flow'], ['modify_flow', 'read_only_tool']], + ['share', false, ['update_flow'], ['modify_flow', 'read_only_tool']], + ['', false, ['modify_flow'], ['update_flow', 'read_only_tool']], + ['readOnly', false, ['modify_flow', 'update_flow'], ['read_only_tool']], + ['guest', false, ['modify_flow'], ['update_flow', 'read_only_tool']], + // OAuth regular user gets modify_flow (different from SAPI regular). + ['', true, ['update_flow'], ['modify_flow', 'read_only_tool']], + ])('role=%j oauth=%s', (tokenRole, isOauth, hidden, visible) => { + const names = new Set(filterToolsList(tools, ctx({ tokenRole, isOauth })).map((t) => t.name)); + for (const n of hidden) expect(names.has(n)).toBe(false); + for (const n of visible) expect(names.has(n)).toBe(true); + }); +}); + +describe('filterToolsList — semantic tools by feature', () => { + const tools = [ + tool('search_semantic_context'), + tool('get_semantic_context'), + tool('get_semantic_schema'), + tool('validate_semantic_query'), + tool('other_tool'), + ]; + + it.each<[string[], string, boolean]>([ + [[], 'search_semantic_context', true], + [[], 'get_semantic_schema', true], + [['mcp-semantic-tooling'], 'search_semantic_context', false], + [['mcp-semantic-tooling'], 'get_semantic_schema', false], + [['other-feature'], 'search_semantic_context', true], + ])('features=%j tool=%s filtered=%s', (features, name, expectFiltered) => { + const names = new Set( + filterToolsList(tools, ctx({ features: new Set(features) })).map((t) => t.name), + ); + expect(names.has(name)).toBe(!expectFiltered); + expect(names.has('other_tool')).toBe(true); + }); +}); + +describe('filterToolsList — conditional vs legacy flow', () => { + const tools = [tool('create_flow'), tool('create_conditional_flow'), tool('other_tool')]; + + it('hides create_conditional_flow when hide-conditional-flows feature is on', () => { + const names = new Set( + filterToolsList(tools, ctx({ features: new Set(['hide-conditional-flows']) })).map( + (t) => t.name, + ), + ); + expect(names.has('create_flow')).toBe(true); + expect(names.has('create_conditional_flow')).toBe(false); + }); + + it('hides create_flow when the feature is off', () => { + const names = new Set(filterToolsList(tools, ctx()).map((t) => t.name)); + expect(names.has('create_flow')).toBe(false); + expect(names.has('create_conditional_flow')).toBe(true); + }); +}); + +describe('filterToolsList / authorizeToolCall — docs tools by index availability', () => { + const docsTools = [tool('docs_query', true), tool('find_component_id', true), tool('get_jobs')]; + + it('keeps docs tools when the index is available, drops them when not', () => { + const withIndex = filterToolsList(docsTools, ctx({ docsIndexAvailable: true })).map( + (t) => t.name, + ); + expect(withIndex).toContain('docs_query'); + expect(withIndex).toContain('find_component_id'); + + const withoutIndex = filterToolsList(docsTools, ctx({ docsIndexAvailable: false })).map( + (t) => t.name, + ); + expect(withoutIndex).not.toContain('docs_query'); + expect(withoutIndex).not.toContain('find_component_id'); + expect(withoutIndex).toContain('get_jobs'); // non-docs tools unaffected + }); + + it('denies a docs tool call when the index is unavailable', () => { + expect( + authorizeToolCall({ + toolName: 'docs_query', + isReadOnly: true, + isSemantic: false, + tokenRole: 'admin', + features: new Set(), + isOauth: false, + isMainBranch: true, + docsIndexAvailable: false, + }), + ).toContain('documentation index is not'); + }); +}); + +describe('authorizeToolCall — flow tools by role / oauth', () => { + it.each<[string, boolean, string, boolean, boolean]>([ + ['admin', false, 'modify_flow', false, false], + ['admin', false, 'update_flow', false, true], + ['share', false, 'modify_flow', false, false], + ['share', false, 'update_flow', false, true], + ['', false, 'modify_flow', false, true], + ['', false, 'update_flow', false, false], + ['guest', false, 'write_tool', false, false], + ['guest', false, 'read_only_tool', true, false], + ['readOnly', false, 'write_tool', false, true], + ['readOnly', false, 'read_only_tool', true, false], + ['', true, 'modify_flow', false, false], + ['', true, 'update_flow', false, true], + ])('role=%j oauth=%s tool=%s', (tokenRole, isOauth, toolName, isReadOnly, expectDenied) => { + const denial = authorizeToolCall({ + toolName, + isReadOnly, + isSemantic: false, + tokenRole, + features: new Set(), + isOauth, + isMainBranch: true, + docsIndexAvailable: true, + }); + expect(denial !== null).toBe(expectDenied); + }); +}); + +describe('authorizeToolCall — data apps by branch', () => { + it.each<[boolean, boolean]>([ + [false, true], + [true, false], + ])('isMainBranch=%s denied=%s', (isMainBranch, expectDenied) => { + const denial = authorizeToolCall({ + toolName: 'modify_streamlit_data_app', + isReadOnly: false, + isSemantic: false, + tokenRole: 'admin', + features: new Set(), + isOauth: false, + isMainBranch, + docsIndexAvailable: true, + }); + if (expectDenied) { + expect(denial).toContain('main production branch'); + } else { + expect(denial).toBeNull(); + } + }); +}); + +describe('authorizeToolCall — semantic tools by feature', () => { + it.each<[string[], string, boolean]>([ + [[], 'search_semantic_context', true], + [[], 'get_semantic_schema', true], + [['mcp-semantic-tooling'], 'search_semantic_context', false], + [['mcp-semantic-tooling'], 'get_semantic_schema', false], + [[], 'other_tool', false], + ])('features=%j tool=%s denied=%s', (features, toolName, expectDenied) => { + const isSemantic = ['search_semantic_context', 'get_semantic_schema'].includes(toolName); + const denial = authorizeToolCall({ + toolName, + isReadOnly: true, + isSemantic, + tokenRole: 'admin', + features: new Set(features), + isOauth: false, + isMainBranch: true, + docsIndexAvailable: true, + }); + if (expectDenied) { + expect(denial).toContain('Semantic Layer Tooling'); + } else { + expect(denial).toBeNull(); + } + }); +}); + +// --- Integration: gating wired into createServer (tools/list + tools/call) --- + +const msw = setupServer(); +beforeAll(() => msw.listen({ onUnhandledRequest: 'error' })); +afterEach(() => msw.resetHandlers()); +afterAll(() => msw.close()); + +const verify = (body: unknown) => + http.get('https://connection.test/*', ({ request }) => { + if (new URL(request.url).pathname.endsWith('/tokens/verify')) { + return HttpResponse.json(body as object); + } + return undefined; + }); + +const connect = async (config: Config) => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 'test', version: '0.0.0' }); + await client.connect(clientT); + return client; +}; + +const baseConfig = (over: Record = {}) => + new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok', ...over }); + +describe('createServer gating integration', () => { + it('hides create_flow for a no-feature project and shows create_conditional_flow', async () => { + msw.use(verify({ owner: { features: [] }, admin: { role: 'admin' } })); + const client = await connect(baseConfig()); + const names = (await client.listTools()).tools.map((t) => t.name); + expect(names).toContain('create_conditional_flow'); + expect(names).not.toContain('create_flow'); + await client.close(); + }); + + it('shows only modify_flow (not update_flow) for an admin token', async () => { + msw.use(verify({ owner: { features: [] }, admin: { role: 'admin' } })); + const client = await connect(baseConfig()); + const names = (await client.listTools()).tools.map((t) => t.name); + expect(names).toContain('modify_flow'); + expect(names).not.toContain('update_flow'); + await client.close(); + }); + + it('shows update_flow (not modify_flow) for a guest token', async () => { + msw.use(verify({ owner: { features: [] }, admin: { role: 'guest' } })); + const client = await connect(baseConfig()); + const names = (await client.listTools()).tools.map((t) => t.name); + expect(names).toContain('update_flow'); + expect(names).not.toContain('modify_flow'); + await client.close(); + }); + + it('keeps data app tools in tools/list even on a non-main branch (discovery forces main)', async () => { + // Parity with Python: tools/list always discovers against the main branch, so data + // app tools stay visible during discovery and are only blocked at call time. + msw.use(verify({ owner: { features: [] }, admin: { role: 'admin' } })); + const client = await connect(baseConfig({ branchId: '123' })); + const names = (await client.listTools()).tools.map((t) => t.name); + expect(names).toContain('get_data_apps'); + expect(names).toContain('deploy_data_app'); + await client.close(); + }); + + it('blocks a data app tools/call on a non-main branch', async () => { + msw.use(verify({ owner: { features: [] }, admin: { role: 'admin' } })); + const client = await connect(baseConfig({ branchId: '123' })); + await expect(client.callTool({ name: 'get_data_apps', arguments: {} })).rejects.toThrow( + /main production branch/, + ); + await client.close(); + }); + + it('restricts list to read-only tools for a readonly role', async () => { + msw.use(verify({ owner: { features: [] }, admin: { role: 'readonly' } })); + const client = await connect(baseConfig()); + const tools = (await client.listTools()).tools; + expect(tools.length).toBeGreaterThan(0); + expect(tools.every((t) => t.annotations?.readOnlyHint === true)).toBe(true); + await client.close(); + }); + + it('hides semantic tools without the semantic feature, shows them with it', async () => { + msw.use(verify({ owner: { features: [] }, admin: { role: 'admin' } })); + let client = await connect(baseConfig()); + let names = (await client.listTools()).tools.map((t) => t.name); + expect(names).not.toContain('search_semantic_context'); + await client.close(); + + msw.use(verify({ owner: { features: ['mcp-semantic-tooling'] }, admin: { role: 'admin' } })); + client = await connect(baseConfig()); + names = (await client.listTools()).tools.map((t) => t.name); + expect(names).toContain('search_semantic_context'); + await client.close(); + }); + + it('blocks a tools/call that the project gating denies (update_flow for admin)', async () => { + msw.use(verify({ owner: { features: [] }, admin: { role: 'admin' } })); + const client = await connect(baseConfig()); + await expect(client.callTool({ name: 'update_flow', arguments: {} })).rejects.toThrow( + /update_flow/, + ); + await client.close(); + }); + + it('blocks a tools/call denied by X-Disallowed-Tools header authorization', async () => { + msw.use(verify({ owner: { features: [] }, admin: { role: 'admin' } })); + const client = await connect(baseConfig({ disallowedTools: 'get_jobs' })); + await expect(client.callTool({ name: 'get_jobs', arguments: { job_ids: [] } })).rejects.toThrow( + /not authorized/, + ); + await client.close(); + }); + + it('hides tools excluded by X-Allowed-Tools header authorization in tools/list', async () => { + msw.use(verify({ owner: { features: [] }, admin: { role: 'admin' } })); + const client = await connect(baseConfig({ allowedTools: 'get_jobs,get_buckets' })); + const names = (await client.listTools()).tools.map((t) => t.name); + expect(names.sort()).toEqual(['get_buckets', 'get_jobs']); + await client.close(); + }); +}); diff --git a/__tests__/mcp.tool.test.ts b/__tests__/mcp.tool.test.ts new file mode 100644 index 000000000..f0aab0a08 --- /dev/null +++ b/__tests__/mcp.tool.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; + +import { describeToolError } from '@/mcp/tool'; + +describe('describeToolError', () => { + it('enriches an api-client ApiError with the response body detail + exception id', () => { + // Shape of @keboola/api-client ApiError: message is the HTTP status text; the real + // reason + support id are on `.data` (this is the run_sync_action "Bad Request" case). + const apiError = Object.assign(new Error('Bad Request'), { + data: { error: 'Invalid access token\n', code: 0, exceptionId: 'exception-abc123' }, + }); + expect(describeToolError(apiError)).toBe( + 'Bad Request: Invalid access token (exception ID: exception-abc123)', + ); + }); + + it('uses data.message when data.error is absent', () => { + const err = Object.assign(new Error('Unprocessable Entity'), { + data: { message: 'query must not be empty' }, + }); + expect(describeToolError(err)).toBe('Unprocessable Entity: query must not be empty'); + }); + + it('passes a plain Error (e.g. our raw client, which already composes detail) through', () => { + expect(describeToolError(new Error('404 Not Found\nAPI error: nope'))).toBe( + '404 Not Found\nAPI error: nope', + ); + }); + + it('does not duplicate when the detail equals the base message', () => { + const err = Object.assign(new Error('Invalid access token'), { + data: { error: 'Invalid access token' }, + }); + expect(describeToolError(err)).toBe('Invalid access token'); + }); + + it('handles non-Error values', () => { + expect(describeToolError('boom')).toBe('boom'); + }); +}); diff --git a/__tests__/oauth.provider.test.ts b/__tests__/oauth.provider.test.ts new file mode 100644 index 000000000..5991b0d95 --- /dev/null +++ b/__tests__/oauth.provider.test.ts @@ -0,0 +1,305 @@ +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { + type AccessToken, + type ExtendedAuthorizationCode, + InvalidRedirectUriError, + type RefreshToken, + SimpleOAuthProvider, + validateRedirectUri, +} from '@/oauth'; + +const JWT_KEY = 'secret'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const newProvider = (): SimpleOAuthProvider => + new SimpleOAuthProvider({ + storageApiUrl: 'https://sapi', + mcpServerUrl: 'https://mcp', + callbackEndpoint: '/callback', + clientId: 'mcp-server-id', + clientSecret: 'mcp-server-secret', + serverUrl: 'https://oauth', + scope: 'scope', + jwtSecret: JWT_KEY, + }); + +const accessToken = (overrides: Partial = {}): AccessToken => ({ + token: 'oauth-access-token', + client_id: 'mcp-server', + scopes: ['foo'], + expires_at: null, + ...overrides, +}); + +const refreshToken = (overrides: Partial = {}): RefreshToken => ({ + token: 'oauth-refresh-token', + client_id: 'mcp-server', + scopes: ['foo'], + expires_at: null, + ...overrides, +}); + +const authorizationCode = (opts: { + scopes?: string[]; + expiresAt?: number; +}): ExtendedAuthorizationCode => ({ + code: 'foo', + scopes: opts.scopes ?? [], + expires_at: opts.expiresAt ?? Date.now() / 1000 + 5 * 60, // 5 minutes from now + client_id: 'foo-client-id', + code_challenge: 'foo-code-challenge', + redirect_uri: 'foo://bar', + redirect_uri_provided_explicitly: true, + oauth_access_token: accessToken(), + oauth_refresh_token: refreshToken(), +}); + +describe('SimpleOAuthProvider', () => { + describe('loadAuthorizationCode', () => { + it.each([ + { name: 'valid, no scopes', code: authorizationCode({}), key: JWT_KEY, valid: true }, + { + name: 'valid, scopes', + code: authorizationCode({ scopes: ['foo', 'bar'] }), + key: JWT_KEY, + valid: true, + }, + { + name: 'expired, no scopes', + code: authorizationCode({ expiresAt: 1 }), + key: JWT_KEY, + valid: true, + }, + { name: 'wrong encryption key', code: authorizationCode({}), key: '!@#$%^&', valid: false }, + ])('$name', async ({ code, key, valid }) => { + const provider = newProvider(); + const authCodeStr = await provider.encode(code, key); + const loaded = await provider.loadAuthorizationCode(authCodeStr); + if (valid) { + expect(loaded).toEqual(code); + } else { + expect(loaded).toBeNull(); + } + }); + }); + + describe('readOauthTokens', () => { + it.each([ + { rawAt: 'foo', rawRt: 'bar', scopes: ['email'], atExpiresIn: 3600, rtExpiresIn: 168 * 3600 }, + { + rawAt: 'foo', + rawRt: 'bar', + scopes: ['user', 'email'], + atExpiresIn: 3600, + rtExpiresIn: 168 * 3600, + }, + { rawAt: 'foo', rawRt: 'bar', scopes: [], atExpiresIn: 3600, rtExpiresIn: 168 * 3600 }, + // 168 * 1 second rounded up to the nearest hour -> 3600 + { rawAt: 'foo', rawRt: 'bar', scopes: [], atExpiresIn: 1, rtExpiresIn: 3600 }, + { rawAt: 'foo', rawRt: 'bar', scopes: [], atExpiresIn: 7200, rtExpiresIn: 168 * 3600 }, + ])( + 'scopes=$scopes atExpiresIn=$atExpiresIn', + ({ rawAt, rawRt, scopes, atExpiresIn, rtExpiresIn }) => { + const provider = newProvider(); + const [at, rt] = provider.readOauthTokens( + { access_token: rawAt, refresh_token: rawRt, expires_in: atExpiresIn }, + scopes, + ); + + const now = Date.now() / 1000; + expect(at.token).toBe(rawAt); + expect(at.scopes).toEqual(scopes); + expect(atExpiresIn - ((at.expires_at ?? 0) - now)).toBeGreaterThanOrEqual(0); + expect(atExpiresIn - ((at.expires_at ?? 0) - now)).toBeLessThan(1); + + expect(rt.token).toBe(rawRt); + expect(rt.scopes).toEqual(scopes); + expect(rtExpiresIn - ((rt.expires_at ?? 0) - now)).toBeGreaterThanOrEqual(0); + expect(rtExpiresIn - ((rt.expires_at ?? 0) - now)).toBeLessThan(1); + }, + ); + }); + + describe('validateRedirectUri', () => { + const cases: [string | null, boolean][] = [ + // === HTTP scheme - localhost only === + ['http://localhost:8080/foo', true], + ['http://localhost:20388/oauth/callback', true], + ['http://localhost/callback', true], + ['http://127.0.0.1:1234/bar', true], + ['http://127.0.0.1:54750/auth/callback', true], + ['http://127.0.0.1/callback', true], + // IPv6 localhost + ['http://[::1]:8080/callback', true], + ['http://[::1]/callback', true], + // HTTP to non-localhost should be rejected + ['http://example.com/callback', false], + ['http://keboola.com/callback', false], + ['http://192.168.1.1/callback', false], + // === HTTPS scheme - whitelisted domains === + ['https://foo.keboola.com/bar/baz', true], + ['https://bar.keboola.dev/baz', true], + ['https://connection.keboola.com/oauth/callback', true], + ['https://keboola.com/callback', false], // requires subdomain + ['https://keboola.dev/callback', false], // requires subdomain + // Data-app 'hub' subdomains are user-deployable and must be rejected (RISK-76) + ['https://my-app.hub.keboola.com/callback', false], + ['https://my-app.hub.north-europe.azure.keboola.com/callback', false], + ['https://my-app.hub.keboola.dev/callback', false], + ['https://hub.keboola.com/callback', false], // the hub root itself + ['https://my-app.hub.us-east4.gcp.keboola.com/callback', false], + // ChatGPT (subdomain optional) + ['https://chatgpt.com', true], + ['https://foo.chatgpt.com/bar', true], + ['https://chatgpt.com/connector_platform_oauth_redirect', true], + // Claude (subdomain optional) + ['https://claude.ai', true], + ['https://foo.claude.ai/bar', true], + ['https://claude.ai/api/mcp/auth_callback', true], + // LibreChat (no subdomains allowed) + ['https://librechat.glami-ml.com', true], + ['https://librechat.glami-ml.com/api/mcp/keboola/oauth/callback', true], + ['https://foo.librechat.glami-ml.com/bar', false], // no subdomains allowed + // Make.com (subdomain optional) + ['https://make.com', true], + ['https://foo.make.com/bar', true], + ['https://www.make.com/oauth/cb/mcp', true], + // Devin (exact domain only) + ['https://api.devin.ai/callback', true], + ['https://api.devin.ai', true], + ['https://devin.ai/callback', false], // must be api.devin.ai + ['https://foo.api.devin.ai/callback', false], // no subdomains + // Onyx (no subdomains allowed) + ['https://cloud.onyx.app', true], + ['https://cloud.onyx.app/mcp/oauth/callback', true], + ['https://foo.cloud.onyx.app/bar', false], // no subdomains allowed + ['https://onyx.app/callback', false], // must be cloud.onyx.app + // Azure APIM (no subdomains allowed) + ['https://global.consent.azure-apim.net', true], + ['https://global.consent.azure-apim.net/oauth/callback', true], + ['https://foo.global.consent.azure-apim.net/bar', false], // no subdomains allowed + // n8n at Groupon (no subdomains allowed) + ['https://n8n.groupondev.com', true], + ['https://n8n.groupondev.com/rest/oauth2-credential/callback', true], + ['https://n8n-business.groupondev.com', true], + ['https://n8n-business.groupondev.com/rest/oauth2-credential/callback', true], + ['https://n8n-merchant.groupondev.com', true], + ['https://n8n-merchant.groupondev.com/rest/oauth2-credential/callback', true], + ['https://n8n-llm-traffic.groupondev.com', true], + ['https://n8n-llm-traffic.groupondev.com/rest/oauth2-credential/callback', true], + ['https://n8n-finance.groupondev.com', true], + ['https://n8n-finance.groupondev.com/rest/oauth2-credential/callback', true], + ['https://n8n-playground.groupondev.com', true], + ['https://n8n-playground.groupondev.com/rest/oauth2-credential/callback', true], + ['https://n8n-staging.groupondev.com', true], + ['https://n8n-staging.groupondev.com/rest/oauth2-credential/callback', true], + ['https://foo.n8n-playground.groupondev.com/bar', false], // no subdomains allowed + ['https://n8n-unknown.groupondev.com', false], // not whitelisted + // Unknown HTTPS domains should be rejected + ['https://foo.bar.com/callback', false], + ['https://evil.com/callback', false], + ['https://fakechatgpt.com/callback', false], + ['https://evilclaude.ai/callback', false], + // === Cursor scheme - specific hosts only === + ['cursor://anysphere.cursor-retrieval/oauth/user-keboola-Data_warehouse/callback', true], + ['cursor://anysphere.cursor-mcp/oauth/callback', true], + ['cursor://anysphere.cursor-mcp/some/path', true], + // Cursor with unknown hosts should be rejected + ['cursor://evil.com/callback', false], + ['cursor://localhost/callback', false], + ['cursor://anysphere.cursor-other/callback', false], + // === Unknown/forbidden schemes should be rejected === + ['ftp://foo.bar.com', false], + ['file:///etc/passwd', false], + ['javascript://alert(1)', false], + ['data://text/html,', false], + // Custom schemes that are NOT whitelisted should be rejected + ['vscode://localhost/callback', false], + ['jetbrains://localhost/callback', false], + ['zed://localhost/callback', false], + ['myapp://localhost/callback', false], + ['evil://localhost/callback', false], + // === Edge cases === + [null, false], // no redirect_uri + ]; + + it.each(cases)('%s -> %s', (uri, valid) => { + if (valid) { + expect(validateRedirectUri(uri)).toBe(uri); + } else { + expect(() => validateRedirectUri(uri)).toThrow(InvalidRedirectUriError); + } + }); + }); + + describe('encode/decode', () => { + it('round-trips an arbitrary payload through the signed gzip JWS', async () => { + const provider = newProvider(); + const payload = { hello: 'world', nested: { n: [1, 2, 3] } }; + const encoded = await provider.encode(payload); + expect(await provider.decode(encoded)).toEqual(payload); + }); + + it('rejects a token signed with a different key', async () => { + const provider = newProvider(); + const encoded = await provider.encode({ a: 1 }, 'other-key'); + await expect(provider.decode(encoded)).rejects.toThrow(); + }); + }); + + describe('createSapiToken', () => { + it('POSTs to the Storage API tokens endpoint and returns the new token', async () => { + let body: Record = {}; + let auth: string | null = null; + server.use( + http.post('https://sapi/v2/storage/tokens', async ({ request }) => { + auth = request.headers.get('authorization'); + body = (await request.json()) as Record; + return HttpResponse.json({ token: 'new-sapi-token' }); + }), + ); + + const provider = newProvider(); + const token = await provider.createSapiToken('oauth-at', 7200); + expect(token).toBe('new-sapi-token'); + expect(auth).toBe('Bearer oauth-at'); + expect(body).toMatchObject({ + expiresIn: 7200, + canReadAllFileUploads: true, + canManageBuckets: true, + }); + }); + }); + + describe('exchangeAuthorizationCode', () => { + it('mints proxy access/refresh tokens carrying a SAPI token', async () => { + server.use( + http.post('https://sapi/v2/storage/tokens', () => + HttpResponse.json({ token: 'sapi-from-exchange' }), + ), + ); + + const provider = newProvider(); + const code = authorizationCode({ scopes: ['email'] }); + code.oauth_access_token = accessToken({ expires_at: Date.now() / 1000 + 3600 }); + code.oauth_refresh_token = refreshToken({ expires_at: Date.now() / 1000 + 168 * 3600 }); + + const result = await provider.exchangeAuthorizationCode('claude', code); + expect(result.token_type).toBe('Bearer'); + expect(result.scope).toBe('email'); + expect(result.expires_in).toBeGreaterThan(0); + + const decodedAccess = await provider.decode(result.access_token); + expect(decodedAccess['sapi_token']).toBe('sapi-from-exchange'); + expect(decodedAccess['client_id']).toBe('claude'); + expect((decodedAccess['token'] as string).startsWith('mcp_')).toBe(true); + }); + }); +}); diff --git a/__tests__/preview.test.ts b/__tests__/preview.test.ts new file mode 100644 index 000000000..b858390c7 --- /dev/null +++ b/__tests__/preview.test.ts @@ -0,0 +1,376 @@ +import { serve } from '@hono/node-server'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import type { AddressInfo } from 'node:net'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { runPreviewConfigDiff } from '@/preview'; +import { createHttpApp } from '@/transports/http'; + +// --------------------------------------------------------------------------- +// HTTP mocks (msw). The preview path issues: tokens/verify, configuration_detail, +// component fetch (ai + storage). Handlers are registered per-test via configure(). +// --------------------------------------------------------------------------- + +const msw = setupServer(); +beforeAll(() => + msw.listen({ + onUnhandledRequest: (request, print) => { + // Let the real Hono test server (localhost) through; error on anything else. + if (new URL(request.url).hostname === 'localhost') return; + print.error(); + }, + }), +); +afterEach(() => msw.resetHandlers()); +afterAll(() => msw.close()); + +const baseConfig = (over: Record = {}) => + new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok', ...over }); + +/** A loosely-typed config record shape for asserting on nested diff fields. */ +type Cfg = { + name?: string; + description?: string; + changeDescription?: string; + configuration: { parameters: Record }; +}; +const asCfg = (value: unknown): Cfg => value as Cfg; + +type ConfigureOpts = { + role?: string; + features?: string[]; + config?: Record | (() => never); + component?: Record; +}; + +/** + * Registers msw handlers for the storage + ai services. `config` is the value returned + * by configuration_detail / configuration_row_detail; pass a function to make it throw. + */ +const configure = (opts: ConfigureOpts = {}) => { + const role = opts.role ?? 'admin'; + const features = opts.features ?? []; + const component = opts.component ?? { + id: 'keboola.ex-test', + name: 'Test Extractor', + type: 'extractor', + configurationSchema: {}, + flags: [], + }; + + msw.use( + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { features }, admin: { role } }); + } + if (path.includes('/components/') && path.includes('/configs/')) { + if (typeof opts.config === 'function') { + return HttpResponse.json({ error: 'Invalid configuration ID' }, { status: 404 }); + } + return HttpResponse.json(opts.config ?? {}); + } + if (path.includes('/components/')) { + return HttpResponse.json(component); + } + return undefined; + }), + // Component docs from the AI service — 404 so fetchComponent falls back to storage. + http.get('https://ai.test/*', () => HttpResponse.json({ error: 'nope' }, { status: 404 })), + ); +}; + +// --------------------------------------------------------------------------- +// runPreviewConfigDiff (logic) — port of tests/test_preview.py. +// --------------------------------------------------------------------------- + +describe('runPreviewConfigDiff — authorization', () => { + it.each<[Record, number]>([ + [{}, 200], + [{ allowedTools: 'update_config,get_tables' }, 200], + [{ allowedTools: 'get_tables,get_buckets' }, 403], + [{ disallowedTools: 'update_config' }, 403], + [{ readOnlyMode: 'true' }, 403], + ])('header auth %j -> %s', async (over, expectedStatus) => { + configure({ config: { id: 'config-123', name: 'C', configuration: { parameters: {} } } }); + const rq = { + toolName: 'update_config', + toolParams: { + component_id: 'keboola.ex-test', + configuration_id: 'config-123', + change_description: 'Test change', + }, + }; + if (expectedStatus === 200) { + const resp = await runPreviewConfigDiff(baseConfig(over), rq); + expect(resp.isValid).toBe(true); + } else { + await expect(runPreviewConfigDiff(baseConfig(over), rq)).rejects.toMatchObject({ + status: 403, + message: expect.stringContaining('not authorized'), + }); + } + }); + + it.each<[string, string, string | undefined, string]>([ + ['update_config', 'readOnly', undefined, 'read-only operations'], + ['modify_streamlit_data_app', 'admin', 'dev-123', 'main production branch'], + ['update_flow', 'admin', undefined, 'admin/OAuth'], + ])('project/role/branch gate: %s role=%s -> %s', async (toolName, role, branchId, fragment) => { + configure({ role }); + const cfg = baseConfig(branchId ? { branchId } : {}); + await expect( + runPreviewConfigDiff(cfg, { toolName, toolParams: { configuration_id: 'cfg-1' } }), + ).rejects.toMatchObject({ status: 403, message: expect.stringContaining(fragment) }); + }); +}); + +describe('runPreviewConfigDiff — update_config diff', () => { + const originalConfigData = { + id: 'config-123', + name: 'Original Config Name', + description: 'Original description', + configuration: { parameters: { foo: 'bar', baz: 42 } }, + }; + + it('previews update_config with parameter updates, name and description', async () => { + configure({ config: structuredClone(originalConfigData) }); + const resp = await runPreviewConfigDiff(baseConfig(), { + toolName: 'update_config', + toolParams: { + component_id: 'keboola.ex-test', + configuration_id: 'config-123', + change_description: 'Test change', + name: 'Updated Config Name', + description: 'Updated description', + parameter_updates: [ + { op: 'set', path: 'foo', value: 'updated_bar' }, + { op: 'set', path: 'new_param', value: 'new_value' }, + ], + }, + }); + + expect(resp.coordinates).toMatchObject({ + componentId: 'keboola.ex-test', + configurationId: 'config-123', + }); + expect(resp.coordinates).not.toHaveProperty('configurationRowId'); + expect(resp.isValid).toBe(true); + expect(resp).not.toHaveProperty('validationErrors'); + + const original = asCfg(resp.originalConfig); + const updated = asCfg(resp.updatedConfig); + expect(original.name).toBe('Original Config Name'); + expect(original.configuration.parameters.foo).toBe('bar'); + expect(updated.name).toBe('Updated Config Name'); + expect(updated.description).toBe('Updated description'); + expect(updated.configuration.parameters.foo).toBe('updated_bar'); + expect(updated.configuration.parameters.new_param).toBe('new_value'); + expect(updated.configuration.parameters.baz).toBe(42); + expect(updated.changeDescription).toBe('Test change'); + }); + + it('returns isValid=false with empty configs when the mutator throws', async () => { + configure({ config: () => undefined as never }); + const resp = await runPreviewConfigDiff(baseConfig(), { + toolName: 'update_config', + toolParams: { + component_id: 'keboola.ex-test', + configuration_id: 'invalid-config', + change_description: 'Test change', + }, + }); + expect(resp.isValid).toBe(false); + expect((resp.validationErrors as string[]).length).toBeGreaterThan(0); + expect(resp.originalConfig).toEqual({}); + expect(resp.updatedConfig).toEqual({}); + }); + + it('leaves name/description unchanged when only required params are given', async () => { + configure({ + config: { + id: 'config-123', + name: 'Original Config', + description: 'Original description', + configuration: { parameters: { foo: 'bar' } }, + }, + }); + const resp = await runPreviewConfigDiff(baseConfig(), { + toolName: 'update_config', + toolParams: { + component_id: 'keboola.ex-test', + configuration_id: 'config-123', + change_description: 'Test change', + }, + }); + const updated = asCfg(resp.updatedConfig); + expect(resp.isValid).toBe(true); + expect(updated.name).toBe('Original Config'); + expect(updated.description).toBe('Original description'); + expect(updated.configuration.parameters.foo).toBe('bar'); + }); +}); + +describe('runPreviewConfigDiff — update_config_row diff', () => { + it('previews update_config_row including the row coordinate', async () => { + configure({ + config: { + id: 'row-456', + name: 'Original Row Name', + description: 'Original row description', + configuration: { parameters: { foo: 'bar', baz: 42 } }, + }, + }); + const resp = await runPreviewConfigDiff(baseConfig(), { + toolName: 'update_config_row', + toolParams: { + component_id: 'keboola.ex-test', + configuration_id: 'config-123', + configuration_row_id: 'row-456', + change_description: 'Row change', + parameter_updates: [{ op: 'set', path: 'foo', value: 'updated_bar' }], + }, + }); + expect(resp.isValid).toBe(true); + expect(resp.coordinates).toMatchObject({ + componentId: 'keboola.ex-test', + configurationId: 'config-123', + configurationRowId: 'row-456', + }); + expect(asCfg(resp.updatedConfig).configuration.parameters.foo).toBe('updated_bar'); + }); +}); + +describe('runPreviewConfigDiff — schema validation', () => { + it('returns isValid=false for a missing required param', async () => { + configure(); + const resp = await runPreviewConfigDiff(baseConfig(), { + toolName: 'update_config', + toolParams: { component_id: 'keboola.ex-test', change_description: 'Test' }, + }); + expect(resp.isValid).toBe(false); + expect(JSON.stringify(resp.validationErrors)).toContain('configuration_id'); + expect(resp.originalConfig).toEqual({}); + expect(resp.updatedConfig).toEqual({}); + }); + + it('returns isValid=false for an invalid parameter_updates entry', async () => { + configure(); + const resp = await runPreviewConfigDiff(baseConfig(), { + toolName: 'update_config', + toolParams: { + component_id: 'keboola.ex-test', + configuration_id: 'config-123', + change_description: 'Test change', + parameter_updates: [ + { op: 'set', path: 'foo', value: 'x' }, + { op: 'foo', path: 'bar', value: 'y' }, + ], + }, + }); + expect(resp.isValid).toBe(false); + expect(JSON.stringify(resp.validationErrors)).toContain('parameter_updates'); + }); +}); + +describe('runPreviewConfigDiff — unsupported tools', () => { + it('rejects an unknown tool name with 400', async () => { + configure(); + await expect( + runPreviewConfigDiff(baseConfig(), { + toolName: 'invalid_tool', + toolParams: { component_id: 'keboola.ex-test', configuration_id: 'config-123' }, + }), + ).rejects.toMatchObject({ status: 400 }); + }); + + it.each(['update_sql_transformation', 'update_flow', 'modify_streamlit_data_app'])( + 'returns 400 for the not-yet-diffable config tool %s', + async (toolName) => { + // Use a role/branch that clears the project/role/branch gate for each tool. + const role = toolName === 'update_flow' ? 'guest' : 'admin'; + configure({ role }); + const params: Record = { + configuration_id: 'cfg-1', + change_description: 'x', + }; + if (toolName === 'update_flow') params.flow_type = 'keboola.orchestrator'; + if (toolName === 'modify_streamlit_data_app') { + params.name = 'My App'; + params.description = 'desc'; + params.source_code = 'print(1)'; + params.packages = ['streamlit']; + params.authentication_type = 'default'; + } + await expect( + runPreviewConfigDiff(baseConfig(), { toolName, toolParams: params }), + ).rejects.toMatchObject({ status: 400, message: expect.stringContaining('not supported') }); + }, + ); +}); + +// --------------------------------------------------------------------------- +// Hono route — status codes + JSON body errors. +// --------------------------------------------------------------------------- + +describe('POST /preview/configuration (Hono route)', () => { + let httpServer: ReturnType; + let port: number; + + beforeAll(async () => { + const app = createHttpApp(baseConfig()); + port = await new Promise((resolve) => { + httpServer = serve({ fetch: app.fetch, port: 0 }, (info: AddressInfo) => resolve(info.port)); + }); + }); + afterAll(() => httpServer.close()); + + const post = (body: unknown, headers: Record = {}) => + fetch(`http://localhost:${port}/preview/configuration`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...headers }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }); + + it('returns 400 for malformed JSON', async () => { + const res = await post('{ not json'); + expect(res.status).toBe(400); + expect(((await res.json()) as { message: string }).message).toContain('Invalid JSON'); + }); + + it('returns 400 for a missing toolName', async () => { + const res = await post({ toolParams: {} }); + expect(res.status).toBe(400); + }); + + it('returns 200 with a diff for a valid request', async () => { + configure({ config: { id: 'config-123', name: 'C', configuration: { parameters: {} } } }); + const res = await post({ + toolName: 'update_config', + toolParams: { + component_id: 'keboola.ex-test', + configuration_id: 'config-123', + change_description: 'Test change', + }, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { isValid: boolean; coordinates: { componentId: string } }; + expect(body.isValid).toBe(true); + expect(body.coordinates.componentId).toBe('keboola.ex-test'); + }); + + it('returns 403 when header authorization denies the tool', async () => { + configure(); + const res = await post( + { + toolName: 'update_config', + toolParams: { component_id: 'keboola.ex-test', configuration_id: 'config-123' }, + }, + { 'X-Disallowed-Tools': 'update_config' }, + ); + expect(res.status).toBe(403); + expect(((await res.json()) as { message: string }).message).toContain('not authorized'); + }); +}); diff --git a/__tests__/prompts.test.ts b/__tests__/prompts.test.ts new file mode 100644 index 000000000..279507c00 --- /dev/null +++ b/__tests__/prompts.test.ts @@ -0,0 +1,44 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { createServer } from '@/server'; + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +const connect = async () => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 'test', version: '0.0.0' }); + await client.connect(clientT); + return client; +}; + +describe('prompts', () => { + it('registers the six one-click Keboola prompts', async () => { + const client = await connect(); + const { prompts } = await client.listPrompts(); + const names = prompts.map((p) => p.name).sort(); + expect(names).toEqual( + [ + 'analyze_project_structure', + 'component_usage_summary', + 'create_project_documentation', + 'data_quality_assessment', + 'error_analysis_report', + 'project_health_check', + ].sort(), + ); + await client.close(); + }); + + it('returns a single user message for a prompt', async () => { + const client = await connect(); + const result = await client.getPrompt({ name: 'project_health_check' }); + expect(result.messages).toHaveLength(1); + expect(result.messages[0]!.role).toBe('user'); + expect((result.messages[0]!.content as { text: string }).text).toContain('health check'); + await client.close(); + }); +}); diff --git a/__tests__/retry.test.ts b/__tests__/retry.test.ts new file mode 100644 index 000000000..b469f2a6f --- /dev/null +++ b/__tests__/retry.test.ts @@ -0,0 +1,65 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { createRetryMiddleware } from '@/clients/retry'; + +// A thrown error shaped like the api-client ApiError ({ response: Response }). +const apiError = (status: number): Error => + Object.assign(new Error(`HTTP ${status}`), { response: { status } }); + +const okResponse = { response: { status: 200 }, data: {} }; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('createRetryMiddleware', () => { + it('retries a 500 then returns the eventual success', async () => { + vi.useFakeTimers(); + let calls = 0; + const next = vi.fn(async () => { + calls += 1; + if (calls <= 2) throw apiError(500); + return okResponse; + }); + + const run = createRetryMiddleware(3)(next as never)({} as never); + await vi.runAllTimersAsync(); + const result = await run; + + expect(calls).toBe(3); // two 500s + one success + expect(result).toBe(okResponse); + }); + + it('does not retry a non-retryable status (400)', async () => { + const next = vi.fn(async () => { + throw apiError(400); + }); + await expect(createRetryMiddleware(3)(next as never)({} as never)).rejects.toThrow('HTTP 400'); + expect(next).toHaveBeenCalledTimes(1); + }); + + it('gives up after maxRetries and rethrows the last error', async () => { + vi.useFakeTimers(); + const next = vi.fn(async () => { + throw apiError(503); + }); + const run = createRetryMiddleware(2)(next as never)({} as never); + const assertion = expect(run).rejects.toThrow('HTTP 503'); + await vi.runAllTimersAsync(); + await assertion; + expect(next).toHaveBeenCalledTimes(3); // initial + 2 retries + }); + + it('retries when a retryable status is returned (not thrown)', async () => { + vi.useFakeTimers(); + let calls = 0; + const next = vi.fn(async () => { + calls += 1; + return calls === 1 ? { response: { status: 502 }, data: {} } : okResponse; + }); + const run = createRetryMiddleware(3)(next as never)({} as never); + await vi.runAllTimersAsync(); + expect(await run).toBe(okResponse); + expect(calls).toBe(2); + }); +}); diff --git a/__tests__/serialize.test.ts b/__tests__/serialize.test.ts new file mode 100644 index 000000000..8a02706dc --- /dev/null +++ b/__tests__/serialize.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest'; + +import { filterToonNulls, toonSerialize, toonSerializeCompact } from '@/serialize'; + +describe('filterToonNulls', () => { + it('drops null/undefined keys from a plain object', () => { + expect(filterToonNulls({ a: 1, b: null, c: undefined, d: 'x' })).toEqual({ a: 1, d: 'x' }); + }); + + it('drops null keys from a single-item object list', () => { + expect(filterToonNulls([{ a: 1, b: null }])).toEqual([{ a: 1 }]); + }); + + it('keeps union of value-bearing keys across a multi-item list, aligned with null', () => { + const input = [ + { id: 1, name: 'a', extra: null }, + { id: 2, name: null, extra: 'y' }, + ]; + // `name` and `extra` each have a value in some row, so both columns survive, + // with null where a given row lacks the value. + expect(filterToonNulls(input)).toEqual([ + { id: 1, name: 'a', extra: null }, + { id: 2, name: null, extra: 'y' }, + ]); + }); + + it('recurses into nested objects', () => { + expect(filterToonNulls({ a: { b: null, c: 2 } })).toEqual({ a: { c: 2 } }); + }); +}); + +describe('toon serializers', () => { + it('encodes an object list as an aligned TOON table', () => { + const text = toonSerialize([ + { id: 'a', name: 'x' }, + { id: 'b', name: 'y' }, + ]); + expect(text).toContain('[2]{id,name}'); + expect(text).toContain('a,x'); + }); + + it('compact form omits all-null columns', () => { + const text = toonSerializeCompact({ a: 1, b: null }); + expect(text).toContain('a: 1'); + expect(text).not.toContain('b:'); + }); +}); diff --git a/__tests__/server.test.ts b/__tests__/server.test.ts new file mode 100644 index 000000000..1b8bec5e5 --- /dev/null +++ b/__tests__/server.test.ts @@ -0,0 +1,39 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { createServer, SERVER_NAME } from '@/server'; + +const connect = async (config: Config) => { + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const server = createServer(config); + await server.connect(serverTransport); + + const client = new Client({ name: 'test-client', version: '0.0.0' }); + await client.connect(clientTransport); + return client; +}; + +describe('MCP server', () => { + it('lists registered tools over an in-memory transport', async () => { + const client = await connect(new Config()); + const { tools } = await client.listTools(); + + const names = tools.map((tool) => tool.name); + expect(names).toContain('get_server_info'); + // Every tool must expose an input schema (drives the tool-filtering / docs). + expect(tools.every((tool) => tool.inputSchema?.type === 'object')).toBe(true); + }); + + it('runs a tool and returns the server name', async () => { + const client = await connect(new Config({ branchId: '123' })); + const result = await client.callTool({ name: 'get_server_info', arguments: {} }); + + // Output is TOON, not JSON: a flat object renders as `key: value` lines. + const content = result.content as { type: string; text: string }[]; + const text = content[0]!.text; + expect(text).toContain(`name: ${SERVER_NAME}`); + expect(text).toContain('branchId: "123"'); + }); +}); diff --git a/__tests__/testproject.test.ts b/__tests__/testproject.test.ts new file mode 100644 index 000000000..f64601803 --- /dev/null +++ b/__tests__/testproject.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest'; + +import type { Locker } from '../integtests/testproject/locker'; +import { createPool } from '../integtests/testproject/pool'; +import { parseProjects } from '../integtests/testproject/projects'; +import { isCompatible, lockKey, type ProjectDefinition } from '../integtests/testproject/types'; + +const def = (over: Partial = {}): ProjectDefinition => ({ + host: 'connection.keboola.com', + project: 1, + token: 'tok', + backend: 'snowflake', + stagingStorage: 's3', + legacyTransformation: false, + isGuest: false, + ...over, +}); + +/** In-memory locker for deterministic pool tests (no redis/fs). */ +const fakeLocker = (): { locker: Locker; held: Set } => { + const held = new Set(); + return { + held, + locker: { + forProject: (d) => ({ + tryLock: async () => { + const k = lockKey(d); + if (held.has(k)) return null; + held.add(k); + return async () => { + held.delete(k); + }; + }, + }), + close: async () => {}, + }, + }; +}; + +describe('parseProjects', () => { + it('parses the array form and applies defaults', () => { + const defs = parseProjects( + JSON.stringify([ + { + host: 'connection.keboola.com', + project: 5684, + token: 't', + backend: 'bigquery', + stagingStorage: 'gcs', + }, + ]), + ); + expect(defs).toHaveLength(1); + expect(defs[0]!.project).toBe(5684); + expect(defs[0]!.legacyTransformation).toBe(false); + expect(defs[0]!.isGuest).toBe(false); + }); + + it('rejects an empty array', () => { + expect(() => parseProjects('[]')).toThrow(/non-empty array/); + }); + + it('rejects an entry missing a required field', () => { + expect(() => + parseProjects( + JSON.stringify([{ host: 'h', project: 1, backend: 'snowflake', stagingStorage: 's3' }]), + ), + ).toThrow(/\[0\] is invalid/); + }); +}); + +describe('isCompatible', () => { + it('matches any backend when none requested, else exact', () => { + expect(isCompatible(def({ backend: 'snowflake' }), {})).toBe(true); + expect(isCompatible(def({ backend: 'snowflake' }), { backend: 'snowflake' })).toBe(true); + expect(isCompatible(def({ backend: 'snowflake' }), { backend: 'bigquery' })).toBe(false); + }); +}); + +describe('createPool.getTestProject', () => { + it('leases a free project and exposes its URL/token', async () => { + const { locker, held } = fakeLocker(); + const pool = createPool([def({ project: 1 })], locker); + const p = await pool.getTestProject(); + expect(p.storageApiUrl).toBe('https://connection.keboola.com'); + expect(p.storageApiToken).toBe('tok'); + expect(held.size).toBe(1); + await p.release(); + expect(held.size).toBe(0); + }); + + it('honors the backend selector', async () => { + const { locker } = fakeLocker(); + const pool = createPool( + [def({ project: 1, backend: 'snowflake' }), def({ project: 2, backend: 'bigquery' })], + locker, + ); + const p = await pool.getTestProject({ backend: 'bigquery' }); + expect(p.definition.project).toBe(2); + }); + + it('throws when no compatible project exists', async () => { + const { locker } = fakeLocker(); + const pool = createPool([def({ backend: 'snowflake' })], locker); + await expect(pool.getTestProject({ backend: 'bigquery' })).rejects.toThrow(/No compatible/); + }); + + it('retries (does not error) until a busy project frees up', async () => { + const { locker, held } = fakeLocker(); + const d = def({ project: 7 }); + held.add(lockKey(d)); // pre-hold the only project + const pool = createPool([d], locker); + + const acquired = pool.getTestProject(); + // Free it shortly after; the pool should keep retrying and then resolve. + setTimeout(() => held.delete(lockKey(d)), 250); + + const p = await acquired; + expect(p.definition.project).toBe(7); + }); +}); diff --git a/__tests__/tools.components.test.ts b/__tests__/tools.components.test.ts new file mode 100644 index 000000000..3cf33a06f --- /dev/null +++ b/__tests__/tools.components.test.ts @@ -0,0 +1,645 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { createServer } from '@/server'; +import { + cleanBucketName, + createTransformationConfiguration, + joinSqlStatements, + splitSqlStatements, + updateParams, + updateTransformationParameters, +} from '@/tools/components'; +import { + __testing, + validateRootParametersConfiguration, + validateRootStorageConfiguration, +} from '@/tools/validation'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +const callTool = async (name: string, args: Record) => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 't', version: '0' }); + await client.connect(clientT); + const result = await client.callTool({ name, arguments: args }); + const text = (result.content as { text: string }[])[0]!.text; + await client.close(); + return { text, isError: result.isError }; +}; + +describe('get_config_examples', () => { + it('renders root and row configuration examples as markdown', async () => { + server.use( + http.get('https://ai.test/*', ({ request }) => { + expect(new URL(request.url).pathname).toBe('/docs/components/keboola.ex-aws-s3'); + return HttpResponse.json({ + rootConfigurationExamples: [{ foo: 'bar' }], + rowConfigurationExamples: [{ baz: 1 }], + }); + }), + ); + + const { text } = await callTool('get_config_examples', { component_id: 'keboola.ex-aws-s3' }); + expect(text).toContain('# Configuration Examples for `keboola.ex-aws-s3`'); + expect(text).toContain('## Root Configuration Examples'); + expect(text).toContain('"foo": "bar"'); + expect(text).toContain('## Row Configuration Examples'); + }); + + it('returns an empty string when the component lookup fails', async () => { + server.use(http.get('https://ai.test/*', () => new HttpResponse(null, { status: 404 }))); + const { text, isError } = await callTool('get_config_examples', { component_id: 'nope' }); + expect(isError).toBeFalsy(); + expect(text).toBe(''); + }); +}); + +const verify = () => + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + return undefined; + }); + +describe('get_components', () => { + it('merges AI catalog metadata with Storage data and derives capabilities', async () => { + server.use( + verify(), + http.get('https://ai.test/*', () => + HttpResponse.json({ + id: 'keboola.ex-aws-s3', + name: 'AWS S3', + type: 'extractor', + flags: ['genericDockerUI-rows', 'genericDockerUI-tableOutput'], + documentation: 'docs here', + }), + ), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (pathname.includes('/components/')) { + return HttpResponse.json({ data: { synchronous_actions: ['testConnection'] } }); + } + return undefined; + }), + ); + + const { text } = await callTool('get_components', { component_ids: ['keboola.ex-aws-s3'] }); + expect(text).toContain('AWS S3'); + expect(text).toContain('is_row_based: true'); + expect(text).toContain('testConnection'); + expect(text).toContain('docs here'); + }); + + it('falls back to the Storage API when the AI catalog returns 404', async () => { + server.use( + verify(), + http.get('https://ai.test/*', () => new HttpResponse(null, { status: 404 })), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + return HttpResponse.json({ id: 'priv.comp', name: 'Private', type: 'writer', flags: [] }); + }), + ); + + const { text } = await callTool('get_components', { component_ids: ['priv.comp'] }); + expect(text).toContain('Private'); + }); +}); + +describe('get_configs', () => { + it('lists configs by component id grouped under the component', async () => { + server.use( + verify(), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (pathname.endsWith('/components/keboola.ex-db-mysql/configs')) { + return HttpResponse.json([{ id: '100', name: 'My DB', isDisabled: false }]); + } + if (pathname.endsWith('/components/keboola.ex-db-mysql')) { + return HttpResponse.json({ + id: 'keboola.ex-db-mysql', + name: 'MySQL', + type: 'extractor', + flags: [], + }); + } + return undefined; + }), + ); + + const { text } = await callTool('get_configs', { component_ids: ['keboola.ex-db-mysql'] }); + expect(text).toContain('My DB'); + expect(text).toContain('MySQL'); + }); + + it('returns full details with redacted secrets for specific configs', async () => { + server.use( + verify(), + http.get('https://ai.test/*', () => new HttpResponse(null, { status: 404 })), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (pathname.endsWith('/configs/100')) { + return HttpResponse.json({ + id: '100', + name: 'My DB', + version: 3, + configuration: { parameters: { host: 'db', '#password': 'plaintext' } }, + }); + } + if (pathname.endsWith('/components/keboola.ex-db-mysql')) { + return HttpResponse.json({ + id: 'keboola.ex-db-mysql', + name: 'MySQL', + type: 'extractor', + flags: [], + }); + } + return undefined; + }), + ); + + const { text } = await callTool('get_configs', { + configs: [{ component_id: 'keboola.ex-db-mysql', configuration_id: '100' }], + }); + expect(text).toContain('[REDACTED]'); + expect(text).not.toContain('plaintext'); + expect(text).toContain('host'); + }); +}); + +describe('run_sync_action', () => { + it('merges row config over root and posts to the sync-actions endpoint', async () => { + let actionBody: { configData?: Record; action?: string } | undefined; + server.use( + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/configs/cfg/rows/r1')) { + return HttpResponse.json({ + configuration: { parameters: { row: 1 }, storage: { input: 'r' } }, + }); + } + if (pathname.endsWith('/configs/cfg')) { + return HttpResponse.json({ + configuration: { + parameters: { base: 1 }, + storage: {}, + authorization: { oauth_api: { id: 'a' } }, + }, + }); + } + return undefined; + }), + http.post('https://sync-actions.test/*', async ({ request }) => { + expect(new URL(request.url).pathname).toBe('/actions'); + actionBody = (await request.json()) as { + configData?: Record; + action?: string; + }; + return HttpResponse.json({ status: 'success', tables: [] }); + }), + ); + + const { text } = await callTool('run_sync_action', { + action_name: 'getTables', + component_id: 'keboola.ex-db-mysql', + configuration_id: 'cfg', + configuration_row_id: 'r1', + }); + + expect(actionBody?.action).toBe('getTables'); + // row parameters merged on top of root; authorization carried from root. + expect(actionBody?.configData).toMatchObject({ + parameters: { base: 1, row: 1 }, + authorization: { oauth_api: { id: 'a' } }, + }); + expect(text).toContain('success'); + }); +}); + +// --------------------------------------------------------------------------- +// WRITE TOOLS +// --------------------------------------------------------------------------- + +/** AI catalog 404 so fetchComponent falls back to Storage component_detail. */ +const aiNotFound = () => + http.get('https://ai.test/*', () => new HttpResponse(null, { status: 404 })); + +describe('create_config', () => { + it('validates, creates the config, and stamps creation metadata', async () => { + let createBody: Record | undefined; + let metadataBody: Record | undefined; + server.use( + verify(), + aiNotFound(), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (pathname.endsWith('/components/keboola.ex-generic')) { + return HttpResponse.json({ + id: 'keboola.ex-generic', + name: 'Generic', + type: 'extractor', + flags: [], + configurationSchema: { + type: 'object', + required: ['host'], + properties: { host: { type: 'string' } }, + }, + }); + } + return undefined; + }), + http.post('https://connection.test/*', async ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/configs')) { + createBody = (await request.json()) as Record; + return HttpResponse.json({ id: '555', version: 1 }); + } + if (pathname.endsWith('/metadata')) { + metadataBody = (await request.json()) as Record; + return HttpResponse.json([]); + } + return undefined; + }), + ); + + const { text, isError } = await callTool('create_config', { + name: 'My config', + description: 'desc', + component_id: 'keboola.ex-generic', + parameters: { host: 'db.example.com' }, + }); + + expect(isError).toBeFalsy(); + expect(createBody?.name).toBe('My config'); + expect((createBody?.configuration as Record).parameters).toMatchObject({ + host: 'db.example.com', + }); + // creation metadata is KBC.MCP.createdBy + expect(JSON.stringify(metadataBody)).toContain('KBC.MCP.createdBy'); + expect(text).toContain('555'); + expect(text).toContain('version: 1'); + }); + + it('fails a schema-required violation with a recoverable hint', async () => { + server.use( + verify(), + aiNotFound(), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + return HttpResponse.json({ + id: 'keboola.ex-generic', + name: 'Generic', + type: 'extractor', + flags: [], + configurationSchema: { type: 'object', required: ['host'], properties: {} }, + }); + }), + ); + + const { text, isError } = await callTool('create_config', { + name: 'X', + description: 'd', + component_id: 'keboola.ex-generic', + parameters: {}, + }); + + expect(isError).toBe(true); + expect(text).toContain('required property'); + expect(text).toContain('HINT: Ensure ALL of the following required fields'); + }); + + it('refuses suitable-only components (SQL transformation)', async () => { + const { text, isError } = await callTool('create_config', { + name: 'X', + description: 'd', + component_id: 'keboola.snowflake-transformation', + parameters: {}, + }); + expect(isError).toBe(true); + expect(text).toContain('cannot be used with keboola.snowflake-transformation'); + }); +}); + +describe('update_config', () => { + it('applies a parameter diff over the existing config and bumps metadata', async () => { + let putBody: Record | undefined; + server.use( + verify(), + aiNotFound(), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (pathname.endsWith('/configs/100')) { + return HttpResponse.json({ + id: '100', + name: 'Cfg', + version: 3, + configuration: { parameters: { host: 'old', port: 5432 }, storage: {} }, + }); + } + if (pathname.endsWith('/components/keboola.ex-db-mysql')) { + return HttpResponse.json({ + id: 'keboola.ex-db-mysql', + name: 'MySQL', + type: 'extractor', + flags: [], + }); + } + return undefined; + }), + http.put('https://connection.test/*', async ({ request }) => { + putBody = (await request.json()) as Record; + return HttpResponse.json({ id: '100', name: 'Cfg', version: 4, description: 'd' }); + }), + http.post('https://connection.test/*', () => HttpResponse.json([])), + ); + + const { text, isError } = await callTool('update_config', { + change_description: 'switch host', + component_id: 'keboola.ex-db-mysql', + configuration_id: '100', + parameter_updates: [{ op: 'set', path: 'host', value: 'new' }], + }); + + expect(isError).toBeFalsy(); + const cfg = putBody?.configuration as Record; + expect(cfg.parameters).toMatchObject({ host: 'new', port: 5432 }); // diff preserved port + expect(putBody?.changeDescription).toBe('switch host'); + expect(text).toContain('version: 4'); + }); +}); + +describe('update_config_row', () => { + it('updates a row with str_replace diff and is_disabled', async () => { + let putBody: Record | undefined; + server.use( + verify(), + aiNotFound(), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (pathname.endsWith('/rows/r1')) { + return HttpResponse.json({ + id: 'r1', + configuration: { parameters: { table: 'old_name' } }, + }); + } + if (pathname.endsWith('/components/keboola.ex-db-mysql')) { + return HttpResponse.json({ + id: 'keboola.ex-db-mysql', + name: 'M', + type: 'extractor', + flags: [], + }); + } + return undefined; + }), + http.put('https://connection.test/*', async ({ request }) => { + putBody = (await request.json()) as Record; + return HttpResponse.json({ id: 'r1', version: 2 }); + }), + http.post('https://connection.test/*', () => HttpResponse.json([])), + ); + + const { isError } = await callTool('update_config_row', { + change_description: 'rename table', + component_id: 'keboola.ex-db-mysql', + configuration_id: '100', + configuration_row_id: 'r1', + parameter_updates: [ + { op: 'str_replace', path: 'table', search_for: 'old', replace_with: 'new' }, + ], + is_disabled: true, + }); + + expect(isError).toBeFalsy(); + const cfg = putBody?.configuration as Record; + expect(cfg.parameters).toMatchObject({ table: 'new_name' }); + expect(putBody?.isDisabled).toBe(true); + }); +}); + +describe('create_sql_transformation', () => { + it('resolves dialect from token defaultBackend and builds the transformation payload', async () => { + let createBody: Record | undefined; + let createPath = ''; + server.use( + aiNotFound(), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42', defaultBackend: 'snowflake' } }); + } + if (pathname.endsWith('/workspaces')) return HttpResponse.json([]); + // search/component-configurations folder lookup + if (pathname.includes('/search/component-configurations')) return HttpResponse.json([]); + if (pathname.endsWith('/configs')) return HttpResponse.json([]); // configuration_list for folder count + if (pathname.includes('/components/keboola.snowflake-transformation')) { + return HttpResponse.json({ + id: 'keboola.snowflake-transformation', + name: 'Snowflake', + type: 'transformation', + flags: [], + }); + } + return undefined; + }), + http.post('https://connection.test/*', async ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/configs')) { + createPath = pathname; + createBody = (await request.json()) as Record; + return HttpResponse.json({ id: 'tf1', version: 1 }); + } + if (pathname.endsWith('/metadata')) return HttpResponse.json([]); + return undefined; + }), + ); + + const { text, isError } = await callTool('create_sql_transformation', { + name: 'My TF', + description: 'transform stuff', + sql_code_blocks: [{ name: 'step', script: 'CREATE TABLE out AS SELECT 1;' }], + created_table_names: ['out'], + }); + + expect(isError).toBeFalsy(); + expect(createPath).toContain('keboola.snowflake-transformation'); + const cfg = createBody?.configuration as Record; + const storage = cfg.storage as { output: { tables: { destination: string }[] } }; + expect(storage.output.tables[0]!.destination).toContain('out.c-'); + expect(text).toContain('tf1'); + }); +}); + +describe('update_sql_transformation', () => { + it('returns a Python/R hint when the config is missing (404)', async () => { + server.use( + aiNotFound(), + http.get('https://connection.test/*', ({ request }) => { + const { pathname } = new URL(request.url); + if (pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42', defaultBackend: 'snowflake' } }); + } + if (pathname.endsWith('/workspaces')) return HttpResponse.json([]); + if (pathname.includes('/configs/nope')) return new HttpResponse(null, { status: 404 }); + return undefined; + }), + ); + + const { text, isError } = await callTool('update_sql_transformation', { + change_description: 'x', + configuration_id: 'nope', + }); + + expect(isError).toBe(true); + expect(text).toContain("use 'update_config'"); + expect(text).toContain('keboola.python-transformation-v2'); + }); +}); + +// --------------------------------------------------------------------------- +// VALIDATION + MODEL UNIT TESTS +// --------------------------------------------------------------------------- + +describe('validation: sanitizeSchema', () => { + it.each([ + [{ type: 'object', required: true }, { type: 'object' }], + [{ type: 'object', required: false }, { type: 'object' }], + [ + { type: 'object', required: ['foo', 'bar'] }, + { type: 'object', required: ['foo', 'bar'] }, + ], + [{ type: 'string', enum: [] }, { type: 'string' }], + ])('normalizes required/enum %#', (input, expected) => { + expect(__testing.sanitizeSchema(input as Record)).toEqual(expected); + }); + + it('propagates a boolean child-required flag up to the parent', () => { + const out = __testing.sanitizeSchema({ + type: 'object', + properties: { foo: { type: 'string', required: true } }, + }); + expect(out).toEqual({ + type: 'object', + required: ['foo'], + properties: { foo: { type: 'string' } }, + }); + }); + + it('converts an empty-list properties to an empty dict', () => { + const out = __testing.sanitizeSchema({ type: 'object', properties: [] }); + expect(out.properties).toEqual({}); + }); +}); + +describe('validation: parameters & storage', () => { + const component = { + component_id: 'keboola.ex-generic', + component_type: 'extractor', + capabilities: { is_row_based: false }, + configuration_schema: { + type: 'object', + required: ['host'], + properties: { host: { type: 'string' }, port: { type: 'integer' } }, + }, + }; + + it('accepts valid parameters', () => { + expect(() => + validateRootParametersConfiguration({ host: 'h', port: 1 }, component), + ).not.toThrow(); + }); + + it('rejects wrong type', () => { + expect(() => + validateRootParametersConfiguration({ host: 'h', port: 'nope' }, component), + ).toThrow(/not of type/); + }); + + it('skips validation when the component has no schema', () => { + const noSchema = { ...component, configuration_schema: null }; + expect(() => validateRootParametersConfiguration({ anything: true }, noSchema)).not.toThrow(); + }); + + it('requires writer root storage to contain input mappings', () => { + const writer = { + component_id: 'keboola.wr-x', + component_type: 'writer', + capabilities: { is_row_based: false }, + configuration_schema: null, + }; + expect(() => validateRootStorageConfiguration({}, writer)).toThrow(/must contain "input"/); + }); +}); + +describe('model: SQL + param utils', () => { + it('splits and joins SQL statements', () => { + const stmts = splitSqlStatements('SELECT 1; SELECT 2;'); + expect(stmts).toHaveLength(2); + expect(joinSqlStatements(stmts)).toContain('SELECT 1'); + }); + + it('updateParams applies a diff without mutating the input', () => { + const original = { a: 1, b: { c: 2 } }; + const out = updateParams(original, [{ op: 'set', path: 'b.c', value: 9 }]); + expect(out).toMatchObject({ a: 1, b: { c: 9 } }); + expect(original.b.c).toBe(2); + }); + + it('updateParams str_replace and remove', () => { + const out = updateParams({ name: 'old_table', drop: 1 }, [ + { op: 'str_replace', path: 'name', search_for: 'old', replace_with: 'new' }, + { op: 'remove', path: 'drop' }, + ]); + expect(out).toEqual({ name: 'new_table' }); + }); + + it('updateTransformationParameters renames a code and summarizes structure on structural change', () => { + const [updated, summary] = updateTransformationParameters( + { blocks: [{ name: 'B', codes: [{ name: 'c', script: 'SELECT 1' }] }] }, + [ + { + op: 'add_code', + block_id: 'b0', + code: { name: 'c2', script: 'SELECT 2' }, + position: 'end', + }, + ], + ); + expect(updated.blocks[0]!.codes).toHaveLength(2); + expect(summary).toContain('Updated Transformation Structure'); + }); + + it('cleanBucketName folds diacritics and strips invalid chars', () => { + expect(cleanBucketName('Český Bucket!')).toBe('Cesky-Bucket'); + }); + + it('createTransformationConfiguration builds output table destinations', () => { + const cfg = createTransformationConfiguration( + [{ name: 's', script: 'CREATE TABLE t AS SELECT 1;' }], + 'My TF', + ['t'], + ) as { storage: { output: { tables: { source: string; destination: string }[] } } }; + expect(cfg.storage.output.tables[0]).toMatchObject({ + source: 't', + destination: 'out.c-My-TF.t', + }); + }); +}); diff --git a/__tests__/tools.data_apps.test.ts b/__tests__/tools.data_apps.test.ts new file mode 100644 index 000000000..2bc4b7beb --- /dev/null +++ b/__tests__/tools.data_apps.test.ts @@ -0,0 +1,553 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { registerDataAppTools } from '@/tools/data_apps'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const config = new Config({ + storageApiUrl: 'https://connection.test', + storageToken: 'tok', + workspaceSchema: 'WS_SCHEMA', +}); + +const connect = async (cfg: Config = config) => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + const mcp = new McpServer({ name: 'test', version: '0.0.0' }); + registerDataAppTools(mcp, cfg); + await mcp.connect(serverT); + const client = new Client({ name: 'test', version: '0.0.0' }); + await client.connect(clientT); + return client; +}; + +// tokens/verify backs both the links manager (project id) and feature checks. +const verifyHandler = (features: string[] = []) => + http.get('https://connection.test/*', ({ request }) => { + if (new URL(request.url).pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42', features } }); + } + return undefined; + }); + +const call = async ( + client: Awaited>, + name: string, + args: Record, +) => { + const result = await client.callTool({ name, arguments: args }); + return result; +}; + +const text = (result: Awaited>): string => + (result.content as { text: string }[])[0]!.text; + +describe('get_data_apps', () => { + it('lists data app summaries filtered to the data-apps component', async () => { + server.use( + verifyHandler(), + http.get('https://data-science.test/apps', () => + HttpResponse.json([ + { + id: 'app1', + projectId: '42', + componentId: 'keboola.data-apps', + branchId: null, + configId: 'cfg1', + configVersion: '3', + type: 'streamlit', + state: 'running', + desiredState: 'running', + url: 'https://app1.run', + }, + { + id: 'other', + projectId: '42', + componentId: 'keboola.other', + branchId: null, + configId: 'cfgX', + configVersion: '1', + type: 'streamlit', + state: 'stopped', + desiredState: 'stopped', + }, + ]), + ), + ); + const result = await call(await connect(), 'get_data_apps', {}); + expect(result.isError).toBeFalsy(); + const out = text(result); + expect(out).toContain('cfg1'); + expect(out).not.toContain('cfgX'); // filtered out (wrong component) + expect(out).toContain('data-apps'); // dashboard link + }); + + it('returns detail with deployment info and drafts for a python-js prod app', async () => { + const prodConfig = { + id: 'prod-cfg', + name: 'My Prod', + version: 5, + configuration: { parameters: { id: 'prod-app' } }, + metadata: [], + }; + const draftConfig = { + id: 'draft-cfg', + name: 'My Draft', + version: 2, + configuration: { + parameters: { + id: 'draft-app', + dataApp: { isDraft: true, parentConfigurationId: 'prod-cfg' }, + }, + }, + metadata: [], + }; + const appResponse = (id: string, cfgId: string) => ({ + id, + projectId: '42', + componentId: 'keboola.data-apps', + branchId: null, + configId: cfgId, + configVersion: '5', + type: 'python-js', + state: 'running', + desiredState: 'running', + url: `https://${id}.run`, + }); + + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (path.endsWith('/configs/prod-cfg')) return HttpResponse.json(prodConfig); + if (path.endsWith('/configs/draft-cfg')) return HttpResponse.json(draftConfig); + if (path.endsWith('/keboola.data-apps/configs')) + return HttpResponse.json([prodConfig, draftConfig]); + return undefined; + }), + http.get('https://data-science.test/apps/prod-app', () => + HttpResponse.json(appResponse('prod-app', 'prod-cfg')), + ), + http.get('https://data-science.test/apps/draft-app', () => + HttpResponse.json(appResponse('draft-app', 'draft-cfg')), + ), + http.get('https://data-science.test/apps/:id/git-repo', () => + HttpResponse.json({ httpsUrl: 'https://git.test/repo.git', isManagedGitRepo: true }), + ), + http.get('https://data-science.test/apps/:id/logs/tail', () => + HttpResponse.text('log line 1\nlog line 2'), + ), + http.get('https://data-science.test/apps/:id/runs', () => + HttpResponse.json([{ id: 'run1', state: 'running', appId: 'prod-app' }]), + ), + ); + + const result = await call(await connect(), 'get_data_apps', { + configuration_ids: ['prod-cfg'], + }); + expect(result.isError).toBeFalsy(); + const out = text(result); + expect(out).toContain('prod-cfg'); + expect(out).toContain('https://git.test/repo.git'); // repo_url on detail path + expect(out).toContain('draft-cfg'); // inline drafts + expect(out).toContain('log line 1'); // deployment logs + }); +}); + +describe('create_python_js_data_app_git_credential', () => { + it('mints a token and returns an authenticated clone URL', async () => { + const cfg = { + id: 'prod-cfg', + name: 'Prod', + version: 1, + configuration: { parameters: { id: 'prod-app' } }, + metadata: [], + }; + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (path.endsWith('/configs/prod-cfg')) return HttpResponse.json(cfg); + return undefined; + }), + http.get('https://data-science.test/apps/prod-app', () => + HttpResponse.json({ + id: 'prod-app', + projectId: '42', + componentId: 'keboola.data-apps', + branchId: null, + configId: 'prod-cfg', + configVersion: '1', + type: 'python-js', + state: 'stopped', + desiredState: 'stopped', + }), + ), + http.get('https://data-science.test/apps/prod-app/git-repo', () => + HttpResponse.json({ httpsUrl: 'https://git.test/my/repo.git', isManagedGitRepo: true }), + ), + http.post('https://data-science.test/apps/prod-app/git-repo/credentials', () => + HttpResponse.json({ + id: 'cred1', + type: 'http_token', + permissions: 'readWrite', + secret: 'sup3r/secret', + }), + ), + ); + + const result = await call(await connect(), 'create_python_js_data_app_git_credential', { + configuration_id: 'prod-cfg', + }); + expect(result.isError).toBeFalsy(); + const out = text(result); + // username embedded + secret URL-encoded. + expect(out).toContain('https://kai:sup3r%2Fsecret@git.test/my/repo.git'); + expect(out).toContain('readWrite'); + }); + + it('rejects a streamlit app', async () => { + const cfg = { + id: 'st-cfg', + name: 'St', + version: 1, + configuration: { parameters: { id: 'st-app' } }, + metadata: [], + }; + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (path.endsWith('/configs/st-cfg')) return HttpResponse.json(cfg); + return undefined; + }), + http.get('https://data-science.test/apps/st-app', () => + HttpResponse.json({ + id: 'st-app', + projectId: '42', + componentId: 'keboola.data-apps', + branchId: null, + configId: 'st-cfg', + configVersion: '1', + type: 'streamlit', + state: 'stopped', + desiredState: 'stopped', + }), + ), + ); + const result = await call(await connect(), 'create_python_js_data_app_git_credential', { + configuration_id: 'st-cfg', + }); + expect(result.isError).toBeTruthy(); + expect(text(result)).toContain('only supports python-js'); + }); +}); + +describe('deploy_data_app', () => { + const streamlitCfg = { + id: 'st-cfg', + name: 'St', + version: 7, + configuration: { parameters: { id: 'st-app' }, authorization: {} }, + metadata: [], + }; + const stApp = (state: string) => ({ + id: 'st-app', + projectId: '42', + componentId: 'keboola.data-apps', + branchId: null, + configId: 'st-cfg', + configVersion: '7', + type: 'streamlit', + state, + desiredState: 'running', + url: 'https://st.run', + }); + + it('deploys a streamlit app fetching the latest config version', async () => { + let patchBody: Record = {}; + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (path.endsWith('/configs/st-cfg/versions')) + return HttpResponse.json([{ version: 6 }, { version: 7 }]); + if (path.endsWith('/configs/st-cfg')) return HttpResponse.json(streamlitCfg); + return undefined; + }), + http.get('https://data-science.test/apps/st-app', () => HttpResponse.json(stApp('running'))), + http.patch('https://data-science.test/apps/st-app', async ({ request }) => { + patchBody = (await request.json()) as Record; + return HttpResponse.json(stApp('running')); + }), + http.get('https://data-science.test/apps/st-app/logs/tail', () => HttpResponse.text('hi')), + http.get('https://data-science.test/apps/st-app/runs', () => HttpResponse.json([])), + ); + const result = await call(await connect(), 'deploy_data_app', { + action: 'deploy', + configuration_id: 'st-cfg', + }); + expect(result.isError).toBeFalsy(); + expect(patchBody.configVersion).toBe('7'); // latest version sent for streamlit + expect(text(result)).toContain('running'); + }); + + it('refuses to deploy an app that is stopping', async () => { + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (path.endsWith('/configs/st-cfg')) return HttpResponse.json(streamlitCfg); + return undefined; + }), + http.get('https://data-science.test/apps/st-app', () => HttpResponse.json(stApp('stopping'))), + ); + const result = await call(await connect(), 'deploy_data_app', { + action: 'deploy', + configuration_id: 'st-cfg', + }); + expect(result.isError).toBeTruthy(); + expect(text(result)).toContain('stopping'); + }); +}); + +describe('delete_python_js_data_app_draft', () => { + it('deletes a draft and surfaces the parent configuration id', async () => { + const draftCfg = { + id: 'draft-cfg', + name: 'Draft', + version: 1, + configuration: { + parameters: { + id: 'draft-app', + dataApp: { isDraft: true, parentConfigurationId: 'prod-cfg' }, + }, + }, + metadata: [], + }; + let deleted = false; + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (path.endsWith('/configs/draft-cfg')) return HttpResponse.json(draftCfg); + return undefined; + }), + http.get('https://data-science.test/apps/draft-app', () => + HttpResponse.json({ + id: 'draft-app', + projectId: '42', + componentId: 'keboola.data-apps', + branchId: null, + configId: 'draft-cfg', + configVersion: '1', + type: 'python-js', + state: 'stopped', + desiredState: 'stopped', + }), + ), + http.get('https://data-science.test/apps/draft-app/git-repo', () => + HttpResponse.json({ httpsUrl: 'https://git.test/repo.git', isManagedGitRepo: true }), + ), + http.delete('https://data-science.test/apps/draft-app', () => { + deleted = true; + return new HttpResponse(null, { status: 204 }); + }), + ); + const result = await call(await connect(), 'delete_python_js_data_app_draft', { + configuration_id: 'draft-cfg', + }); + expect(result.isError).toBeFalsy(); + expect(deleted).toBe(true); + const out = text(result); + expect(out).toContain('deleted'); + expect(out).toContain('prod-cfg'); + }); + + it('refuses to delete a prod (non-draft) app', async () => { + const prodCfg = { + id: 'prod-cfg', + name: 'Prod', + version: 1, + configuration: { parameters: { id: 'prod-app' } }, + metadata: [], + }; + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (path.endsWith('/configs/prod-cfg')) return HttpResponse.json(prodCfg); + return undefined; + }), + http.get('https://data-science.test/apps/prod-app', () => + HttpResponse.json({ + id: 'prod-app', + projectId: '42', + componentId: 'keboola.data-apps', + branchId: null, + configId: 'prod-cfg', + configVersion: '1', + type: 'python-js', + state: 'stopped', + desiredState: 'stopped', + }), + ), + http.get('https://data-science.test/apps/prod-app/git-repo', () => + HttpResponse.json({ httpsUrl: 'https://git.test/repo.git', isManagedGitRepo: true }), + ), + ); + const result = await call(await connect(), 'delete_python_js_data_app_draft', { + configuration_id: 'prod-cfg', + }); + expect(result.isError).toBeTruthy(); + expect(text(result)).toContain('prod** app'); + }); +}); + +describe('modify_python_js_data_app', () => { + it('rejects branch on the update path', async () => { + const result = await call(await connect(), 'modify_python_js_data_app', { + name: 'X', + description: 'd', + configuration_id: 'cfg1', + branch: 'feature', + }); + expect(result.isError).toBeTruthy(); + expect(text(result)).toContain('branch is only valid'); + }); + + it('requires slug on create', async () => { + const result = await call(await connect(), 'modify_python_js_data_app', { + name: 'X', + description: 'd', + }); + expect(result.isError).toBeTruthy(); + expect(text(result)).toContain('slug is required'); + }); + + it('creates a prod app with a managed repo (feature enabled, no workspace secret lookup)', async () => { + let createBody: Record = {}; + server.use( + verifyHandler(['data-apps-storage-workspace']), + http.post('https://encryption.test/encrypt', async ({ request }) => + HttpResponse.json((await request.json()) as Record), + ), + http.post('https://data-science.test/apps', async ({ request }) => { + createBody = (await request.json()) as Record; + return HttpResponse.json({ + id: 'new-app', + projectId: '42', + componentId: 'keboola.data-apps', + branchId: null, + configId: 'new-cfg', + configVersion: '1', + type: 'python-js', + state: 'created', + desiredState: 'stopped', + }); + }), + http.get('https://data-science.test/apps/new-app/git-repo', () => + HttpResponse.json({ httpsUrl: 'https://git.test/new.git', isManagedGitRepo: true }), + ), + http.post('https://connection.test/*', ({ request }) => { + if (new URL(request.url).pathname.endsWith('/metadata')) return HttpResponse.json([]); + return undefined; + }), + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) + return HttpResponse.json({ + owner: { id: '42', features: ['data-apps-storage-workspace'] }, + }); + if (path.endsWith('/keboola.data-apps/configs')) return HttpResponse.json([]); + return undefined; + }), + ); + + const result = await call(await connect(), 'modify_python_js_data_app', { + name: 'My App', + description: 'desc', + slug: 'my-app', + }); + expect(result.isError).toBeFalsy(); + const out = text(result); + expect(createBody.useManagedGitRepo).toBe(true); + expect(createBody.type).toBe('python-js'); + expect(out).toContain('created'); + expect(out).toContain('https://git.test/new.git'); + }); +}); + +describe('modify_streamlit_data_app', () => { + it('creates a streamlit app, injecting the query function and encrypting the config', async () => { + let createBody: Record = {}; + server.use( + http.get('https://connection.test/*', ({ request }) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/tokens/verify')) return HttpResponse.json({ owner: { id: '42' } }); + if (path.endsWith('/dev-branches')) + return HttpResponse.json([{ id: 'main-branch', isDefault: true }]); + if (path.endsWith('/workspaces')) + return HttpResponse.json([ + { id: 'ws-1', connection: { schema: 'WS_SCHEMA', backend: 'snowflake' } }, + ]); + if (path.endsWith('/keboola.data-apps/configs')) return HttpResponse.json([]); + return undefined; + }), + http.post('https://connection.test/*', ({ request }) => { + if (new URL(request.url).pathname.endsWith('/metadata')) return HttpResponse.json([]); + return undefined; + }), + http.post('https://encryption.test/encrypt', async ({ request }) => + HttpResponse.json((await request.json()) as Record), + ), + http.post('https://data-science.test/apps', async ({ request }) => { + createBody = (await request.json()) as Record; + return HttpResponse.json({ + id: 'st-app', + projectId: '42', + componentId: 'keboola.data-apps', + branchId: null, + configId: 'st-cfg', + configVersion: '1', + type: 'streamlit', + state: 'created', + desiredState: 'stopped', + url: 'https://st.run', + }); + }), + ); + + const result = await call(await connect(), 'modify_streamlit_data_app', { + name: 'Dashboard App', + description: 'desc', + source_code: 'import streamlit as st\n{QUERY_DATA_FUNCTION}\nst.write("hi")', + packages: ['plotly'], + authentication_type: 'default', + }); + expect(result.isError).toBeFalsy(); + const out = text(result); + expect(out).toContain('created'); + // injected query_data function replaced the placeholder. + const script = ( + (createBody.config as Record).parameters as Record + ).script as string[]; + expect(script[0]).toContain('def query_data'); + expect(script[0]).not.toContain('{QUERY_DATA_FUNCTION}'); + }); +}); diff --git a/__tests__/tools.doc.test.ts b/__tests__/tools.doc.test.ts new file mode 100644 index 000000000..063006ce4 --- /dev/null +++ b/__tests__/tools.doc.test.ts @@ -0,0 +1,74 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import type { DocsSearch } from '@/clients/docsSearch'; +import { setDocsSearchForTests } from '@/clients/docsSearch'; +import { Config } from '@/config'; +import { createServer } from '@/server'; + +const server = setupServer( + http.get('https://connection.test/*', ({ request }) => + new URL(request.url).pathname.endsWith('/tokens/verify') + ? HttpResponse.json({ owner: { id: '42' } }) + : undefined, + ), +); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +/** A minimal fake docs-search provider; only the methods under test are implemented. */ +const fakeDocsSearch = (overrides: Partial = {}): DocsSearch => ({ + search: async () => [], + answerQuestion: async () => ({ text: '', sourceUrls: [] }), + recommendComponents: async () => [], + isReady: async () => true, + close: async () => {}, + ...overrides, +}); + +const connect = async (): Promise => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 't', version: '0' }); + await client.connect(clientT); + return client; +}; + +afterEach(() => setDocsSearchForTests(undefined)); + +describe('docs_query', () => { + it('answers via the docs-search index and returns text + source urls', async () => { + let asked: string | undefined; + setDocsSearchForTests( + fakeDocsSearch({ + answerQuestion: async (question) => { + asked = question; + return { text: 'Use the API.', sourceUrls: ['https://help.keboola.com/x'] }; + }, + }), + ); + + const client = await connect(); + const result = await client.callTool({ name: 'docs_query', arguments: { query: 'how to?' } }); + const text = (result.content as { text: string }[])[0]!.text; + + expect(asked).toBe('how to?'); + expect(text).toContain('Use the API.'); + expect(text).toContain('https://help.keboola.com/x'); + await client.close(); + }); + + it('is filtered out of tools/list when no docs index is configured', async () => { + setDocsSearchForTests(null); + const client = await connect(); + const { tools } = await client.listTools(); + expect(tools.map((t) => t.name)).not.toContain('docs_query'); + await client.close(); + }); +}); diff --git a/__tests__/tools.flow.test.ts b/__tests__/tools.flow.test.ts new file mode 100644 index 000000000..38de40d2d --- /dev/null +++ b/__tests__/tools.flow.test.ts @@ -0,0 +1,521 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { registerFlowTools } from '@/tools/flow'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +// A standalone server registering only the flow tools (the real server.ts is not edited). +const makeServer = (cfg = config) => { + const mcp = new McpServer({ name: 'test', version: '0.0.0' }); + registerFlowTools(mcp, cfg); + return mcp; +}; + +const connect = async (cfg = config) => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await makeServer(cfg).connect(serverT); + const client = new Client({ name: 'test', version: '0.0.0' }); + await client.connect(clientT); + return client; +}; + +const call = async (name: string, args: Record, cfg = config) => { + const client = await connect(cfg); + const result = await client.callTool({ name, arguments: args }); + const text = (result.content as { text: string }[])[0]!.text; + await client.close(); + return { text, isError: result.isError }; +}; + +// Token verify backs links manager (project id) + project context (name/features). +const verifyOwner = (features: unknown = []) => + http.get('https://connection.test/*', ({ request }) => { + if (new URL(request.url).pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42', name: 'Proj', features } }); + } + return undefined; + }); + +// Scheduler list (used by get_flows). Returns [] unless overridden. +const schedulerEmpty = () => http.get('https://scheduler.test/*', () => HttpResponse.json([])); + +describe('get_flow_examples', () => { + it('renders legacy examples as markdown', async () => { + server.use(verifyOwner()); + const { text, isError } = await call('get_flow_examples', { + flow_type: 'keboola.orchestrator', + }); + expect(isError).toBeFalsy(); + expect(text).toContain('# Flow Configuration Examples for `keboola.orchestrator`'); + expect(text).toContain('1. Flow Configuration:'); + expect(text).toContain('```json'); + }); + + it('refuses conditional examples when the feature is disabled', async () => { + server.use(verifyOwner(['hide-conditional-flows'])); + const { text, isError } = await call('get_flow_examples', { flow_type: 'keboola.flow' }); + expect(isError).toBeTruthy(); + expect(text).toContain('Conditional flows are not supported'); + expect(text).toContain('legacy flow examples'); + }); + + it('renders conditional examples when enabled', async () => { + server.use(verifyOwner([])); + const { text, isError } = await call('get_flow_examples', { flow_type: 'keboola.flow' }); + expect(isError).toBeFalsy(); + expect(text).toContain('# Flow Configuration Examples for `keboola.flow`'); + }); +}); + +describe('get_flow_schema', () => { + it('returns the bundled legacy schema as markdown', async () => { + server.use(verifyOwner()); + const { text, isError } = await call('get_flow_schema', { + flow_type: 'keboola.orchestrator', + }); + expect(isError).toBeFalsy(); + expect(text).toContain('"phases"'); + expect(text).toContain('"tasks"'); + expect(text).toContain('componentId'); + }); + + it('fetches the live conditional schema from the AI catalog', async () => { + server.use( + verifyOwner([]), + http.get('https://ai.test/*', () => + HttpResponse.json({ + id: 'keboola.flow', + name: 'Flow', + type: 'application', + configurationSchema: { type: 'object', properties: { phases: { type: 'array' } } }, + }), + ), + http.get('https://connection.test/v2/storage/*', () => HttpResponse.json({ data: {} })), + ); + const { text, isError } = await call('get_flow_schema', { flow_type: 'keboola.flow' }); + expect(isError).toBeFalsy(); + expect(text).toContain('"phases"'); + }); + + it('refuses conditional schema when the feature is disabled', async () => { + server.use(verifyOwner(['hide-conditional-flows'])); + const { text, isError } = await call('get_flow_schema', { flow_type: 'keboola.flow' }); + expect(isError).toBeTruthy(); + expect(text).toContain('conditional_flows=false'); + }); +}); + +describe('create_flow (legacy)', () => { + it('creates a flow, normalizes ids, sets MCP metadata, returns links', async () => { + let createdBody: Record = {}; + server.use( + verifyOwner(), + // create config + http.post( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs', + async ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/configs')) { + createdBody = (await request.json()) as Record; + return HttpResponse.json({ + id: '123', + name: 'My Flow', + description: 'desc', + version: 1, + }); + } + return undefined; + }, + ), + // metadata POST (createdBy) + http.post( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/123/metadata', + () => HttpResponse.json([]), + ), + // folder search + config list (folder hint, returns few configs -> no hint) + http.get( + 'https://connection.test/v2/storage/branch/default/search/component-configurations', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs', + () => HttpResponse.json([{ id: '123' }]), + ), + ); + + const { text, isError } = await call('create_flow', { + name: 'My Flow', + description: 'desc', + phases: [{ name: 'Phase 1' }], + tasks: [{ name: 'T', phase: 1, task: { componentId: 'keboola.ex-aws-s3', configId: 'c1' } }], + }); + expect(isError).toBeFalsy(); + expect(text).toContain('123'); + expect(text).toContain('keboola.orchestrator'); + expect(text).toContain('/flows/123'); // flow detail link + // phase got id=1, task got id=20001 and mode=run + const cfg = createdBody.configuration as { + phases: { id: number }[]; + tasks: { id: number; task: { mode: string } }[]; + }; + expect(cfg.phases[0]!.id).toBe(1); + expect(cfg.tasks[0]!.id).toBe(20001); + expect(cfg.tasks[0]!.task.mode).toBe('run'); + }); + + it('rejects a task referencing a non-existent phase', async () => { + server.use(verifyOwner()); + const { text, isError } = await call('create_flow', { + name: 'Bad', + description: 'd', + phases: [{ id: 1, name: 'P1' }], + tasks: [{ id: 5, name: 'T', phase: 99, task: { componentId: 'x' } }], + }); + expect(isError).toBeTruthy(); + expect(text).toContain('non-existent phase'); + }); +}); + +describe('create_conditional_flow', () => { + const conditionalSchema = () => + http.get('https://ai.test/*', () => + HttpResponse.json({ + id: 'keboola.flow', + name: 'Flow', + type: 'application', + configurationSchema: { type: 'object' }, + }), + ); + + it('validates structure (entry/reachability) and creates the flow', async () => { + let body: Record = {}; + server.use( + verifyOwner([]), + conditionalSchema(), + http.get('https://connection.test/v2/storage/branch/default/components/keboola.flow', () => + HttpResponse.json({ data: {} }), + ), + http.post( + 'https://connection.test/v2/storage/branch/default/components/keboola.flow/configs', + async ({ request }) => { + body = (await request.json()) as Record; + return HttpResponse.json({ id: 'f1', name: 'CF', description: 'd', version: 1 }); + }, + ), + http.post( + 'https://connection.test/v2/storage/branch/default/components/keboola.flow/configs/f1/metadata', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/search/component-configurations', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.flow/configs', + () => HttpResponse.json([]), + ), + ); + + const { text, isError } = await call('create_conditional_flow', { + name: 'CF', + description: 'd', + phases: [ + { id: 'a', name: 'A', next: [{ id: 't1', goto: 'b' }] }, + { id: 'b', name: 'B', next: [] }, + ], + tasks: [ + { id: 't', name: 'Task', phase: 'a', task: { type: 'job', componentId: 'x', mode: 'run' } }, + ], + }); + expect(isError).toBeFalsy(); + expect(text).toContain('keboola.flow'); + expect(text).toContain('/flows-v2/f1'); + // ending phase 'b' had empty next -> dropped from serialized config + const cfg = body.configuration as { phases: Record[] }; + expect(cfg.phases[1]!.next).toBeUndefined(); + }); + + it('rejects multiple entry phases', async () => { + server.use(verifyOwner([]), conditionalSchema()); + const { text, isError } = await call('create_conditional_flow', { + name: 'CF', + description: 'd', + phases: [ + { id: 'a', name: 'A', next: [] }, + { id: 'b', name: 'B', next: [] }, + ], + tasks: [], + }); + expect(isError).toBeTruthy(); + expect(text).toContain('entry phase'); + }); + + it('fails when the conditional feature is disabled', async () => { + server.use(verifyOwner(['hide-conditional-flows'])); + const { text, isError } = await call('create_conditional_flow', { + name: 'CF', + description: 'd', + phases: [{ id: 'a', name: 'A', next: [] }], + tasks: [], + }); + expect(isError).toBeTruthy(); + expect(text).toContain('Conditional flows are not supported'); + }); +}); + +describe('get_flows', () => { + it('lists all flow summaries with dashboard links', async () => { + server.use( + verifyOwner(), + schedulerEmpty(), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.flow/configs', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs', + () => + HttpResponse.json([ + { + id: '7', + name: 'Flow7', + version: 2, + configuration: { phases: [{ id: 1 }], tasks: [{ id: 2 }, { id: 3 }] }, + }, + ]), + ), + ); + const { text, isError } = await call('get_flows', { flow_ids: [] }); + expect(isError).toBeFalsy(); + expect(text).toContain('Flow7'); + expect(text).toContain('phases_count'); + expect(text).toContain('/flows'); // dashboard link + }); + + it('returns full details for a specific flow id, resolving its type', async () => { + server.use( + verifyOwner(), + schedulerEmpty(), + // keboola.flow lookup 404s -> falls back to orchestrator + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.flow/configs/7', + () => new HttpResponse(null, { status: 404 }), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/7', + () => + HttpResponse.json({ + id: '7', + name: 'Flow7', + version: 2, + configuration: { phases: [{ id: 1, name: 'P' }], tasks: [] }, + }), + ), + ); + const { text, isError } = await call('get_flows', { flow_ids: ['7'] }); + expect(isError).toBeFalsy(); + expect(text).toContain('Flow7'); + expect(text).toContain('keboola.orchestrator'); + expect(text).toContain('phases'); + }); +}); + +describe('modify_flow', () => { + it('updates phases/tasks and sets update metadata', async () => { + let putBody: Record = {}; + server.use( + verifyOwner([]), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/9', + () => + HttpResponse.json({ + id: '9', + name: 'F9', + version: 3, + configuration: { phases: [{ id: 1, name: 'Old' }], tasks: [] }, + }), + ), + http.put( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/9', + async ({ request }) => { + putBody = (await request.json()) as Record; + return HttpResponse.json({ id: '9', name: 'F9', description: 'd', version: 4 }); + }, + ), + http.post( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/9/metadata', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/search/component-configurations', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs', + () => HttpResponse.json([{ id: '9' }]), + ), + ); + const { text, isError } = await call('modify_flow', { + configuration_id: '9', + flow_type: 'keboola.orchestrator', + change_description: 'update phases', + phases: [{ id: 1, name: 'New Phase' }], + tasks: [{ id: 2, name: 'T', phase: 1, task: { componentId: 'x' } }], + }); + expect(isError).toBeFalsy(); + expect(text).toContain('version: 4'); + // The typed Storage client (updateConfiguration) serializes `configuration` as a JSON + // string field (the canonical SAPI contract), unlike the previous raw nested-object body. + const cfg = JSON.parse(putBody.configuration as string) as { phases: { name: string }[] }; + expect(cfg.phases[0]!.name).toBe('New Phase'); + expect(putBody.changeDescription).toBe('update phases'); + }); + + it('processes an add-schedule request and appends a scheduler link', async () => { + server.use( + verifyOwner([]), + // no config changes -> just detail fetch + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/9', + () => + HttpResponse.json({ + id: '9', + name: 'F9', + description: 'd', + version: 3, + configuration: {}, + }), + ), + // folder hint + http.get( + 'https://connection.test/v2/storage/branch/default/search/component-configurations', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs', + () => HttpResponse.json([{ id: '9' }]), + ), + // scheduler list (current schedules) -> empty + http.get('https://scheduler.test/schedules', () => HttpResponse.json([])), + // create scheduler config in storage + http.post( + 'https://connection.test/v2/storage/branch/default/components/keboola.scheduler/configs', + () => HttpResponse.json({ id: 'sched1', version: 1 }), + ), + http.post( + 'https://connection.test/v2/storage/branch/default/components/keboola.scheduler/configs/sched1/metadata', + () => HttpResponse.json([]), + ), + // activate schedule + http.post('https://scheduler.test/schedules', () => + HttpResponse.json({ + id: 'act1', + configurationId: 'sched1', + schedule: { cronTab: '0 8 * * 1', timezone: 'UTC', state: 'enabled' }, + }), + ), + ); + const { text, isError } = await call('modify_flow', { + configuration_id: '9', + flow_type: 'keboola.orchestrator', + change_description: 'add schedule', + schedules: [{ action: 'add', cron_tab: '0 8 * * 1', state: 'enabled', timezone: 'UTC' }], + }); + expect(isError).toBeFalsy(); + expect(text).toContain('Schedules request processed successfully'); + expect(text).toContain('Created schedule: sched1'); + expect(text).toContain('/schedules'); // scheduler detail link + }); + + it('rejects an invalid cron expression', async () => { + server.use( + verifyOwner([]), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/9', + () => + HttpResponse.json({ + id: '9', + name: 'F9', + description: 'd', + version: 3, + configuration: {}, + }), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/search/component-configurations', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs', + () => HttpResponse.json([{ id: '9' }]), + ), + http.get('https://scheduler.test/schedules', () => HttpResponse.json([])), + ); + const { text, isError } = await call('modify_flow', { + configuration_id: '9', + flow_type: 'keboola.orchestrator', + change_description: 'bad cron', + schedules: [{ action: 'add', cron_tab: '99 99 * *', state: 'enabled' }], + }); + expect(isError).toBeTruthy(); + expect(text).toContain('Invalid cron tab expression'); + }); +}); + +describe('update_flow', () => { + it('delegates to modify_flow without schedules', async () => { + let putBody: Record = {}; + server.use( + verifyOwner([]), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/5', + () => + HttpResponse.json({ + id: '5', + name: 'F5', + version: 1, + configuration: { phases: [{ id: 1, name: 'P' }], tasks: [] }, + }), + ), + http.put( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/5', + async ({ request }) => { + putBody = (await request.json()) as Record; + return HttpResponse.json({ id: '5', name: 'Renamed', description: 'd', version: 2 }); + }, + ), + http.post( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs/5/metadata', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/search/component-configurations', + () => HttpResponse.json([]), + ), + http.get( + 'https://connection.test/v2/storage/branch/default/components/keboola.orchestrator/configs', + () => HttpResponse.json([{ id: '5' }]), + ), + ); + const { text, isError } = await call('update_flow', { + configuration_id: '5', + flow_type: 'keboola.orchestrator', + change_description: 'rename', + name: 'Renamed', + }); + expect(isError).toBeFalsy(); + expect(putBody.name).toBe('Renamed'); + expect(text).toContain('Renamed'); + }); +}); diff --git a/__tests__/tools.jobs.test.ts b/__tests__/tools.jobs.test.ts new file mode 100644 index 000000000..67e2c154a --- /dev/null +++ b/__tests__/tools.jobs.test.ts @@ -0,0 +1,183 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { createServer } from '@/server'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +const connect = async () => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 'test', version: '0.0.0' }); + await client.connect(clientT); + return client; +}; + +// Token verify backs the links manager (project id). Always available. +const verifyHandler = () => + http.get('https://connection.test/*', ({ request }) => { + if (new URL(request.url).pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42' } }); + } + return undefined; + }); + +const callText = async ( + client: Awaited>, + args: Record, +) => { + const result = await client.callTool({ name: 'get_jobs', arguments: args }); + expect(result.isError).toBeFalsy(); + return (result.content as { text: string }[])[0]!.text; +}; + +describe('get_jobs', () => { + it('lists job summaries and maps component/config aliases', async () => { + server.use( + verifyHandler(), + http.get('https://queue.test/*', () => + HttpResponse.json([ + { + id: '1', + status: 'success', + component: 'keboola.ex-aws-s3', + config: 'c1', + isFinished: true, + }, + ]), + ), + ); + + const text = await callText(await connect(), { job_ids: [] }); + expect(text).toContain('keboola.ex-aws-s3'); + expect(text).toContain('componentId'); + // Listing surfaces the jobs dashboard link. + expect(text).toContain('queue'); + }); + + it('returns full details for specific job ids with a job link', async () => { + server.use( + verifyHandler(), + http.get('https://queue.test/*', () => + HttpResponse.json({ + id: '99', + status: 'error', + component: 'x', + config: 'c', + url: 'https://job/99', + }), + ), + ); + + const text = await callText(await connect(), { job_ids: ['99'] }); + expect(text).toContain('https://job/99'); + expect(text).toContain('/queue/99'); // job detail link + }); + + it('includes filtered, chronologically-ordered logs when requested', async () => { + server.use( + verifyHandler(), + http.get('https://queue.test/*', () => + HttpResponse.json({ id: '7', status: 'error', component: 'x', config: 'c', url: 'u' }), + ), + ); + // Events come newest-first; the tool filters by type then reverses to chronological. + server.use( + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) + return HttpResponse.json({ owner: { id: '42' } }); + if (url.pathname.endsWith('/events')) { + return HttpResponse.json([ + { message: 'boom', type: 'error', created: 't2' }, + { message: 'starting', type: 'info', created: 't1' }, + ]); + } + return undefined; + }), + ); + + const text = await callText(await connect(), { + job_ids: ['7'], + include_logs: true, + log_event_types: ['error'], + }); + expect(text).toContain('boom'); + expect(text).not.toContain('starting'); + }); +}); + +describe('run_job', () => { + const runJob = async (args: Record, capture: (body: unknown) => void) => { + server.use( + verifyHandler(), + http.post('https://queue.test/*', async ({ request }) => { + capture(await request.json()); + return HttpResponse.json({ + id: '500', + status: 'created', + component: args.component_id, + config: args.configuration_id, + url: 'https://job/500', + }); + }), + ); + const client = await connect(); + const result = await client.callTool({ name: 'run_job', arguments: args }); + expect(result.isError).toBeFalsy(); + const text = (result.content as { text: string }[])[0]!.text; + await client.close(); + return text; + }; + + it('creates a job with mode=run and returns its details + link', async () => { + let body: Record = {}; + const text = await runJob( + { component_id: 'keboola.ex-aws-s3', configuration_id: 'c1' }, + (b) => { + body = b as Record; + }, + ); + expect(body).toMatchObject({ component: 'keboola.ex-aws-s3', config: 'c1', mode: 'run' }); + expect(body.branchId).toBeUndefined(); // production: no branchId in payload + expect(text).toContain('https://job/500'); + expect(text).toContain('/queue/500'); + }); + + it('passes config row ids and the branch id on a development branch', async () => { + let body: Record = {}; + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config.replaceBy({ branchId: '789' })).connect(serverT); + const client = new Client({ name: 't', version: '0' }); + await client.connect(clientT); + server.use( + verifyHandler(), + http.post('https://queue.test/*', async ({ request }) => { + body = (await request.json()) as Record; + return HttpResponse.json({ + id: '501', + status: 'created', + component: 'x', + config: 'c', + url: 'u', + }); + }), + ); + await client.callTool({ + name: 'run_job', + arguments: { component_id: 'x', configuration_id: 'c', configuration_row_ids: ['r1', 'r2'] }, + }); + expect(body.branchId).toBe('789'); + expect(body.configRowIds).toEqual(['r1', 'r2']); + await client.close(); + }); +}); diff --git a/__tests__/tools.oauth.test.ts b/__tests__/tools.oauth.test.ts new file mode 100644 index 000000000..06bfcdc47 --- /dev/null +++ b/__tests__/tools.oauth.test.ts @@ -0,0 +1,45 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { createServer } from '@/server'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +describe('create_oauth_url', () => { + it('mints a scoped short-lived token and builds the external OAuth URL', async () => { + let body: Record = {}; + server.use( + http.post('https://connection.test/*', async ({ request }) => { + expect(new URL(request.url).pathname).toMatch(/\/v2\/storage\/tokens$/); + body = (await request.json()) as Record; + return HttpResponse.json({ token: 'short-lived-123' }); + }), + ); + + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 't', version: '0' }); + await client.connect(clientT); + + const result = await client.callTool({ + name: 'create_oauth_url', + arguments: { component_id: 'keboola.ex-gmail', config_id: 'cfg1' }, + }); + const url = (result.content as { text: string }[])[0]!.text; + + expect(body).toMatchObject({ componentAccess: ['keboola.ex-gmail'], expiresIn: 3600 }); + expect(url).toBe( + 'https://external.keboola.com/oauth/index.html?token=short-lived-123&sapiUrl=https%3A%2F%2Fconnection.test#/keboola.ex-gmail/cfg1', + ); + await client.close(); + }); +}); diff --git a/__tests__/tools.project.test.ts b/__tests__/tools.project.test.ts new file mode 100644 index 000000000..39d7eb177 --- /dev/null +++ b/__tests__/tools.project.test.ts @@ -0,0 +1,222 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { createServer } from '@/server'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const connect = async (config: Config) => { + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverTransport); + const client = new Client({ name: 'test-client', version: '0.0.0' }); + await client.connect(clientTransport); + return client; +}; + +const config = new Config({ + storageApiUrl: 'https://connection.test', + storageToken: 'test-token', +}); + +describe('update_project_description', () => { + it('posts the description to the current branch metadata endpoint', async () => { + let captured: { pathname: string; body: unknown } | undefined; + server.use( + http.post('https://connection.test/*', async ({ request }) => { + captured = { pathname: new URL(request.url).pathname, body: await request.json() }; + return HttpResponse.json([{ key: 'KBC.projectDescription', value: 'New desc' }]); + }), + ); + + const client = await connect(config); + const result = await client.callTool({ + name: 'update_project_description', + arguments: { description: 'New desc' }, + }); + + // Branch resolution: production maps to the `default` branch alias. + expect(captured?.pathname).toMatch(/\/branch\/default\/metadata$/); + expect(JSON.stringify(captured?.body)).toContain('New desc'); + + const content = result.content as { type: string; text: string }[]; + expect(content[0]!.text).toContain('updated successfully'); + expect(result.isError).toBeFalsy(); + await client.close(); + }); + + it('targets a development branch when one is configured', async () => { + let pathname: string | undefined; + server.use( + http.post('https://connection.test/*', ({ request }) => { + pathname = new URL(request.url).pathname; + return HttpResponse.json([]); + }), + ); + + const client = await connect(config.replaceBy({ branchId: '567' })); + await client.callTool({ name: 'update_project_description', arguments: { description: 'x' } }); + expect(pathname).toMatch(/\/branch\/567\/metadata$/); + await client.close(); + }); +}); + +// --------------------------------------------------------------------------- +// get_project_info +// --------------------------------------------------------------------------- + +type ProjectInfoHandlers = { + verifyToken?: Record; + devBranches?: unknown[]; + branchMetadata?: { key: string; value: string }[]; + workspaces?: unknown[]; +}; + +/** Registers GET handlers for the four storage endpoints get_project_info reads. */ +const useProjectInfoHandlers = (opts: ProjectInfoHandlers = {}) => { + const { + verifyToken = { + owner: { id: '42', name: 'My Project', features: [] }, + organization: { id: '7' }, + admin: { role: 'admin' }, + }, + devBranches = [{ id: 123, name: 'Main', isDefault: true }], + branchMetadata = [ + { key: 'KBC.projectDescription', value: 'Some description' }, + { key: 'KBC.McpServer.v2.workspaceId', value: '999' }, + ], + workspaces = [ + { + id: 999, + connection: { backend: 'snowflake', schema: 'WORKSPACE_SCHEMA' }, + readOnlyStorageAccess: true, + }, + ], + } = opts; + + server.use( + http.get('https://connection.test/v2/storage/tokens/verify', () => + HttpResponse.json(verifyToken), + ), + http.get('https://connection.test/v2/storage/dev-branches', () => + HttpResponse.json(devBranches), + ), + http.get('https://connection.test/v2/storage/branch/:branchId/metadata', () => + HttpResponse.json(branchMetadata), + ), + // WorkspaceManager fetches a workspace by id (resolved from the MCP metadata key)… + http.get( + 'https://connection.test/v2/storage/branch/:branchId/workspaces/:wsId', + ({ params }) => { + const ws = (workspaces as { id: number }[]).find( + (w) => String(w.id) === String(params.wsId), + ); + return ws ? HttpResponse.json(ws) : new HttpResponse(null, { status: 404 }); + }, + ), + // …or lists them when resolving by the configured workspace schema. + http.get('https://connection.test/v2/storage/branch/:branchId/workspaces', () => + HttpResponse.json(workspaces), + ), + ); +}; + +const callProjectInfo = async (cfg: Config) => { + const client = await connect(cfg); + const result = await client.callTool({ name: 'get_project_info', arguments: {} }); + await client.close(); + const content = (result.content as { type: string; text: string }[])[0]!.text; + return { result, text: content }; +}; + +describe('get_project_info', () => { + it('returns unified project info on the default branch', async () => { + useProjectInfoHandlers(); + const { result, text } = await callProjectInfo(config); + + expect(result.isError).toBeFalsy(); + expect(text).toContain('project_id: "42"'); + expect(text).toContain('My Project'); + expect(text).toContain('Some description'); + expect(text).toContain('organization_id: "7"'); + expect(text).toContain('sql_dialect: Snowflake'); + expect(text).toContain('workspace_id: 999'); + expect(text).toContain('user_role: admin'); + // Default branch resolution. + expect(text).toContain('branch_id: 123'); + expect(text).toContain('branch_name: Main'); + expect(text).toContain('is_development_branch: false'); + // Conditional flows enabled when the feature flag is absent. + expect(text).toContain('conditional_flows: true'); + // The base system prompt (and the Snowflake dialect section) is embedded. + expect(text).toContain('### SQL Identifiers'); + expect(text).toContain('Snowflake'); + expect(text).toContain('Finding Items'); + // admin role => no toolset restrictions (null dropped from compact TOON output). + expect(text).not.toContain('toolset_restrictions'); + }); + + it('resolves the configured development branch and surfaces role restrictions', async () => { + useProjectInfoHandlers({ + verifyToken: { + owner: { id: '42', name: 'My Project', features: ['hide-conditional-flows'] }, + organization: { id: '7' }, + admin: { role: 'readonly' }, + }, + devBranches: [ + { id: 123, name: 'Main', isDefault: true }, + { id: 456, name: 'feature-x', isDefault: false }, + ], + }); + + const { text } = await callProjectInfo(config.replaceBy({ branchId: '456' })); + + expect(text).toContain('branch_id: 456'); + expect(text).toContain('branch_name: feature-x'); + expect(text).toContain('is_development_branch: true'); + // hide-conditional-flows feature present => conditional flows disabled. + expect(text).toContain('conditional_flows: false'); + expect(text).toContain('user_role: readonly'); + expect(text).toContain('read-only tools are available'); + }); + + it('selects a BigQuery workspace by the configured workspace schema', async () => { + useProjectInfoHandlers({ + workspaces: [ + { + id: 111, + connection: { backend: 'snowflake', schema: 'OTHER' }, + readOnlyStorageAccess: true, + }, + { + id: 222, + connection: { + backend: 'bigquery', + schema: 'TARGET_SCHEMA', + // BigQuery workspaces carry the service-account JSON in `connection.user`; + // WorkspaceManager parses project_id out of it. + user: JSON.stringify({ project_id: 'my-bq-project' }), + }, + readOnlyStorageAccess: true, + }, + ], + }); + + const { text } = await callProjectInfo(config.replaceBy({ workspaceSchema: 'TARGET_SCHEMA' })); + + expect(text).toContain('sql_dialect: BigQuery'); + expect(text).toContain('workspace_id: 222'); + // BigQuery dialect section uses backtick FQNs. + expect(text).toContain('`project`.`dataset`.`table`'); + }); + + // Note: when no workspace exists, get_project_info now CREATES one via WorkspaceManager + // (parity with the Python behavior / query_data), rather than erroring. The create-and-poll + // path is covered in __tests__/workspace.test.ts. +}); diff --git a/__tests__/tools.search.test.ts b/__tests__/tools.search.test.ts new file mode 100644 index 000000000..a303b9905 --- /dev/null +++ b/__tests__/tools.search.test.ts @@ -0,0 +1,272 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import type { DocsSearch, RetrievedDoc } from '@/clients/docsSearch'; +import { setDocsSearchForTests } from '@/clients/docsSearch'; +import { Config } from '@/config'; +import { createServer } from '@/server'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => { + server.resetHandlers(); + setDocsSearchForTests(undefined); +}); +afterAll(() => server.close()); + +/** A minimal fake docs-search provider; only the methods under test are implemented. */ +const fakeDocsSearch = (overrides: Partial = {}): DocsSearch => ({ + search: async () => [], + answerQuestion: async () => ({ text: '', sourceUrls: [] }), + recommendComponents: async () => [], + isReady: async () => true, + close: async () => {}, + ...overrides, +}); + +const retrieved = (sourceKey: string, score: number): RetrievedDoc => ({ + id: 'doc-1', + sourceKey, + sourceUrl: 'https://components.keboola.com/x', + title: 'x', + content: 'x', + componentType: 'extractor', + score, +}); + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +const callSearch = async (args: Record): Promise => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 't', version: '0' }); + await client.connect(clientT); + const result = await client.callTool({ name: 'search', arguments: args }); + const text = (result.content as { text: string }[])[0]!.text; + await client.close(); + return text; +}; + +describe('search (global textual)', () => { + it('runs server-side global search when the feature is enabled and maps hits', async () => { + let searchQuery: URL | undefined; + server.use( + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42', features: ['global-search'] } }); + } + if (url.pathname.endsWith('/global-search')) { + searchQuery = url; + return HttpResponse.json({ + all: 1, + byType: { table: 1 }, + items: [ + { + id: 'in.c-main.customers', + name: 'customers', + type: 'table', + fullPath: { bucket: { id: 'in.c-main' } }, + created: '2024-01-01T00:00:00Z', + }, + ], + }); + } + return undefined; + }), + ); + + const text = await callSearch({ patterns: ['customer'], item_types: ['table'] }); + + // Production branch context -> branchTypes[]=production, scoped to the project. + expect(searchQuery?.searchParams.get('branchTypes[]')).toBe('production'); + expect(searchQuery?.searchParams.get('projectIds[]')).toBe('42'); + expect(searchQuery?.searchParams.get('query')).toBe('customer'); + expect(text).toContain('in.c-main.customers'); + expect(text).toContain('current-branch'); + // Link to the table detail page. + expect(text).toContain('storage/in.c-main/table/customers'); + }); + + it('widens to all branches when the current branch context returns nothing', async () => { + const scopes: string[] = []; + server.use( + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42', features: ['global-search'] } }); + } + if (url.pathname.endsWith('/global-search')) { + const hasBranch = url.searchParams.has('branchTypes[]'); + scopes.push(hasBranch ? 'current' : 'all'); + if (hasBranch) { + return HttpResponse.json({ all: 0, byType: {}, items: [] }); + } + return HttpResponse.json({ + all: 1, + byType: { bucket: 1 }, + items: [ + { + id: 'in.c-dev', + name: 'dev-bucket', + type: 'bucket', + fullPath: { branch: { id: '789', name: 'feature-x' } }, + created: '2024-02-02T00:00:00Z', + }, + ], + }); + } + return undefined; + }), + ); + + const text = await callSearch({ patterns: ['dev'], item_types: ['bucket'] }); + + expect(scopes).toEqual(['current', 'all']); + expect(text).toContain('all-branches'); + expect(text).toContain('feature-x'); + expect(text).toContain('in.c-dev'); + }); +}); + +describe('search (enumeration fallback)', () => { + it('config-based search matches inside configuration JSON and reports scopes', async () => { + server.use( + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42', features: [] } }); + } + if (url.pathname.endsWith('/components')) { + return HttpResponse.json([ + { + id: 'keboola.snowflake-transformation', + type: 'transformation', + configurations: [ + { + id: '123', + name: 'My SQL', + description: 'desc', + created: '2024-03-03T00:00:00Z', + configuration: { + storage: { input: { tables: [{ source: 'in.c-prod.customers' }] } }, + }, + rows: [], + }, + ], + }, + ]); + } + return undefined; + }), + ); + + const text = await callSearch({ + patterns: ['in.c-prod.customers'], + item_types: ['transformation'], + search_type: 'config-based', + }); + + expect(text).toContain('123'); + expect(text).toContain('keboola.snowflake-transformation'); + // The matched JSONPath scope is reported. + expect(text).toContain('storage.input.tables[0].source'); + }); + + it('textual search without the global-search feature enumerates configurations', async () => { + server.use( + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42', features: [] } }); + } + if (url.pathname.endsWith('/components')) { + return HttpResponse.json([ + { + id: 'keboola.ex-db', + type: 'extractor', + configurations: [ + { + id: '7', + name: 'Sales report', + description: null, + created: '2024-04-04T00:00:00Z', + rows: [], + }, + ], + }, + ]); + } + return undefined; + }), + ); + + const text = await callSearch({ patterns: ['sales'], item_types: ['configuration'] }); + expect(text).toContain('Sales report'); + expect(text).toContain('keboola.ex-db'); + }); +}); + +describe('find_component_id', () => { + it('recommends component ids from the docs index with scores and a dashboard link', async () => { + let asked: string | undefined; + server.use( + http.get('https://connection.test/*', ({ request }) => + new URL(request.url).pathname.endsWith('/tokens/verify') + ? HttpResponse.json({ owner: { id: '42' } }) + : undefined, + ), + ); + setDocsSearchForTests( + fakeDocsSearch({ + recommendComponents: async (query) => { + asked = query; + return [ + retrieved('component:keboola.ex-salesforce', 0.9), + // A non-component doc (no `component:` prefix) is dropped from the result. + retrieved('help:some-doc', 0.8), + ]; + }, + }), + ); + + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 't', version: '0' }); + await client.connect(clientT); + const result = await client.callTool({ + name: 'find_component_id', + arguments: { query: 'salesforce extractor' }, + }); + const text = (result.content as { text: string }[])[0]!.text; + + expect(asked).toBe('salesforce extractor'); + expect(text).toContain('keboola.ex-salesforce'); + expect(text).toContain('components/keboola.ex-salesforce'); // dashboard link + expect(text).not.toContain('help:some-doc'); + await client.close(); + }); + + it('is denied on call when no docs index is configured', async () => { + server.use( + http.get('https://connection.test/*', ({ request }) => + new URL(request.url).pathname.endsWith('/tokens/verify') + ? HttpResponse.json({ owner: { id: '42' } }) + : undefined, + ), + ); + setDocsSearchForTests(null); + + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 't', version: '0' }); + await client.connect(clientT); + await expect( + client.callTool({ name: 'find_component_id', arguments: { query: 'salesforce' } }), + ).rejects.toThrow(/documentation index is not/); + await client.close(); + }); +}); diff --git a/__tests__/tools.semantic.test.ts b/__tests__/tools.semantic.test.ts new file mode 100644 index 000000000..9e1b5f93a --- /dev/null +++ b/__tests__/tools.semantic.test.ts @@ -0,0 +1,327 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { registerSemanticTools } from '@/tools/semantic'; + +const mswServer = setupServer(); +beforeAll(() => mswServer.listen({ onUnhandledRequest: 'error' })); +afterEach(() => mswServer.resetHandlers()); +afterAll(() => mswServer.close()); + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +const connect = async () => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + const server = new McpServer({ name: 'test', version: '0.0.0' }); + registerSemanticTools(server, config); + await server.connect(serverT); + const client = new Client({ name: 'test', version: '0.0.0' }); + await client.connect(clientT); + return client; +}; + +const callText = async ( + client: Awaited>, + name: string, + args: Record, +) => { + const result = await client.callTool({ name, arguments: args }); + return result; +}; + +const text = (result: Awaited>): string => + (result.content as { text: string }[])[0]!.text; + +// JSON:API list envelope; each item has top-level type/id/attributes/meta. +const listResponse = ( + items: { + type: string; + id: string; + attributes?: Record; + meta?: Record; + }[], +) => HttpResponse.json({ data: items }); + +const objectResponse = (item: { + type: string; + id: string; + attributes?: Record; + meta?: Record; +}) => HttpResponse.json({ data: item }); + +// Helper: route metastore repository/schema requests on metastore.test. +// Paged list endpoints page until a short page is returned; to make finite, +// only serve list items on offset=0 and return an empty page otherwise. +const metastore = (handlers: (url: URL, objectType: string) => Response | undefined) => + http.get('https://metastore.test/*', ({ request }) => { + const url = new URL(request.url); + const m = url.pathname.match(/\/api\/v1\/(?:repository|schema)\/([^/]+)/); + const objectType = m ? m[1]! : ''; + const isList = + url.pathname.includes('/repository/') && !/\/repository\/[^/]+\/.+/.test(url.pathname); + const offset = Number(url.searchParams.get('offset') ?? '0'); + if (isList && offset > 0) { + return HttpResponse.json({ data: [] }); + } + return handlers(url, objectType) ?? new HttpResponse(null, { status: 500 }); + }); + +describe('get_semantic_schema', () => { + it('returns JSON schema per requested semantic type', async () => { + mswServer.use( + metastore((url, objectType) => { + if (url.pathname.includes('/schema/')) { + return HttpResponse.json({ objectType, schema: { type: 'object', title: objectType } }); + } + return undefined; + }), + ); + const result = await callText(await connect(), 'get_semantic_schema', { + semantic_types: ['semantic-dataset'], + }); + expect(result.isError).toBeFalsy(); + expect(text(result)).toContain('semantic-dataset'); + }); + + it('errors on empty semantic_types', async () => { + const result = await callText(await connect(), 'get_semantic_schema', { semantic_types: [] }); + expect(result.isError).toBe(true); + expect(text(result)).toContain('At least one semantic type'); + }); +}); + +describe('get_semantic_context', () => { + it('lists compact objects when ids are empty', async () => { + mswServer.use( + metastore((url, objectType) => { + if (objectType === 'semantic-model' && url.pathname.endsWith('/semantic-model')) { + // First page full (limit=20 -> return < limit to stop). + return listResponse([ + { + type: 'semantic-model', + id: 'm1', + attributes: { name: 'Sales Model', sql_dialect: 'snowflake' }, + }, + ]); + } + return undefined; + }), + ); + const result = await callText(await connect(), 'get_semantic_context', { + semantic_objects: [{ object_type: 'semantic-model' }], + }); + expect(result.isError).toBeFalsy(); + const out = text(result); + expect(out).toContain('Sales Model'); + expect(out).toContain('snowflake'); + }); + + it('returns full attributes when ids are provided', async () => { + mswServer.use( + metastore((url, objectType) => { + if (objectType === 'semantic-dataset' && url.pathname.endsWith('/d1')) { + return objectResponse({ + type: 'semantic-dataset', + id: 'd1', + attributes: { name: 'Orders', tableId: 'in.c-x.orders', secretKey: 'keepme' }, + }); + } + return undefined; + }), + ); + const result = await callText(await connect(), 'get_semantic_context', { + semantic_objects: [{ object_type: 'semantic-dataset', ids: ['d1'] }], + }); + expect(result.isError).toBeFalsy(); + const out = text(result); + // Full attributes view includes the raw attributes map. + expect(out).toContain('keepme'); + expect(out).toContain('attributes'); + }); + + it('errors on empty semantic_objects', async () => { + const result = await callText(await connect(), 'get_semantic_context', { + semantic_objects: [], + }); + expect(result.isError).toBe(true); + }); +}); + +describe('search_semantic_context', () => { + it('matches by attribute value and groups by model, with matched paths', async () => { + mswServer.use( + metastore((_url, objectType) => { + if (objectType === 'semantic-dataset') { + return listResponse([ + { + type: 'semantic-dataset', + id: 'd1', + attributes: { name: 'Revenue Facts', tableId: 'in.c-x.rev', modelUUID: 'm1' }, + }, + ]); + } + // All other types empty. + return listResponse([]); + }), + ); + const result = await callText(await connect(), 'search_semantic_context', { + patterns: ['revenue'], + semantic_types: ['semantic-dataset'], + }); + expect(result.isError).toBeFalsy(); + const out = text(result); + expect(out).toContain('m1'); // grouped by semantic_model_id + expect(out).toContain('Revenue Facts'); + expect(out).toContain('meta.name'); // matched on display name + }); + + it('errors when no usable patterns are provided', async () => { + const result = await callText(await connect(), 'search_semantic_context', { + patterns: [' '], + }); + expect(result.isError).toBe(true); + expect(text(result)).toContain('At least one regex pattern'); + }); + + it('errors on invalid regex', async () => { + mswServer.use(metastore(() => listResponse([]))); + const result = await callText(await connect(), 'search_semantic_context', { patterns: ['('] }); + expect(result.isError).toBe(true); + expect(text(result)).toContain('Invalid regex pattern'); + }); +}); + +describe('validate_semantic_query', () => { + // A model with a dataset (in.table), a metric SUM("AMOUNT") on that dataset, and a + // post-query 'range' constraint scoped to the metric. + const buildModelHandlers = () => + metastore((url, objectType) => { + if (url.pathname.includes('/schema/')) return undefined; + // get model by id + if (objectType === 'semantic-model' && url.pathname.endsWith('/m1')) { + return objectResponse({ + type: 'semantic-model', + id: 'm1', + attributes: { name: 'M1', sql_dialect: 'snowflake' }, + }); + } + if (objectType === 'semantic-model') { + return listResponse([ + { + type: 'semantic-model', + id: 'm1', + attributes: { name: 'M1', sql_dialect: 'snowflake' }, + }, + ]); + } + if (objectType === 'semantic-dataset' && url.pathname.endsWith('/d1')) { + return objectResponse({ + type: 'semantic-dataset', + id: 'd1', + attributes: { + name: 'Orders', + tableId: 'orders_tbl', + fqn: 'DB.SCHEMA.ORDERS', + modelUUID: 'm1', + }, + }); + } + if (objectType === 'semantic-dataset') { + return listResponse([ + { + type: 'semantic-dataset', + id: 'd1', + attributes: { + name: 'Orders', + tableId: 'orders_tbl', + fqn: 'DB.SCHEMA.ORDERS', + modelUUID: 'm1', + }, + }, + ]); + } + if (objectType === 'semantic-metric') { + return listResponse([ + { + type: 'semantic-metric', + id: 'mt1', + attributes: { + name: 'Total Amount', + sql: 'SUM("AMOUNT")', + dataset: 'orders_tbl', + modelUUID: 'm1', + }, + }, + ]); + } + if (objectType === 'semantic-relationship') return listResponse([]); + if (objectType === 'semantic-constraint') { + return listResponse([ + { + type: 'semantic-constraint', + id: 'c1', + attributes: { + name: 'Amount range check', + constraintType: 'range', + severity: 'warning', + metrics: ['Total Amount'], + modelUUID: 'm1', + }, + }, + ]); + } + return undefined; + }); + + it('auto-detects dataset + metric and surfaces a post-execution check', async () => { + mswServer.use(buildModelHandlers()); + const result = await callText(await connect(), 'validate_semantic_query', { + sql_query: 'SELECT SUM("AMOUNT") FROM DB.SCHEMA.ORDERS', + semantic_model_ids: ['m1'], + }); + expect(result.isError).toBeFalsy(); + const out = text(result); + expect(out).toContain('validation_auto_detected'); + expect(out).toContain('orders_tbl'); // used dataset tableId + expect(out).toContain('Total Amount'); // used metric + expect(out).toContain('post_execution_checks'); + expect(out).toContain('Amount range check'); + expect(out).toContain('valid'); // valid true (warning severity) + }); + + it('compares expected objects and reports missing ones', async () => { + mswServer.use(buildModelHandlers()); + const result = await callText(await connect(), 'validate_semantic_query', { + sql_query: 'SELECT 1', + semantic_model_ids: ['m1'], + expected_semantic_objects: [{ object_type: 'semantic-dataset', ids: ['d1'] }], + }); + expect(result.isError).toBeFalsy(); + const out = text(result); + // d1 was expected but not detected in `SELECT 1`. + expect(out).toContain('missing_expected_objects'); + expect(out).toContain('validation_detected_from_expected'); + }); + + it('errors on empty sql', async () => { + const result = await callText(await connect(), 'validate_semantic_query', { + sql_query: ' ', + semantic_model_ids: ['m1'], + }); + expect(result.isError).toBe(true); + expect(text(result)).toContain('sql_query must not be empty'); + }); + + it('errors when no model ids', async () => { + const result = await callText(await connect(), 'validate_semantic_query', { + sql_query: 'SELECT 1', + semantic_model_ids: [], + }); + expect(result.isError).toBe(true); + }); +}); diff --git a/__tests__/tools.sql.test.ts b/__tests__/tools.sql.test.ts new file mode 100644 index 000000000..b1a98168d --- /dev/null +++ b/__tests__/tools.sql.test.ts @@ -0,0 +1,238 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { registerSqlTools } from '@/tools/sql'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +const connect = async (cfg: Config = config) => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + const mcp = new McpServer({ name: 'test', version: '0.0.0' }); + registerSqlTools(mcp, cfg); + await mcp.connect(serverT); + const client = new Client({ name: 'test', version: '0.0.0' }); + await client.connect(clientT); + return client; +}; + +const callTool = async ( + client: Awaited>, + args: Record, +) => client.callTool({ name: 'query_data', arguments: args }); + +const callText = async ( + client: Awaited>, + args: Record, +) => { + const result = await callTool(client, args); + expect(result.isError).toBeFalsy(); + return (result.content as { text: string }[])[0]!.text; +}; + +// --- Storage-side mocks (workspace discovery) --- + +const SNOWFLAKE_WS = { + id: 123, + connection: { backend: 'snowflake', schema: 'WORKSPACE_123', user: 'u' }, + readOnlyStorageAccess: true, +}; + +/** + * Handles every Storage API call (`connection.test/v2/storage/...`) needed to resolve + * a workspace from the production-branch metadata. `wsOverride` swaps the workspace + * detail payload (e.g. for the BigQuery case). + */ +const storageHandler = (wsOverride?: Record) => + http.all('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + const p = url.pathname; + if (p.endsWith('/branch/default/metadata')) { + return HttpResponse.json([{ key: 'KBC.McpServer.v2.workspaceId', value: 123 }]); + } + if (p.endsWith('/branch/default/workspaces/123')) { + return HttpResponse.json(wsOverride ?? SNOWFLAKE_WS); + } + if (p.endsWith('/dev-branches')) { + return HttpResponse.json([{ id: '999', isDefault: true }]); + } + return undefined; + }); + +// --- Query Service mocks --- + +const querySuccessHandlers = (opts: { + columns: { name: string }[]; + data: unknown[][]; + numberOfRows?: number; + message?: string; +}) => [ + http.post('https://query.test/api/v1/branches/:bid/workspaces/:wid/queries', () => + HttpResponse.json({ queryJobId: 'job-1' }), + ), + http.get('https://query.test/api/v1/queries/job-1', () => + HttpResponse.json({ status: 'completed', statements: [{ id: 'stmt-1' }] }), + ), + http.get('https://query.test/api/v1/queries/job-1/stmt-1/results', () => + HttpResponse.json({ + status: 'completed', + columns: opts.columns, + data: opts.data, + numberOfRows: opts.numberOfRows ?? opts.data.length, + message: opts.message, + }), + ), +]; + +describe('query_data', () => { + it('runs a SELECT and returns CSV with a selected-rows message', async () => { + server.use( + storageHandler(), + ...querySuccessHandlers({ + columns: [{ name: 'id' }, { name: 'name' }], + data: [ + ['1', 'Alice'], + ['2', 'Bob'], + ], + numberOfRows: 2, + }), + ); + + const text = await callText(await connect(), { + sql_query: 'SELECT * FROM t', + query_name: 'My Query', + }); + expect(text).toContain('id,name'); + expect(text).toContain('Alice'); + expect(text).toContain('Bob'); + // Selected-rows message is surfaced. + expect(text).toContain('Returning 2 of 2 selected rows.'); + expect(text).toContain('My Query'); + }); + + it('quotes CSV fields that contain commas or quotes', async () => { + server.use( + storageHandler(), + ...querySuccessHandlers({ + columns: [{ name: 'val' }], + data: [['a,b'], ['he said "hi"']], + }), + ); + + const text = await callText(await connect(), { + sql_query: 'SELECT val FROM t', + query_name: 'Q', + }); + // The CSV is TOON-encoded as a quoted string, so embedded double-quotes are escaped. + expect(text).toContain('\\"a,b\\"'); + expect(text).toContain('\\"he said \\"\\"hi\\"\\"\\"'); + }); + + it('returns an error result when the query fails', async () => { + server.use( + storageHandler(), + http.post('https://query.test/api/v1/branches/:bid/workspaces/:wid/queries', () => + HttpResponse.json({ queryJobId: 'job-1' }), + ), + http.get('https://query.test/api/v1/queries/job-1', () => + HttpResponse.json({ status: 'failed', statements: [{ id: 'stmt-1' }] }), + ), + http.get('https://query.test/api/v1/queries/job-1/stmt-1/results', () => + HttpResponse.json({ + status: 'failed', + columns: [], + data: [], + message: 'boom syntax error', + }), + ), + ); + + const result = await callTool(await connect(), { + sql_query: 'SELECT bad', + query_name: 'Bad Query', + }); + expect(result.isError).toBe(true); + expect((result.content as { text: string }[])[0]!.text).toContain('boom syntax error'); + }); + + it('normalizes BigQuery error messages and uses backtick quoting', async () => { + server.use( + storageHandler({ + id: 123, + connection: { + backend: 'bigquery', + schema: 'dataset_123', + user: JSON.stringify({ project_id: 'my-proj' }), + }, + readOnlyStorageAccess: true, + }), + http.post('https://query.test/api/v1/branches/:bid/workspaces/:wid/queries', () => + HttpResponse.json({ queryJobId: 'job-1' }), + ), + http.get('https://query.test/api/v1/queries/job-1', () => + HttpResponse.json({ status: 'failed', statements: [{ id: 'stmt-1' }] }), + ), + http.get('https://query.test/api/v1/queries/job-1/stmt-1/results', () => + HttpResponse.json({ + status: 'failed', + columns: [], + data: [], + message: + 'Location: "query"; Message: "Syntax error: Unexpected identifier"; Reason: "invalidQuery"', + }), + ), + ); + + const result = await callTool(await connect(), { + sql_query: 'SELECT bad', + query_name: 'BQ Query', + }); + expect(result.isError).toBe(true); + const text = (result.content as { text: string }[])[0]!.text; + expect(text).toContain('Syntax error: Unexpected identifier'); + expect(text).not.toContain('Reason:'); + }); + + it('surfaces a cancelled query as a clean cancellation error', async () => { + server.use( + storageHandler(), + http.post('https://query.test/api/v1/branches/:bid/workspaces/:wid/queries', () => + HttpResponse.json({ queryJobId: 'job-1' }), + ), + http.get('https://query.test/api/v1/queries/job-1', () => + HttpResponse.json({ status: 'canceled', statements: [{ id: 'stmt-1' }] }), + ), + ); + + const result = await callTool(await connect(), { + sql_query: 'SELECT 1', + query_name: 'Cancelled Query', + }); + expect(result.isError).toBe(true); + expect((result.content as { text: string }[])[0]!.text).toContain('Query was cancelled'); + }); + + it('truncates by max rows across pagination', async () => { + // Page returns more than MAX_ROWS would, but the tool requests pageSize capped to remaining. + // Here we just verify a short result passes through; pagination math is covered by the prefix. + server.use( + storageHandler(), + ...querySuccessHandlers({ + columns: [{ name: 'n' }], + data: [['1'], ['2'], ['3']], + numberOfRows: 3, + }), + ); + const text = await callText(await connect(), { sql_query: 'SELECT n', query_name: 'Rows' }); + expect(text).toContain('Returning 3 of 3 selected rows.'); + }); +}); diff --git a/__tests__/tools.storage.test.ts b/__tests__/tools.storage.test.ts new file mode 100644 index 000000000..f7cbf69df --- /dev/null +++ b/__tests__/tools.storage.test.ts @@ -0,0 +1,264 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { Config } from '@/config'; +import { createServer } from '@/server'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const config = new Config({ storageApiUrl: 'https://connection.test', storageToken: 'tok' }); + +const connect = async () => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + await createServer(config).connect(serverT); + const client = new Client({ name: 'test', version: '0.0.0' }); + await client.connect(clientT); + return client; +}; + +const call = async (updates: { item_id: string; description: string }[]) => { + const client = await connect(); + const result = await client.callTool({ name: 'update_descriptions', arguments: { updates } }); + const text = (result.content as { text: string }[])[0]!.text; + await client.close(); + return text; +}; + +// Token verify backs the links manager (project id). +const verifyHandler = () => + http.get('https://connection.test/*', ({ request }) => { + if (new URL(request.url).pathname.endsWith('/tokens/verify')) { + return HttpResponse.json({ owner: { id: '42' } }); + } + return undefined; + }); + +const callTool = async (name: string, args: Record) => { + const client = await connect(); + const result = await client.callTool({ name, arguments: args }); + expect(result.isError).toBeFalsy(); + const text = (result.content as { text: string }[])[0]!.text; + await client.close(); + return text; +}; + +describe('update_descriptions', () => { + it('updates a bucket description via the bucket metadata endpoint', async () => { + let captured: { path: string; body: unknown } | undefined; + server.use( + http.post('https://connection.test/*', async ({ request }) => { + captured = { path: new URL(request.url).pathname, body: await request.json() }; + return HttpResponse.json([ + { key: 'KBC.description', value: 'desc', timestamp: '2026-01-01' }, + ]); + }), + ); + + const text = await call([{ item_id: 'in.c-main', description: 'desc' }]); + expect(captured?.path).toMatch(/\/v2\/storage\/buckets\/in\.c-main\/metadata$/); + expect(JSON.stringify(captured?.body)).toContain('KBC.description'); + expect(text).toContain('successful: 1'); + }); + + it('updates a column description via the table metadata endpoint', async () => { + let body: { columnsMetadata?: Record } | undefined; + server.use( + http.post('https://connection.test/*', async ({ request }) => { + body = (await request.json()) as { columnsMetadata?: Record }; + return HttpResponse.json({ + columnsMetadata: { + age: [{ key: 'KBC.description', value: 'years', timestamp: '2026-01-02' }], + }, + }); + }), + ); + + const text = await call([{ item_id: 'in.c-main.users.age', description: 'years' }]); + expect(body?.columnsMetadata).toBeDefined(); + expect(text).toContain('successful: 1'); + }); + + it('reports invalid item ids without calling the API', async () => { + // No handlers registered -> any HTTP call would error (onUnhandledRequest: 'error'). + const text = await call([{ item_id: 'bad-id', description: 'x' }]); + expect(text).toContain('failed: 1'); + expect(text).toContain('Invalid item_id format'); + }); +}); + +describe('get_buckets', () => { + it('lists all buckets with stage counts and the dashboard link', async () => { + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) + return HttpResponse.json({ owner: { id: '42' } }); + if (url.pathname.endsWith('/buckets')) { + return HttpResponse.json([ + { + id: 'in.c-main', + name: 'main', + displayName: 'Main', + stage: 'in', + created: '2026-01-01T00:00:00+0000', + dataSizeBytes: 100, + description: 'legacy', + }, + { + id: 'out.c-result', + name: 'result', + displayName: 'Result', + stage: 'out', + created: '2026-01-02T00:00:00+0000', + }, + ]); + } + return undefined; + }), + ); + + const text = await callTool('get_buckets', { bucket_ids: [] }); + expect(text).toContain('in.c-main'); + expect(text).toContain('out.c-result'); + expect(text).toContain('total_buckets: 2'); + expect(text).toContain('input_buckets: 1'); + expect(text).toContain('output_buckets: 1'); + // metadata description takes precedence over legacy, but absent here -> legacy used + expect(text).toContain('legacy'); + // bucket dashboard link + expect(text).toContain('/storage'); + }); + + it('prefers KBC.description metadata over the legacy description and reports missing ids', async () => { + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) + return HttpResponse.json({ owner: { id: '42' } }); + if (url.pathname.endsWith('/buckets/in.c-main')) { + return HttpResponse.json({ + id: 'in.c-main', + name: 'main', + displayName: 'Main', + stage: 'in', + created: '2026-01-01T00:00:00+0000', + description: 'legacy auto-generated', + metadata: [ + { + key: 'KBC.description', + value: 'curated description', + timestamp: '2026-01-03T00:00:00+0000', + }, + ], + }); + } + if (url.pathname.endsWith('/buckets/in.c-missing')) { + return new HttpResponse(JSON.stringify({ error: 'not found' }), { status: 404 }); + } + return undefined; + }), + ); + + const text = await callTool('get_buckets', { bucket_ids: ['in.c-main', 'in.c-missing'] }); + expect(text).toContain('curated description'); + expect(text).not.toContain('legacy auto-generated'); + expect(text).toContain('in.c-missing'); // buckets_not_found + }); +}); + +describe('get_tables', () => { + it('lists table summaries for a bucket (no FQN / columns)', async () => { + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) + return HttpResponse.json({ owner: { id: '42' } }); + if (url.pathname.endsWith('/buckets/in.c-main')) { + return HttpResponse.json({ id: 'in.c-main', name: 'main', stage: 'in', created: 'c' }); + } + if (url.pathname.endsWith('/buckets/in.c-main/tables')) { + return HttpResponse.json([ + { + id: 'in.c-main.users', + name: 'users', + displayName: 'Users', + primaryKey: ['id', 'email'], + rowsCount: 5, + }, + ]); + } + return undefined; + }), + ); + + const text = await callTool('get_tables', { bucket_ids: ['in.c-main'] }); + expect(text).toContain('in.c-main.users'); + expect(text).toContain('id|email'); // primary key serialized as joined string + expect(text).not.toContain('fullyQualifiedName'); + }); + + it('returns full table detail with columns and a Snowflake FQN from backendPath', async () => { + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) + return HttpResponse.json({ owner: { id: '42' } }); + if (url.pathname.endsWith('/tables/in.c-main.users')) { + return HttpResponse.json({ + id: 'in.c-main.users', + name: 'users', + displayName: 'Users', + created: 'c', + bucket: { id: 'in.c-main', backendPath: ['DB', 'in.c-main'] }, + columns: ['id', 'name'], + columnMetadata: { + id: [ + { key: 'KBC.datatype.type', value: 'NUMBER', timestamp: 't' }, + { key: 'KBC.datatype.nullable', value: '0', timestamp: 't' }, + { key: 'KBC.description', value: 'identifier', timestamp: 't' }, + ], + name: [{ key: 'KBC.datatype.nullable', value: '1', timestamp: 't' }], + }, + }); + } + return undefined; + }), + ); + + const text = await callTool('get_tables', { table_ids: ['in.c-main.users'] }); + expect(text).toContain('fullyQualifiedName'); // FQN derived from backendPath + expect(text).toContain('DB'); // Snowflake-quoted backendPath parts present + expect(text).toContain('NUMBER'); + expect(text).toContain('identifier'); + // name column has no datatype.type -> defaults to VARCHAR + expect(text).toContain('VARCHAR'); + }); + + it('reports tables_not_found for unknown table ids', async () => { + server.use( + verifyHandler(), + http.get('https://connection.test/*', ({ request }) => { + const url = new URL(request.url); + if (url.pathname.endsWith('/tokens/verify')) + return HttpResponse.json({ owner: { id: '42' } }); + if (url.pathname.includes('/tables/')) { + return new HttpResponse(JSON.stringify({ error: 'nope' }), { status: 404 }); + } + return undefined; + }), + ); + + const text = await callTool('get_tables', { table_ids: ['in.c-main.ghost'] }); + expect(text).toContain('in.c-main.ghost'); + }); +}); diff --git a/__tests__/workspace.test.ts b/__tests__/workspace.test.ts new file mode 100644 index 000000000..48e98467f --- /dev/null +++ b/__tests__/workspace.test.ts @@ -0,0 +1,528 @@ +import { http, HttpResponse } from 'msw'; +import { setupServer } from 'msw/node'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; + +import { createRawClient } from '@/clients/raw'; +import { Config } from '@/config'; +import type { JobSubmittedInfo } from '@/workspace'; +import { WorkspaceManager } from '@/workspace'; + +/** + * Port of `tests/test_workspace.py`. + * + * The TypeScript workspace layer is built on the `@keboola/api-client` Query Service + * client + a raw Storage client (no per-workspace `QueryServiceClient` mock surface like + * the Python version had), so these tests exercise the *exported* `WorkspaceManager` + * end-to-end over msw — mirroring `tools.sql.test.ts` — rather than reaching into private + * internals. They cover the same behaviors as the Python suite: branch-aware resolution, + * storage-branches fallback, schema/metadata/auto-create discovery, the on-job-submitted + * callback, cancellation short-circuiting, dialect quoting, and BigQuery error normalization. + */ + +const STORAGE_URL = 'https://connection.test'; +const QUERY_URL = 'https://query.test'; + +const server = setupServer(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const SNOWFLAKE_WS = { + id: 123, + connection: { backend: 'snowflake', schema: 'WORKSPACE_123', user: 'u' }, + readOnlyStorageAccess: true, +}; + +const BIGQUERY_WS = { + id: 123, + connection: { + backend: 'bigquery', + schema: 'dataset_123', + user: JSON.stringify({ project_id: 'my-proj' }), + }, + readOnlyStorageAccess: true, +}; + +/** Builds a WorkspaceManager wired to the msw-backed Storage + Query Service. */ +const makeManager = async (cfg: Config): Promise => { + const token = cfg.bearerToken ? `Bearer ${cfg.bearerToken}` : (cfg.storageToken ?? ''); + const makeStorage = () => createRawClient({ baseUrl: `${STORAGE_URL}/v2/storage`, token }); + return WorkspaceManager.create(cfg, { + rawStorage: makeStorage(), + makeProdRawStorage: makeStorage, + queryServiceUrl: QUERY_URL, + queryServiceToken: token, + }); +}; + +/** Records every Storage API path requested, so tests can assert branch routing. */ +const recordingStorageHandler = ( + paths: string[], + opts: { + features?: string[]; + wsDetail?: Record; + metadata?: { key: string; value: unknown }[]; + wsList?: Record[]; + } = {}, +) => + http.all(`${STORAGE_URL}/*`, ({ request }) => { + const p = new URL(request.url).pathname; + paths.push(p); + if (p.endsWith('/tokens/verify')) { + return HttpResponse.json({ + owner: { + id: '42', + features: opts.features ?? [], + defaultBackend: 'snowflake', + }, + }); + } + if (/\/branch\/[^/]+\/metadata$/.test(p)) { + return HttpResponse.json( + opts.metadata ?? [{ key: WorkspaceManager.MCP_META_KEY, value: 123 }], + ); + } + if (/\/branch\/[^/]+\/workspaces\/123$/.test(p)) { + return HttpResponse.json(opts.wsDetail ?? SNOWFLAKE_WS); + } + if (/\/branch\/[^/]+\/workspaces$/.test(p)) { + return HttpResponse.json(opts.wsList ?? [SNOWFLAKE_WS]); + } + if (p.endsWith('/dev-branches')) { + return HttpResponse.json([{ id: '999', isDefault: true }]); + } + return undefined; + }); + +const querySuccessHandlers = (opts: { + columns?: { name: string }[]; + data?: unknown[][]; + numberOfRows?: number; + message?: string; + jobStatus?: string; + resultsStatus?: string; +}) => [ + http.post(`${QUERY_URL}/api/v1/branches/:bid/workspaces/:wid/queries`, () => + HttpResponse.json({ queryJobId: 'job-1' }), + ), + http.get(`${QUERY_URL}/api/v1/queries/job-1`, () => + HttpResponse.json({ + status: opts.jobStatus ?? 'completed', + statements: [{ id: 'stmt-1' }], + }), + ), + http.get(`${QUERY_URL}/api/v1/queries/job-1/stmt-1/results`, () => + HttpResponse.json({ + status: opts.resultsStatus ?? 'completed', + columns: opts.columns ?? [{ name: 'col' }], + data: opts.data ?? [['v']], + numberOfRows: opts.numberOfRows ?? (opts.data ?? [['v']]).length, + message: opts.message ?? 'ok', + }), + ), +]; + +describe('WorkspaceManager.create — branch awareness', () => { + it.each([ + // [label, branchId, hasSbFeature, expectedBranchInPath] + ['default branch always production (feature on)', undefined, true, 'default'], + ['default branch always production (feature off)', undefined, false, 'default'], + ['dev branch + storage-branches keeps dev branch', '456', true, '456'], + ['dev branch without storage-branches falls back to production', '456', false, 'default'], + ])('%s', async (_label, branchId, hasSb, expectedBranch) => { + const paths: string[] = []; + server.use( + recordingStorageHandler(paths, { + features: hasSb ? ['storage-branches'] : [], + }), + ...querySuccessHandlers({}), + ); + + const cfg = new Config({ + storageApiUrl: STORAGE_URL, + storageToken: 'tok', + branchId, + }); + const manager = await makeManager(cfg); + // Force workspace resolution (which issues the branch-scoped metadata lookup). + await manager.getWorkspaceId(); + + const metaPath = paths.find((p) => /\/branch\/[^/]+\/metadata$/.test(p)); + expect(metaPath).toBeDefined(); + expect(metaPath).toContain(`/branch/${expectedBranch}/metadata`); + }); + + it('skips the feature lookup on the default branch', async () => { + const paths: string[] = []; + server.use(recordingStorageHandler(paths, {}), ...querySuccessHandlers({})); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + await manager.getWorkspaceId(); + + // On the default branch the feature check (tokens/verify) is short-circuited. + expect(paths.some((p) => p.endsWith('/tokens/verify'))).toBe(false); + }); + + it('performs the feature lookup on a dev branch', async () => { + const paths: string[] = []; + server.use( + recordingStorageHandler(paths, { features: ['storage-branches'] }), + ...querySuccessHandlers({}), + ); + + const cfg = new Config({ + storageApiUrl: STORAGE_URL, + storageToken: 'tok', + branchId: '456', + }); + const manager = await makeManager(cfg); + await manager.getWorkspaceId(); + + expect(paths.some((p) => p.endsWith('/tokens/verify'))).toBe(true); + }); +}); + +describe('WorkspaceManager — workspace discovery', () => { + it('resolves a workspace via branch metadata + read-only detail', async () => { + const paths: string[] = []; + server.use(recordingStorageHandler(paths, {}), ...querySuccessHandlers({})); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + + expect(await manager.getWorkspaceId()).toBe(123); + expect(await manager.getSqlDialect()).toBe('Snowflake'); + expect(paths.some((p) => /\/branch\/[^/]+\/metadata$/.test(p))).toBe(true); + }); + + it('resolves a workspace by explicit schema (never touches branch metadata)', async () => { + const paths: string[] = []; + server.use( + recordingStorageHandler(paths, { wsList: [SNOWFLAKE_WS] }), + ...querySuccessHandlers({}), + ); + + const cfg = new Config({ + storageApiUrl: STORAGE_URL, + storageToken: 'tok', + workspaceSchema: 'WORKSPACE_123', + }); + const manager = await makeManager(cfg); + + expect(await manager.getWorkspaceId()).toBe(123); + // The schema path lists workspaces; it must not read branch metadata. + expect(paths.some((p) => /\/branch\/[^/]+\/workspaces$/.test(p))).toBe(true); + expect(paths.some((p) => /\/branch\/[^/]+\/metadata$/.test(p))).toBe(false); + }); + + it('throws when an explicit schema matches no workspace', async () => { + const paths: string[] = []; + server.use(recordingStorageHandler(paths, { wsList: [] })); + + const cfg = new Config({ + storageApiUrl: STORAGE_URL, + storageToken: 'tok', + workspaceSchema: 'MISSING', + }); + const manager = await makeManager(cfg); + + await expect(manager.getWorkspaceId()).rejects.toThrow(/No Keboola workspace found/); + }); + + it('auto-creates a workspace + writes metadata when none exists', async () => { + const posts: { path: string; body: unknown }[] = []; + server.use( + http.all(`${STORAGE_URL}/*`, async ({ request }) => { + const p = new URL(request.url).pathname; + if (request.method === 'POST') { + posts.push({ path: p, body: await request.clone().json() }); + } + if (p.endsWith('/tokens/verify')) { + return HttpResponse.json({ + owner: { id: '42', defaultBackend: 'snowflake' }, + }); + } + // No existing workspace recorded in metadata. + if (/\/branch\/[^/]+\/metadata$/.test(p) && request.method === 'GET') { + return HttpResponse.json([]); + } + // POST metadata write-back. + if (/\/branch\/[^/]+\/metadata$/.test(p) && request.method === 'POST') { + return HttpResponse.json([{ key: WorkspaceManager.MCP_META_KEY, value: 123 }]); + } + // Create config under the billing component. + if (/\/components\/[^/]+\/configs$/.test(p) && request.method === 'POST') { + return HttpResponse.json({ id: 'cfg-1' }); + } + // Create workspace -> returns an async job id. + if (/\/configs\/[^/]+\/workspaces$/.test(p) && request.method === 'POST') { + return HttpResponse.json({ id: 9001 }); + } + // Poll the job: immediately successful. + if (p.endsWith('/jobs/9001')) { + return HttpResponse.json({ status: 'success', results: { id: 123 } }); + } + // Resolve the created workspace by id. + if (/\/branch\/[^/]+\/workspaces\/123$/.test(p)) { + return HttpResponse.json(SNOWFLAKE_WS); + } + return undefined; + }), + ); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + + expect(await manager.getWorkspaceId()).toBe(123); + // A config + a workspace were created, and the id was written back to metadata. + expect(posts.some((x) => /\/components\/[^/]+\/configs$/.test(x.path))).toBe(true); + expect(posts.some((x) => /\/configs\/[^/]+\/workspaces$/.test(x.path))).toBe(true); + const metaWrite = posts.find((x) => /\/branch\/[^/]+\/metadata$/.test(x.path)); + expect(metaWrite).toBeDefined(); + expect(JSON.stringify(metaWrite!.body)).toContain(WorkspaceManager.MCP_META_KEY); + }); + + it('cleans up the created config when workspace creation fails', async () => { + const deleted: string[] = []; + server.use( + http.all(`${STORAGE_URL}/*`, ({ request }) => { + const p = new URL(request.url).pathname; + if (request.method === 'DELETE') deleted.push(p); + if (p.endsWith('/tokens/verify')) { + return HttpResponse.json({ + owner: { id: '42', defaultBackend: 'snowflake' }, + }); + } + if (/\/branch\/[^/]+\/metadata$/.test(p) && request.method === 'GET') { + return HttpResponse.json([]); + } + if (/\/components\/[^/]+\/configs$/.test(p) && request.method === 'POST') { + return HttpResponse.json({ id: 'cfg-1' }); + } + // Workspace creation fails hard. + if (/\/configs\/[^/]+\/workspaces$/.test(p) && request.method === 'POST') { + return new HttpResponse('boom', { status: 500 }); + } + // Config cleanup. + if (/\/components\/[^/]+\/configs\/cfg-1$/.test(p) && request.method === 'DELETE') { + return new HttpResponse(null, { status: 204 }); + } + return undefined; + }), + ); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + + await expect(manager.getWorkspaceId()).rejects.toThrow(); + expect(deleted.some((p) => /\/components\/[^/]+\/configs\/cfg-1$/.test(p))).toBe(true); + }); +}); + +describe('WorkspaceManager — dialect quoting', () => { + it('Snowflake quotes identifiers with double quotes', async () => { + server.use(recordingStorageHandler([], {})); + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + expect(await manager.getQuotedName('foo')).toBe('"foo"'); + }); + + it('BigQuery quotes identifiers with backticks', async () => { + server.use(recordingStorageHandler([], { wsDetail: BIGQUERY_WS })); + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + expect(await manager.getQuotedName('foo')).toBe('`foo`'); + expect(await manager.getSqlDialect()).toBe('BigQuery'); + }); + + it('rejects a BigQuery workspace without a project id in credentials', async () => { + server.use( + recordingStorageHandler([], { + wsDetail: { + id: 123, + connection: { backend: 'bigquery', schema: 'ds', user: '{}' }, + readOnlyStorageAccess: true, + }, + }), + ); + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + await expect(manager.getSqlDialect()).rejects.toThrow(/no project ID/i); + }); +}); + +describe('WorkspaceManager.executeQuery — Query Service submit/poll/paginate', () => { + it('runs a SELECT and returns columns/rows + selected-rows message', async () => { + server.use( + recordingStorageHandler([], {}), + ...querySuccessHandlers({ + columns: [{ name: 'id' }, { name: 'name' }], + data: [ + ['1', 'Alice'], + ['2', 'Bob'], + ], + numberOfRows: 2, + }), + ); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + const result = await manager.executeQuery('SELECT * FROM t'); + + expect(result.status).toBe('ok'); + expect(result.data?.columns).toEqual(['id', 'name']); + expect(result.data?.rows).toEqual([ + { id: '1', name: 'Alice' }, + { id: '2', name: 'Bob' }, + ]); + expect(result.message).toContain('Returning 2 of 2 selected rows.'); + }); + + it('invokes on_job_submitted once with the full job info', async () => { + server.use(recordingStorageHandler([], {}), ...querySuccessHandlers({})); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + + const received: JobSubmittedInfo[] = []; + await manager.executeQuery('SELECT 1', { + onJobSubmitted: async (info) => { + received.push(info); + }, + }); + + expect(received).toHaveLength(1); + expect(received[0]).toEqual({ + job_id: 'job-1', + cancellation_url: `${QUERY_URL}/api/v1/queries/job-1/cancel`, + backend: 'snowflake', + }); + }); + + it('swallows a callback exception and still completes the query', async () => { + server.use(recordingStorageHandler([], {}), ...querySuccessHandlers({})); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + + const result = await manager.executeQuery('SELECT 1', { + onJobSubmitted: async () => { + throw new Error('progress send failed'); + }, + }); + + expect(result.status).toBe('ok'); + }); + + it('rejects non-positive max_rows / max_chars', async () => { + server.use(recordingStorageHandler([], {}), ...querySuccessHandlers({})); + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + + await expect(manager.executeQuery('SELECT 1', { maxRows: 0 })).rejects.toThrow(/max_rows/); + await expect(manager.executeQuery('SELECT 1', { maxChars: 0 })).rejects.toThrow(/max_chars/); + }); + + it('truncates results to max_rows', async () => { + server.use( + recordingStorageHandler([], {}), + ...querySuccessHandlers({ + columns: [{ name: 'n' }], + data: [['1'], ['2'], ['3'], ['4'], ['5']], + numberOfRows: 5, + }), + ); + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + + const result = await manager.executeQuery('SELECT n', { maxRows: 2 }); + expect(result.data?.rows).toHaveLength(2); + expect(result.message).toContain('Returning 2 of 5 selected rows.'); + }); + + it('truncates results to max_chars on the first row that does not fit', async () => { + server.use( + recordingStorageHandler([], {}), + ...querySuccessHandlers({ + columns: [{ name: 'v' }], + data: [['aaa'], ['bbb'], ['ccc']], // 3 chars each + numberOfRows: 3, + }), + ); + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + + // Room for two rows (6 chars), not the third. + const result = await manager.executeQuery('SELECT v', { maxChars: 6 }); + expect(result.data?.rows).toEqual([{ v: 'aaa' }, { v: 'bbb' }]); + }); +}); + +describe('WorkspaceManager.executeQuery — error handling', () => { + it('returns an error result when the query fails', async () => { + server.use( + recordingStorageHandler([], {}), + ...querySuccessHandlers({ + jobStatus: 'failed', + resultsStatus: 'failed', + columns: [], + data: [], + message: 'boom syntax error', + }), + ); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + const result = await manager.executeQuery('SELECT bad'); + + expect(result.status).toBe('error'); + expect(result.data).toBeFalsy(); + expect(result.message).toContain('boom syntax error'); + }); + + it('short-circuits a cancelled job with a clean message', async () => { + server.use( + recordingStorageHandler([], {}), + http.post(`${QUERY_URL}/api/v1/branches/:bid/workspaces/:wid/queries`, () => + HttpResponse.json({ queryJobId: 'job-1' }), + ), + http.get(`${QUERY_URL}/api/v1/queries/job-1`, () => + HttpResponse.json({ + status: 'canceled', + statements: [{ id: 'stmt-1' }], + }), + ), + ); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + const result = await manager.executeQuery('SELECT 1'); + + expect(result.status).toBe('error'); + expect(result.data).toBeNull(); + expect(result.message).toBe('Query was cancelled'); + }); + + it('normalizes BigQuery error messages to the Message: "..." part', async () => { + server.use( + recordingStorageHandler([], { wsDetail: BIGQUERY_WS }), + ...querySuccessHandlers({ + jobStatus: 'failed', + resultsStatus: 'failed', + columns: [], + data: [], + message: + 'Location: "query"; Message: "Syntax error: Unexpected identifier"; Reason: "invalidQuery"', + }), + ); + + const cfg = new Config({ storageApiUrl: STORAGE_URL, storageToken: 'tok' }); + const manager = await makeManager(cfg); + const result = await manager.executeQuery('SELECT bad'); + + expect(result.status).toBe('error'); + expect(result.message).toBe('Syntax error: Unexpected identifier'); + expect(result.message).not.toContain('Reason:'); + }); +}); diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..f96eda377 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,82 @@ +# Local dev stack for the MCP server + its docs-search index. +# +# Three services: +# pgvector — Postgres + pgvector, backs the docs-search index (docs_query / find_component_id). +# docs-seed — one-shot: applies the schema + seeds the fixture corpus (migrate + seed), then exits. +# mcp — the MCP server (production image) over streamable-HTTP on :8000. +# +# The MCP only ever *reads* the index; docs-seed is the local mirror of the production +# out-of-band build (see feature_spec/docs-search-pgvector/). +# +# Embedder: defaults to the deterministic `stub` (offline, no download, no key) — good for a +# smoke stack, but NOT semantic, so realistic queries return nothing. For real semantic +# search with no external service, flip the shared `x-docs-embedder` anchor below to the +# in-process HuggingFace model: +# DOCS_EMBEDDER_MODEL: local +# DOCS_EMBEDDER_DIM: '384' +# (or point DOCS_EMBEDDER_ENDPOINT/API_KEY/MODEL at a remote embedder). The anchor is shared +# by docs-seed and mcp so build-time and query-time embedders can never drift. +# +# Usage: +# docker compose up -d --wait pgvector # just Postgres (used by `npm run test:integ`) +# docker compose up --build # full stack: pgvector -> docs-seed -> mcp +# curl localhost:8000/health-check # MCP health once up ({"status":"ok"}) + +# Shared docs-index env — edit here to switch stub / local / remote for BOTH services at once. +x-docs-embedder: &docs-embedder + DATABASE_URL: postgres://mcp:mcp@pgvector:5432/docs + DOCS_EMBEDDER_MODEL: stub + DOCS_EMBEDDER_DIM: '3072' + +services: + pgvector: + # pgvector >= 0.7 is required for halfvec(3072); the pg17 tag ships it. + image: pgvector/pgvector:pg17 + environment: + POSTGRES_USER: mcp + POSTGRES_PASSWORD: mcp + POSTGRES_DB: docs + ports: + - '5432:5432' + volumes: + - pgvector-data:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U mcp -d docs'] + interval: 5s + timeout: 3s + retries: 10 + + # Migrate the schema + seed the fixture corpus, then exit. Runs the compiled seeder + # (`dist/docs-build.js`) from the same image the MCP runs — no tsx/scripts needed. + docs-seed: + build: . + image: keboola-mcp-server:local + command: ['node', 'dist/docs-build.js'] + environment: *docs-embedder + depends_on: + pgvector: + condition: service_healthy + restart: 'no' + + mcp: + image: keboola-mcp-server:local + # Reuses the image built by docs-seed (build: .). streamable-HTTP is the deployed mode. + command: ['node', 'dist/index.js', '--transport', 'streamable-http'] + ports: + - '8000:8000' + environment: + # Shared docs-index embedder (same as the seeder — must match, hence the anchor). + <<: *docs-embedder + HOST: 0.0.0.0 + PORT: '8000' + # HTTP mode is multi-tenant: clients pass the Storage token per request via + # X-StorageApi-Token / Authorization headers, so no KBC_* token is needed at boot. + # For a single-tenant local run you can bake them in instead: + # KBC_STORAGE_API_URL: https://connection..keboola.com + # KBC_STORAGE_TOKEN: + depends_on: + docs-seed: + condition: service_completed_successfully + +volumes: + pgvector-data: diff --git a/feature_spec/docs-search-pgvector/RFC.md b/feature_spec/docs-search-pgvector/RFC.md new file mode 100644 index 000000000..91f4b86ec --- /dev/null +++ b/feature_spec/docs-search-pgvector/RFC.md @@ -0,0 +1,183 @@ +# RFC: Replace the AI docs-service with the pgvector docs-search SDK + +Linear: PSGO-268 (TypeScript rewrite follow-up) · depends on +[`keboola/ui#6672`](https://github.com/keboola/ui/pull/6672) — +`@keboola/docs-search` (pgvector index builder + retrieval SDK). + +## Problem + +Three MCP tools currently reach the **AI service** (`ai.` / the legacy Python +LanceDB-Qdrant docs index) for documentation intelligence: + +| Tool | Today (AI service) | What it needs | +| --- | --- | --- | +| `docs_query` | `rawAi` `POST docs/question` | Q&A over Keboola docs with source URLs | +| `find_component_id` | `ai.suggestComponent` (`POST suggest/component`) | component recommendation for a task | +| `get_config_examples` / `fetchComponent` docs | `rawAi` `GET docs/components/{id}` | a component's documentation page | + +Problems with the AI-service dependency: +- **Reliability**: live runs show intermittent `422 Request contents is not valid` from + `docs/question` and read-timeouts — the docs path is the flakiest surface in the suite. +- **Opaque, external, single-region**: an out-of-band service the MCP can't reason about, + version, or run locally; couples MCP availability to it. +- **Duplication**: `keboola/ui#6672` introduces `@keboola/docs-search`, a first-class SDK + that builds the docs index into **pgvector** and serves the exact same three retrieval + shapes. Continuing to call the AI service means maintaining two doc backends. + +The rewrite already reuses `@keboola/api-client` for everything else; docs should likewise +move to the published SDK, backed by a Postgres/pgvector index that is **built out-of-band +and only read at runtime**. + +## Required Behavior + +| # | Requirement | +| --- | --- | +| 1 | `docs_query`, `find_component_id`, and component-docs retrieval are served by `@keboola/docs-search` (`answerQuestion` / `recommendComponents` / `getComponentDoc`) against a pgvector index — **same tool names, params, and output shapes** (`DocsAnswer{text, source_urls}`, suggested-component list, component doc). | +| 2 | The **AI-service integration is removed entirely**: `rawAi`, `ai` (typed client), the `ai`/`docs/*`/`suggest/*` endpoints and their URL derivation are deleted from the MCP. | +| 3 | **Postgres is added to `docker-compose`** (dev) and consumed via a single `DATABASE_URL` (pgvector-enabled instance) at runtime. | +| 4 | The index is **filled by a separate build job** (the `@keboola/docs-search` `runIndexBuild` + the source connectors), not by the MCP. A **cron job** rebuilds it periodically (incremental: only changed docs are re-embedded). | +| 5 | The docs tools are available **only when the MCP has access to the docs index** (a reachable, migrated, non-empty pgvector DB). When it isn't, the three docs tools degrade gracefully (filtered out / clear "docs index unavailable" error) — the rest of the server is unaffected. | +| 6 | **Auth/gate** (open question, see below): the docs index is **global, non-tenant data**; the MCP reads it with its own server-side `DATABASE_URL`. No per-user docs token is introduced; the tools follow normal read-only tool visibility. | +| 7 | The MCP has **no build-time dependency on index creation** — it starts and serves even if the index is stale/missing; index building lives entirely in the aside job. | + +### Retrieval mapping (SDK → tools) + +``` +docs_query(query) -> createDocsSearch({pool,embedder,llm}).answerQuestion(query) -> {text, source_urls} +find_component_id(query) -> .recommendComponents(query) -> [{componentId, score}] +get_config_examples(id) / + component docs (fetchComponent) -> .getComponentDoc(id) -> ParentDoc | null +``` + +The SDK is **dependency-injected** (`pool: pg.Pool`, `embedder: Embedder`, optional `llm: Llm`) +and owns no globals — it fits the MCP's per-request client-factory model. + +### Open question — the access gate (point 5) + +The docs index is **global** (Keboola help/dev docs + the public component catalog), not +project data, so reading it does not require a Storage-scoped grant. Candidate gates, with a +recommendation: + +| Option | Meaning | Assessment | +| --- | --- | --- | +| **No auth on the index; tool gated by a valid MCP session** (recommended) | MCP holds `DATABASE_URL` (infra secret); the three docs tools are offered to any authenticated session, like other read-only tools | Simplest; matches that docs are non-sensitive/global; the DB is never exposed to the client | +| Storage token required | Reuse the per-request Storage token as the gate | Adds no real security (docs aren't project-scoped) but keeps "must be a Keboola user" | +| Management token | Gate on a management-scoped token | Overkill; docs are not org-admin data | +| Application/session token to the embedder | Needed only for the **embedder** (query embedding) if it calls a hosted embedding API | Orthogonal to index access — see below | + +**Recommendation:** treat index access as **infrastructure** (server-side `DATABASE_URL`), keep +the tools visible to any authenticated session, and make the **embedder** credential a +deployment secret (the same way the AI service key is a deployment concern today), not a user +token. Final call to be confirmed with platform security before implementation. + +## Resolution Strategy + +Runtime (MCP) — thin, read-only: +- Add a `docsSearch` to the client factory: build a shared `pg.Pool` from `DATABASE_URL` and an + `Embedder` from deployment config, then `createDocsSearch({ pool, embedder, llm })`. The pool + is process-scoped (not per-request) and injected where the three tools need it. +- Rewrite `src/tools/doc.ts` (`docs_query` → `answerQuestion`), the `find_component_id` handler + in `src/tools/search/` (→ `recommendComponents`), and the component-docs reads in + `src/tools/components/` + `get_config_examples` (→ `getComponentDoc`). Preserve tool + names/descriptions/output shapes exactly. +- **Delete** the AI surface: `rawAi`, the typed `ai` client, `urls.ai`, and related config. +- Availability gate: at server build, probe the index (`index_meta.last_success_at` present + + reachable). If absent, filter the three docs tools out of `tools/list` and deny calls with a + clear message (parity with the existing feature-gating in `src/mcp/filtering.ts`). +- Config: add `DATABASE_URL` (+ embedder endpoint/key/model) to `src/env.ts` as **optional** + deployment env; the server boots without them (docs tools just gate off). + +Index build (aside — NOT in the MCP request path): +- A separate job (own package/app or a small `scripts/` entry) runs the connectors (git clone + of help/dev docs + component catalog + frontmatter parse), then `runIndexBuild(pool, {sources, + embedder, gates})`. Transactional + incremental: COMMIT iff gates pass, else ROLLBACK (the + prior index stays intact). This is scheduled by **cron** (see the architecture doc). +- The MCP and the build job share only the Postgres database (the index), never code paths. + +Local dev: +- `docker-compose` gains a `pgvector/pgvector` service; `migrate(pool)` (from the SDK) applies + `migrations/*.sql` (creates the `vector` extension + `doc`/`doc_chunk`/`index_manifest`/ + `index_meta` tables + HNSW index). A `npm run docs:build` convenience wires the connectors + + `runIndexBuild` against the local DB so a developer can populate and query locally. + +### Non-obvious trade-offs +- **Embedder at query time**: `answerQuestion`/`recommendComponents` embed the query, so the MCP + needs an `Embedder` (a hosted embedding endpoint or a local model). This is the one remaining + external call; it replaces the AI service with a much narrower dependency (embeddings only, + no bespoke retrieval service). `getComponentDoc` needs **no** embedder (direct lookup). +- **`llm` for `answerQuestion`**: Q&A needs an LLM to synthesize the answer from retrieved docs. + Provided via deployment config; if absent, `docs_query` can fall back to returning top + retrieved snippets (`search`) rather than a synthesized answer. +- **Stale index tolerance**: because the MCP only reads, a failed/late build serves the last + good index; the MCP never blocks on building. + +## Scope + +In scope: swapping the three docs tools onto `@keboola/docs-search`; removing the AI-service +client/URLs/config; adding Postgres to docker-compose + optional `DATABASE_URL`/embedder env; +the availability gate; a local `docs:build` path; unit tests (mock the SDK) + an integ test +(against a seeded local pgvector). + +Out of scope (tracked in the architecture doc): the production index-build **app + cron** +deployment, the **source connectors** (git clone/frontmatter — owned by the build side per +#6672), Terraform for the provisioned Postgres + pgvector extension, and embedder-provider +selection/procurement. + +## Implementation status + +Landed in the TypeScript server (PSGO-268), with two deliberate deviations from the +original plan, agreed during implementation: + +1. **`get_config_examples` and `fetchComponent` stay on the AI catalog.** The pgvector + index holds *markdown documentation pages*, not the structured component metadata + (`configurationSchema`, `rootConfigurationExamples`, sync actions) these paths need — + `getComponentDoc` returns `{content, sourceUrl, title}`, which cannot back config + validation. So **only the two genuine semantic-retrieval tools were migrated**: + `docs_query` → `answerQuestion`, `find_component_id` → `recommendComponents`. The raw + AI-catalog client (`rawAi`) is retained solely for `docs/components/{id}`; the unused + typed `ai` client (`suggestComponent`) was removed. Full AI removal (point 2) is a + follow-up that needs a component-metadata replacement, not just docs-search. +2. **The SDK is vendored, temporarily.** `@keboola/docs-search` is a private, + unpublished workspace package (keboola/ui#6672), so its ~150-line retrieval tier is + copied into `src/clients/docsSearch.ts` behind the exact `DocsSearch` interface, with a + documented swap-to-published-package path. The one SDK gap this needed — + `recommendComponents` exposing each result's `sourceKey` (the component id lives in + `source_key = 'component:'`, which the SDK's SELECT dropped) — was fixed on + keboola/ui#6672 (`aa8a779`), so the vendored SELECT matches the published shape and the + swap will be a clean drop-in. + +The gate (point 5) is wired via `DOCS_INDEX_TOOL_NAMES` in `mcp/filtering.ts`: the two +tools are filtered from `tools/list` and denied on call when `getDocsSearch()` returns +`null` (no `DATABASE_URL` / embedder creds). Env (point 3) is `DATABASE_URL` + +`DOCS_EMBEDDER_*` + optional `DOCS_LLM_*` in `env.ts`; `docker-compose.yml` adds the +`pgvector/pgvector` service. The production index build (point 4) remains out of scope +here — see the architecture doc. + +**Local + CI end-to-end.** A developer can run the two tools against a real local index +with no live services and no API key: + +``` +docker compose up -d pgvector +DATABASE_URL=postgres://mcp:mcp@localhost:5432/docs DOCS_EMBEDDER_MODEL=stub npm run docs:build +# then point the MCP at DATABASE_URL + DOCS_EMBEDDER_MODEL=stub +``` + +`npm run docs:build` (`scripts/docs-build.ts` + `scripts/docsIndex.ts`) migrates the schema +and seeds a small fixture corpus using the deterministic offline `StubEmbedder` +(`DOCS_EMBEDDER_MODEL=stub`), which the provider also uses at query time so build and query +embed identically. This is the dev mirror of the production out-of-band build (it seeds +fixtures, not real docs — the source connectors stay on the build side per #6672). + +## Testing / Verification +- **Unit**: `docs_query` / `find_component_id` / `get_config_examples` with an injected fake + `DocsSearch` (stub `search`/`answerQuestion`/`getComponentDoc`/`recommendComponents`) — assert + the tools map inputs/outputs unchanged. Availability-gate tests (index present/absent → tool + visible/denied). No network. +- **Integration** (`integtests/tools/doc.test.ts`): provisions a real `pgvector` via + testcontainers, seeds the fixture corpus with the deterministic `StubEmbedder`, and drives + `docs_query` + `find_component_id` through the MCP client, asserting real retrieval and + component-id recovery. Self-contained (needs Docker; skips with a warning if unavailable), + so the existing `integration_tests` CI job runs it with no extra wiring. Mirrors the SDK's + own testcontainers integ tier. +- **Parity check**: compare outputs against the current AI-service tools for a fixed query set + before deleting the AI path. diff --git a/feature_spec/docs-search-pgvector/architecture.md b/feature_spec/docs-search-pgvector/architecture.md new file mode 100644 index 000000000..01304eb59 --- /dev/null +++ b/feature_spec/docs-search-pgvector/architecture.md @@ -0,0 +1,156 @@ +# Architecture & Rollout: Docs-search on Postgres/pgvector + +Companion to [`RFC.md`](./RFC.md). This document covers the **infrastructure and rollout** for +serving MCP documentation intelligence from a pgvector index built out-of-band — the pieces the +platform doesn't have yet (a provisioned Postgres with `pgvector`, the extension install path, a +scheduled index-build job) and how they fit together. + +Guiding principle: **the MCP server reads a prebuilt index; it never builds one.** Index +creation is a separate, scheduled concern. If the builder is down or mid-run, the MCP keeps +serving the last committed index. + +## 1. Components & topology + +``` + ┌──────────────────────────────────────────────────────────┐ + docs sources │ Index Builder (aside job / cron) │ + (help repo, │ connectors: git clone + frontmatter parse + URL derive │ + dev-portal, │ @keboola/docs-search: chunk → hash-diff → embed(changed)│ + component │ → runIndexBuild(): txn COMMIT iff gates pass else ROLLBACK│ + catalog) └───────────────┬──────────────────────────────────────────┘ + │ writes (transactional) + ▼ + ┌────────────────────────┐ embeddings + │ Postgres + pgvector │◀─────── Embedder (hosted embedding API + │ doc / doc_chunk / │ or self-hosted model) + │ index_manifest / │ + │ index_meta (HNSW idx) │ + └───────────┬─────────────┘ + │ reads only (SELECT + vector search) + ▼ + ┌────────────────────────┐ + │ MCP server (per pod) │ createDocsSearch({pool, embedder, llm}) + │ docs_query / │ → answerQuestion / recommendComponents + │ find_component_id / │ → getComponentDoc + │ get_config_examples │ + └────────────────────────┘ +``` + +Two independent deployables share **only the database**: +- **Index Builder** — writes the index. Scheduled (cron). Owns the connectors + `runIndexBuild`. +- **MCP server** — reads the index. Stateless w.r.t. the index; one shared `pg.Pool` per pod. + +## 2. Postgres + pgvector provisioning (new dependency) + +The platform has no Postgres for the MCP today. Rollout: + +- **Instance**: a managed Postgres (per stack/region) reachable by both the MCP pods and the + builder job. Sized for the index (docs are small; the embeddings dominate — `halfvec(3072)` + ≈ 6 KB/chunk; a few 10k chunks ⇒ low hundreds of MB + the HNSW index). Start small; it is + read-mostly with a periodic write burst. +- **Extension `pgvector` (≥ 0.7 for `halfvec`)**: must be installed/allow-listed on the + instance. Managed-Postgres offerings differ: + - Cloud SQL / RDS / Azure Flexible: enable `vector` from the supported-extensions list, then + `CREATE EXTENSION vector;` (the SDK's idempotent migration does this — but the extension + must be *permitted* first). + - Self-managed: install the `pgvector` package into the image/host. +- **Terraform** (per the platform's `kbc-stacks` conventions): + - a `postgresql` instance/database + a least-privilege role for the MCP (read: `SELECT` on + `doc`/`doc_chunk`/`index_meta`) and a separate role for the builder (read/write + DDL for + migrations). + - enable the `vector` extension flag on the instance (provider-specific: e.g. Cloud SQL + `database_flags`/enabled-extensions, or an `apt`/image step for self-managed). + - output `DATABASE_URL` (or split host/port/db/user/password) into the MCP + builder secrets. + - network/SG rules so both workloads can reach the DB; TLS required. +- **Migrations**: `@keboola/docs-search` ships idempotent `migrations/*.sql` (`migrate(pool)`), + applied by the **builder** at build start (it owns DDL). The MCP role needs no DDL. + +## 3. The index build job (cron) + +- **What it runs**: the source connectors (git clone of the help + dev-portal repos, fetch the + component catalog, parse frontmatter, derive canonical URLs) → `runIndexBuild(pool, {sources, + embedder, gates})`. +- **Incremental**: content-hash diff via `index_manifest` — only changed docs are re-embedded + (embedding is the cost/latency driver). Unchanged docs are untouched. +- **Transactional + gated**: the build COMMITs only if validation gates pass + (`minDocs`/`minComponents`/`maxDocDropPct`) — otherwise ROLLBACK, leaving the previous index + intact. A bad docs push or a connector regression can never publish an empty/broken index. +- **Schedule**: cron (e.g. hourly/daily depending on docs churn) as a k8s `CronJob` (or the + platform's scheduler). Single-flight: overlapping runs must not race — use a build advisory + lock or `concurrencyPolicy: Forbid`. +- **Embedder**: the builder needs the embedding provider credential (deployment secret). Same + model/dim as the MCP's query-time embedder (must match — `index_meta.embedding_model`/`dim` + record it; the MCP can assert compatibility on startup). +- **Observability**: emit build duration, docs added/changed/removed, embed calls, gate + outcome, and `index_meta.last_success_at`. Alert if `last_success_at` age exceeds an SLO. + +## 4. Index lifecycle & the MCP's read contract + +- **Freshness**: MCP reads whatever is committed; staleness bounded by the cron cadence. No + runtime coupling to the builder. +- **Availability probe**: on server build the MCP checks the index is reachable + populated + (`index_meta.last_success_at IS NOT NULL`, `doc_count > 0`). Result drives tool visibility + (RFC point 5): index healthy → docs tools offered; else filtered out + calls denied with a + clear message. The rest of the MCP is unaffected. +- **Model-compatibility guard**: if the MCP's query embedder model/dim ≠ `index_meta`'s, the + docs tools gate off (mismatched vectors would return garbage) and log a loud warning. +- **Connection management**: one `pg.Pool` per MCP pod (small max; the docs tools are + low-QPS), created at startup, closed on shutdown. + +## 5. Failure modes + +| Failure | Effect | Mitigation | +| --- | --- | --- | +| Builder fails / gates fail | No new index published | ROLLBACK keeps last good index; MCP unaffected; alert on `last_success_at` age | +| Postgres unreachable from MCP | Docs tools unavailable | Availability probe gates the 3 tools off; other tools keep working | +| Embedder (query) down | `docs_query`/`find_component_id` error | Tool-level error; `getComponentDoc` still works (no embedder); optional `search`-only fallback for `docs_query` | +| Extension not enabled | Migration/build fails | Terraform enables `vector`; builder migration is the canary | +| Model/dim drift builder↔MCP | Wrong results | Startup compatibility guard gates docs tools off | +| Index empty on first rollout | Docs tools gated off until first successful build | Ship builder + run once before enabling docs tools in prod | + +## 6. Rollout sequence + +1. **Land `@keboola/docs-search`** (#6672) + the source connectors (its follow-up). +2. **Provision Postgres + `pgvector`** per stack via Terraform; wire `DATABASE_URL` secrets to + both the MCP and the builder. +3. **Deploy the builder + cron**; run once; verify `index_meta` populated and gates pass. +4. **Ship the MCP docs-search integration** behind the availability probe (RFC) — with the + AI-service path still present, feature-flagged, so we can A/B the outputs. +5. **Parity-verify** the three tools against the AI service on a fixed query set. +6. **Cut over**: enable docs-search, **remove the AI-service integration** (RFC point 2). +7. **Decommission** the legacy AI docs service once no MCP/stack references it. + +## 7. Security & cost notes + +- **Index access is infrastructure**, not a user grant: the MCP holds `DATABASE_URL`; clients + never touch Postgres. Docs are global/non-tenant, so no per-project authorization is needed + (see RFC §"the access gate"). Confirm with platform security before cutover. +- **Least privilege**: distinct DB roles for MCP (read) vs builder (read/write/DDL). +- **Embedder credential** is a deployment secret (MCP query-time + builder), rotated like any + other; it is the only external call left on the docs path. +- **Cost**: embeddings are computed only for changed docs (incremental) at build time, and once + per query at read time — far cheaper than a bespoke hosted retrieval service; Postgres is + read-mostly and small. + +## 8. Embedding model & dimensions + +pgvector (like LanceDB) is only the vector **store** — it does not generate embeddings. A +model must turn text → vector; that model is the one external piece. The MCP + the seeder +select it via `DOCS_EMBEDDER_MODEL`, and **build-time and query-time must use the same model +and dimension** (mismatched vectors return garbage): + +| `DOCS_EMBEDDER_MODEL` | Embedder | Infra to provision | Dim | +| --- | --- | --- | --- | +| `stub` | Deterministic offline hash (CI/tests only — **not semantic**) | none | 3072 (default) | +| `local` | In-process HuggingFace model via transformers.js (ONNX, CPU) — no service, no key | **none** (optional `@huggingface/transformers` dep; model weights cached on first use) | model-native (e.g. 384 `all-MiniLM-L6-v2`, 1024 `bge-large`) | +| *(remote model name)* | OpenAI/Azure-compatible endpoint (`DOCS_EMBEDDER_ENDPOINT`/`API_KEY`) — e.g. reuse kai-bot's `embeddings` deployment | reuse existing / provision one | 3072 `text-embedding-3-large` (or 1536 small) | + +- **Dimension is configurable** end-to-end: `DOCS_EMBEDDER_DIM` drives the embedder and the + `halfvec(N)` column (`halfvec` indexes up to 4000 dims, so 384/768/1024/1536/3072 all work; + under 2000 a plain `vector(N)` is also possible). The local seeder recreates the tables if + the dim changes. +- **3072 vs 1024**: more dims = marginally better retrieval + ~3× storage/search cost. On a + bounded docs corpus 1024 is within a couple % of 3072 — a good default for local models. +- **To avoid provisioning any embedding infra**, use `local` (only Postgres is provisioned) or + reuse kai-bot's already-provisioned Azure `embeddings` deployment. Provisioning a *new* + dedicated deployment is the only option that adds infra. diff --git a/feature_spec/integration-tests/RFC.md b/feature_spec/integration-tests/RFC.md new file mode 100644 index 000000000..7ded0c5f8 --- /dev/null +++ b/feature_spec/integration-tests/RFC.md @@ -0,0 +1,133 @@ +# RFC: TypeScript Integration Tests — per-case redis-leased project pool + +Linear: PSGO-268 (TypeScript rewrite) · branch +`martinvasko-psgo-268-rewrite-keboola-mcp-server-from-python-to-typescript-11` + +## Problem + +The Python integration suite acquires **one** shared test project for an entire CI run +(`integtests/conftest.py` → `ProjectPool`, session-scoped) and serializes concurrent runners +with a **Storage-API branch-metadata** lock (`integtests/project_lock.py`). Two problems: + +1. **Coarse acquisition.** A whole CI run holds a single project for its full duration, even + though most test cases are independent. With a pool of N projects, at most N runs can + proceed; a run that only needs a project for one case still blocks one for minutes. +2. **Inconsistent locking.** The branch-metadata lock works but is intricate + (write-and-verify window, oldest-timestamp-wins, stale-TTL cleanup, anti-collision sleeps) + and the team already has a battle-tested redis lease (`keboola/go-utils` + `pkg/testproject`) used across the go monorepo that "works perfectly". Reimplementing a + second bespoke protocol in TS is wasted risk. + +The TS rewrite has **no integration tests yet**. Rather than port the branch-metadata lock, +this RFC adopts the go-utils model: a **redis-leased pool**, acquired **per test case**, with +the same `projects.json` layout and CI export mechanism the go monorepo uses. + +## Required Behavior + +| Concern | Behavior | +| --- | --- | +| Pool source | A `projects.json` array of project definitions (host, project id, token, backend, stagingStorage), the same schema as [`go-monorepo/build/ci/projects.json`](https://github.com/keboola/go-monorepo/blob/main/build/ci/projects.json). Path from `TEST_KBC_PROJECTS_FILE` (absolute). | +| CI secret injection | A composite action mirroring [`export-kbc-projects`](https://github.com/keboola/go-monorepo/blob/main/.github/actions/export-kbc-projects/action.yml): `envsubst` the `$TEST_KBC_PROJECT__TOKEN` placeholders in `.github/ci/projects.json` from `TEST_KBC_PROJECT_*` secrets into a runtime `projects.json`. | +| Acquisition granularity | **Per test case**, not per run. A test calls `getTestProject(...)`; the lease is released automatically at the end of that test. | +| Mutual exclusion | A **redis lease** per `(host, projectId)` key (port of go-utils `redislocker.go`): `SET key token NX PX ttl`; auto-extend at `ttl/4`; release via compare-and-delete. TTL 2 min. | +| Local fallback | When no redis is configured, fall back to a **host-local file lock** (port of `fslocker.go`) so the suite runs on a developer machine without redis. | +| Exhaustion policy | When every compatible project is currently leased, **do not error** — sleep briefly and retry the whole pool **forever** (matching `go-utils` `GetTestProject`). The only failure is "no *compatible* project exists at all" (e.g. asked for BigQuery, pool has none). | +| Compatibility filter | Optional `{ backend }` selector so a test can require `snowflake` / `bigquery`; only matching definitions are considered. | +| Project hygiene | On acquisition, the project is reset to a known-clean state (`cleanProject`, port of the Python `_purge_project` guard + wipe) before the test runs; a dedicated-project guard refuses to wipe a project holding non-`*.c-test*` buckets. | +| Parallelism | Vitest runs files in parallel workers; each worker leases independently. Two cases on different workers may hold two different projects at once; two cases never share one project. | + +### Environment contract + +| Variable | Meaning | Required | +| --- | --- | --- | +| `TEST_KBC_PROJECTS_FILE` | Absolute path to the generated `projects.json` | yes | +| `TEST_MCP_PROJECTS_LOCK_HOST` | redis URL (`redis://host:port`, `+tls` suffix for TLS) | CI only | +| `TEST_MCP_PROJECTS_LOCK_PASSWORD` | redis password | CI only | +| `TEST_MCP_PROJECTS_LOCK_DIR_NAME` | dir for the fs-locker fallback | local optional | + +No per-token env arrays (`INTEGTEST_STORAGE_TOKENS`) anymore — the pool is the JSON file. + +## Resolution Strategy + +New harness under `integtests/testproject/` (a faithful TS port of the go-utils package + +the go-monorepo `internal/utils/testproject/project.go` wrapper): + +| File | Responsibility | Ported from | +| --- | --- | --- | +| `types.ts` | `ProjectDefinition` (zod-validated), `LockedProject`, `Backend`/`StagingStorage` enums | `testproject.go` `Definition` | +| `projects.ts` | Load + parse + validate `projects.json` from `TEST_KBC_PROJECTS_FILE`; process-singleton pool | `getProjects`/`GetProjectsFrom` | +| `redisLocker.ts` | ioredis lease: `SET NX PX` obtain, Lua compare-and-`pexpire` refresh, Lua compare-and-`del` release; background auto-extend timer at `TTL/4` | `redislocker.go` | +| `fsLocker.ts` | `proper-lockfile`/`O_CREAT|O_EXCL` host-local lock | `fslocker.go` | +| `pool.ts` | `getTestProject({backend?})`: loop compatible defs, `tryLock` each; if none free sleep 100 ms and retry forever; throw only when no compatible def exists | `ProjectsPool.GetTestProject` | +| `clean.ts` | `cleanProject(endpoint)`: dedicated-project guard + wipe buckets / configs / workspaces / MCP branch metadata via `@keboola/api-client` | Python `_purge_project` | +| `fixture.ts` | `getTestProjectForTest()` — acquires, registers `onTestFinished(release)`, returns `{ config, storageApiUrl, storageApiToken, backend, cleanup }` | `GetTestProjectForTest` | + +Test harness: + +- `vitest.integ.config.ts` — separate config (`include: integtests/**/*.test.ts`, long + `testTimeout`, `fileParallelism: true`, `globalSetup` validates the pool loads once). +- npm scripts: `test:integ` (run), `test:integ:watch`. +- Each ported test calls `getTestProjectForTest()` in the body (or a small per-test fixture), + builds the in-memory MCP client against that project's `Config`, exercises tools, asserts. +- Reuse the existing unit-test harness shape (`InMemoryTransport` + MCP `Client`) — the only + difference is the `Config` comes from a leased project and the calls hit the real stack. + +Redis semantics (port of `redislocker.go`, no extra lock library — raw ioredis + Lua so the +compare-and-swap matches `bsm/redislock` exactly): + +``` +obtain : SET - NX PX -> ok ? leased : busy +refresh: if GET key == token then PEXPIRE key (Lua, every ttl/4) +release: if GET key == token then DEL key (Lua) +``` + +The auto-extend timer holds the lease for the (unknown, possibly long) duration of a single +test and is cleared on release, so a crashed worker's lease expires after at most `TTL`. + +CI: a dedicated `integration_tests` job (separate from `ci.yml`) runs +`export-kbc-projects` → `npm run test:integ` with redis service + `TEST_KBC_PROJECT_*` +secrets. Sketch lives in this folder's `ci-job.yml` and is wired into the workflow at +implementation time. + +### Non-obvious trade-offs + +- **Per-case vs per-file acquisition.** Per *case* maximizes pool utilization but multiplies + lease churn and `cleanProject` cost (a wipe per case). We default to per-case (as + requested) but expose a per-file helper for suites whose cases share expensive fixtures + (e.g. the storage suite that seeds buckets/tables once). The lease helper is the same; only + the scope of `onTestFinished` vs `beforeAll/afterAll` differs. +- **Infinite retry.** Matches go-utils and is correct for a bounded CI pool, but a + genuinely deadlocked pool would hang until the job timeout rather than failing fast. We + rely on the job-level timeout + the `TTL`-bounded lease expiry as the backstop, and log the + wait every few seconds so a stuck pool is visible. +- **Redis as the single point of coordination.** If redis is down, CI integtests can't + coordinate. The fs-locker fallback is host-local only (no cross-runner safety), so CI + always uses redis; the fallback exists purely for local single-host runs. + +## Scope + +In scope: +- The `integtests/testproject/` harness, `projects.json` layout, `export-kbc-projects` action, + `vitest.integ.config.ts`, npm scripts, `integtests/README.md`, and the CI job. +- Porting the Python integration test *cases* (`integtests/**`) to vitest on top of the new + harness — done incrementally per module after this RFC is agreed. + +Out of scope: +- Changing production server behavior. Integtests exercise the shipped tools as-is. +- Cross-host fs-locking (redis is the cross-runner mechanism). +- Removing the Python integtests until the TS ports reach parity (kept as the reference). +- The unit-test suite (already complete: 383 tests). + +## Testing / Verification + +- **Harness unit tests** (run in the normal `vitest` suite, no redis/projects needed): + `projects.ts` parsing/validation, `pool.ts` selection + infinite-retry (with a fake + in-memory locker), `redisLocker.ts` against `ioredis-mock`, the dedicated-project guard in + `clean.ts`. These give us confidence without a live stack. +- **Live integration run** (CI `integration_tests` job): redis service + `TEST_KBC_PROJECT_*` + secrets → `export-kbc-projects` → `npm run test:integ`. Verifies real leasing, parallel + workers never collide on a project, and the ported tool tests pass against a real stack. +- **Parity check:** each ported module is cross-checked against its Python counterpart in + `integtests/` (same scenarios, same assertions) before the Python file is removed. +- **Manual local run:** developer sets `TEST_KBC_PROJECTS_FILE` to a one-project file (no + redis) and runs `npm run test:integ` — exercises the fs-locker path. diff --git a/feature_spec/mcp-typescript-rewrite/CICD-RUNBOOK.md b/feature_spec/mcp-typescript-rewrite/CICD-RUNBOOK.md new file mode 100644 index 000000000..1f0ebc680 --- /dev/null +++ b/feature_spec/mcp-typescript-rewrite/CICD-RUNBOOK.md @@ -0,0 +1,110 @@ +# Keboola MCP Server (TS) — Config, Distribution & CI/CD Runbook + +Operational runbook for running, publishing, and CI-testing the rewritten +`@keboola/mcp-server`. **No credential values here** — only what to set and where. +Repo stays standalone (`keboola/keboola-mcp-server`); CI workflows are adapted in place. + +--- + +## 1. Runtime config (the server itself) + +Resolution precedence (highest first): **CLI arg → `KBC_*` env var → `X-*` HTTP header** +(same model as the Python `Config`). + +| Purpose | Env var | CLI flag | HTTP header | Required | +| ----------------------------------------------------------- | ------------------------------------------------- | -------------------- | ------------------------------- | ----------------- | +| Storage API URL (`https://connection..keboola.com`) | `KBC_STORAGE_API_URL` | `--api-url` | `X-StorageApiUrl` | yes | +| Storage API token | `KBC_STORAGE_TOKEN` | `--storage-token` | `X-StorageApiToken` | yes (token mode) | +| Branch id (`null`/`default`/`production` ⇒ prod) | `KBC_BRANCH_ID` | — | `X-Branch-Id` | no | +| Workspace schema (SQL) | `KBC_WORKSPACE_SCHEMA` | `--workspace-schema` | `X-Workspace-Schema` | for SQL tools | +| OAuth bearer (HTTP transports) | — | — | `Authorization: Bearer ` | OAuth mode | +| OAuth client id / secret | `KBC_OAUTH_CLIENT_ID` / `KBC_OAUTH_CLIENT_SECRET` | — | — | OAuth server mode | +| OAuth server URL / scope | `KBC_OAUTH_SERVER_URL` / `KBC_OAUTH_SCOPE` | — | — | OAuth server mode | +| MCP server public URL | `KBC_MCP_SERVER_URL` | — | — | OAuth mode | +| JWT signing key | `KBC_JWT_SECRET` | — | — | OAuth mode | +| Read-only mode | — | — | `X-Read-Only-Mode` | no | +| Conversation id | — | — | `X-Conversation-Id` | no | +| Log level | — | `--log-level` | — | no | +| App env / version (telemetry) | `APP_ENV` / `APP_VERSION` | — | — | no | +| Datadog tracing | `DD_*` (dd-trace) | — | — | container only | + +Rules (port from Python): stdio transport **rejects** OAuth config (HTTP only); secret +fields redacted in logs; never commit credentials (`.env` gitignored); local dev via +`--env-file=.env`. + +--- + +## 2. npm distribution (`npx @keboola/mcp-server`) + +Published from this repo (standalone). `package.json` must have: `"type":"module"`, +`bin` entry, `files:["dist"]`, `"publishConfig":{"access":"public"}`, ESM+CJS exports +(model `@keboola/api-client`). Build with tsup before publish. + +Required CI repo secret: + +- **`NPM_TOKEN`** — npm automation token with publish rights to the `@keboola` scope. + +Publish trigger: on `v*` release tags (wire into `release.yml` or a dedicated +`npm-publish.yml`). Verify after first publish: `npx @keboola/mcp-server` (stdio), +`npm i -g`, `npm i @keboola/mcp-server` as a lib. + +--- + +## 3. Docker image distribution + +Image `keboola/mcp-server` on Docker Hub. Tag→image mapping kept **identical** to the +current Python `release.yml` so `kbc-stacks` routing is unchanged: + +| Git tag | Image tag | Stack | +| --------------------------- | ------------------------------- | ------------ | +| `vX.Y.Z` | `production-` + `latest` | production | +| `agent-vX.Y.Z` | `production-` (agent helm) | production | +| `canary-orion-vX.Y.Z-dev.N` | `canary-orion-` | canary-orion | +| `dev-vX.Y.Z-dev.N` | `dev-` | testing | + +Required repo secrets (already present for the Python build): + +- **`DOCKERHUB_PUSH_USER`** + **`DOCKERHUB_PUSH_TOKEN`**. + +Dockerfile: Node 24 multi-stage, non-root user, `dd-trace` preloaded, +`ENTRYPOINT ["node","dist/index.js","--transport","streamable-http"]`. + +--- + +## 4. CI/CD secret summary (set in `keboola/keboola-mcp-server`) + +| Repo secret / var | Used by | Purpose | Status | +| ------------------------------------------------------------ | ------------ | ----------------------------- | -------------------- | +| `DOCKERHUB_PUSH_USER` / `DOCKERHUB_PUSH_TOKEN` | release.yml | Docker Hub push | already set (Python) | +| `NPM_TOKEN` | npm publish | publish `@keboola/mcp-server` | **add** | +| `CODECOV_TOKEN` | ci.yml | coverage upload (optional) | already set | +| KaiBench: model/credential secrets, kai-assistant image pull | kaibench.yml | eval run | already set (Python) | + +KaiBench: the workflow spins up MCP server (from branch) + kai-assistant (prebuilt image) + +- Postgres + Redis and runs the eval suite on production `v*` tags. Reuse the existing + `kaibench.yml`; only the MCP server build step changes (Node instead of uv/Python). + +--- + +## 5. Coordinated `@keboola/api-client` work (cross-repo, in `keboola/ui`) + +Endpoints the MCP server needs that api-client v4 does not yet expose — add as new +subpaths in `ui/packages/api-client`, publish, then bump the dependency here: + +- **scheduler** (flow activation/scheduling) +- **AI service**: `docs_query`, semantic context/schema/search, global `search`, + config examples, component finder, description suggestions + +Until published, ship thin local clients in `src/clients/` and swap to api-client later. + +--- + +## 6. First-run checklist + +- [ ] `NPM_TOKEN` added to repo secrets; `@keboola` scope publish dry-run OK. +- [ ] `DOCKERHUB_PUSH_*` confirmed (carried over from Python build). +- [ ] KaiBench services/credentials confirmed in `kaibench.yml`. +- [ ] Local `.env` (gitignored) with `KBC_STORAGE_API_URL` + `KBC_STORAGE_TOKEN`. +- [ ] Smoke: `npx @keboola/mcp-server` lists 39 tools over stdio. +- [ ] api-client gap endpoints tracked (scheduler, AI service) — local clients vs. published. diff --git a/feature_spec/mcp-typescript-rewrite/PLAN.md b/feature_spec/mcp-typescript-rewrite/PLAN.md new file mode 100644 index 000000000..1d78976bd --- /dev/null +++ b/feature_spec/mcp-typescript-rewrite/PLAN.md @@ -0,0 +1,204 @@ +# Plan: Rewrite Keboola MCP Server (Python → TypeScript) + +**Linear:** PSGO-268 · branch `martinvasko-psgo-268-rewrite-keboola-mcp-server-from-python-to-typescript-11` + +**Goal:** Replace the Python (FastMCP, v1.72.8) implementation **in this same standalone +public repo** with a TypeScript implementation at **1:1 functional parity**. Ship as a +Docker container AND publish to npm so it installs via `npx @keboola/mcp-server` / +`npm i -g`. Migrate unit tests to vitest and iterate until green. Integtests last. +Delivered as a **draft branch + draft PR**. + +> Parity source of truth: the current Python tree (before replacement), `TOOLS.md` +> (39 tools), and the unit test suite (`tests/`, 33 files). + +--- + +## 0. Decisions (confirmed) + +- **Standalone repo.** Rewrite in place; do NOT fold into the `ui` monorepo. Later, + consider extracting reusable packages _out of_ MCP for `ui` to consume. +- **Reuse `@keboola/api-client`** as a **published npm dependency** (v4) for the client + layer: `storage`, `queue` (jobs), `queryService`, `oauth`, `encryption`, `metastore`, + `syncActions`, `dataScience`, `management`. +- **MCP protocol**: `@modelcontextprotocol/sdk` (replaces FastMCP) — stdio + + streamable-HTTP transports, zod tool schemas. +- **HTTP server**: Hono + `@hono/node-server` (model: `ui/apps/kai-agent`). +- **API gaps** (scheduler, AI service: `docs_query` / semantic / global `search`): + **add to `@keboola/api-client`** in the `ui` repo (coordinated cross-repo workstream), + publish, then consume here. Track as a dependency of the relevant tool phases. +- **MCP server only** (In Platform Agent variant deferred), but keep this repo's release + tag scheme (`v*`, `agent-v*`, `canary-orion-*`, `dev-*`) and CI workflows, adapted to TS. +- **Models**: pydantic → **zod v4**. +- **SQL**: runs via the Query Service **HTTP API** — no DB drivers. + +--- + +## 1. What we reuse vs. rewrite (laziness budget) + +| Python component | TS strategy | +| ------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| `fastmcp` / `mcp`, stdio + streamable-http | `@modelcontextprotocol/sdk`. No reimplementation. | +| `clients/` (storage, jobs_queue, query, oauth, encryption, metastore, sync_actions, data_science) | `@keboola/api-client` subpaths. Thin adapters only. | +| `clients/scheduler.py`, `clients/ai_service.py`, semantic + global search + docs | Gap → add to `@keboola/api-client`, then consume. | +| `clients/base.py` (httpx + retries) | api-client `fetchClient`/middlewares. | +| pydantic models | zod v4. | +| sqlglot (identifier quoting / dialect) | port the small quoting helpers; `node-sql-parser` only if strictly needed. | +| Starlette/uvicorn | Hono + `@hono/node-server`. | +| `cli.py` argparse | small arg+env parser; `bin` for npx. | +| Dockerfile (uv/python) | Node 24 multi-stage. | +| pytest (parametrize, fixtures) | vitest 4 (`test.each`) + msw. | +| `generate_tool_docs.py` | `gen:tools-docs` script; keep `check-tools-docs` CI gate. | + +**Net:** client layer (~3.4k LOC) and MCP protocol layer become dependencies. The real +work is **tools (15.5k LOC) + models + tests**. + +--- + +## 2. Target repo layout (replacing Python in place) + +``` +package.json # name @keboola/mcp-server, bin, type module, exports, tsup, vitest +tsconfig.json +tsup.config.ts # ESM+CJS + bin; model ui/apps/kai-agent/tsup.config.ts +oxlint.config.ts # match monorepo lint (oxlint/oxfmt) OR keep eslint — see §9 +vitest.config.ts +Dockerfile # node:24 multi-stage; replaces the Python one +.github/workflows/ # ci.yml, release.yml, kaibench.yml — adapted to Node/TS +src/ + index.ts # bin entry: parse args/env -> Config -> start transport + config.ts # Config (env KBC_*, X-* headers, CLI) — port config.py + server.ts # build McpServer, register tools/prompts/resources, lifespan + transports/{stdio.ts,http.ts} # http.ts = Hono app + routes + mcp/{toolFiltering.ts,authorization.ts,errors.ts} + oauth.ts # SimpleOAuthProvider (oauth.py) + preview.ts # /preview/configuration (preview.py) + workspace.ts # Workspace + SQL dialect quoting (workspace.py) + clients/ # thin adapters over @keboola/api-client (+ gap clients until api-client ships) + tools/ # one module per Python tools/ submodule (see §4) + models/ # zod schemas + prompts/ resources/ +__tests__/ # ported unit tests, mirror Python tests/ +README.md +TOOLS.md # regenerated +feature_spec/mcp-typescript-rewrite/ # this PLAN + SECRETS doc +``` + +HTTP routes on the Hono app (parity with cli.py/server.py): `/mcp` (streamable-http), +`/` (info), `/health-check`, `/preview/configuration` (POST), `/oauth/callback` (GET). +Default transport `stdio`; `--transport streamable-http` (alias `http-compat`) for server. + +--- + +## 3. Core infra (Phase 1) + +1. **Config** — port `config.py` field set and resolution (CLI → `KBC_*` env → `X-*` + header), alias/normalize logic, URL amendment, branch_id "production/default/none"→null, + secret redaction. +2. **Transports** via SDK: `StdioServerTransport`; stateless streamable-HTTP at `/mcp`. + Reject OAuth config on stdio (cli.py parity). +3. **Per-request config**: Hono middleware reads `X-*` + `Authorization: Bearer` per + request, layered over base Config via AsyncLocalStorage context. +4. **Logging**: pino (JSON), `--log-level`. **Errors**: 400 for validation/value/JSON, + 500 otherwise; debug includes stack. **dd-trace** for the container. + +**Async stance:** native async. + +- Await-to-completion: `query_data` (submit query job → poll Query Service), reads, + `run_sync_action`. Parallelize independent fetches with `Promise.all`. +- Fire-and-return-progress: `run_job`, `deploy_data_app` — return job/task id + status. + +--- + +## 4. Tools (Phase 4) — all 39 + +Each = zod input schema + handler + `server.registerTool` (preserve names, descriptions, +read-only annotations exactly — they drive TOOLS.md and tool filtering). + +| Module | Tools | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| components | get_components, find_component_id, get_configs, get_config_examples, create_config, add_config_row, update_config, update_config_row, create_sql_transformation, update_sql_transformation, run_sync_action | +| flow | get_flows, get_flow_schema, get_flow_examples, create_flow, create_conditional_flow, update_flow, modify_flow | +| storage | get_buckets, get_tables, update_descriptions | +| jobs | get_jobs, run_job | +| sql | query_data | +| project | get_project_info, update_project_description | +| search | search | +| semantic | get_semantic_context, get_semantic_schema, search_semantic_context, validate_semantic_query | +| oauth | create_oauth_url | +| data_apps | get_data_apps, deploy_data_app, modify_streamlit_data_app, modify_python_js_data_app, create_python_js_data_app_git_credential, delete_python_js_data_app_draft | +| doc | docs_query | + +Cross-cutting (Phase 5): tool filtering (`is_read_only_tool`/`is_semantic_tool`, +`X-Read-Only-Mode`, semantic gating), authorization + `/preview/configuration` +(AI-3438 hardening — port checks exactly), OAuth provider, prompts, resources, +TOOLS.md generator + `check-tools-docs` gate. + +--- + +## 5. Models (Phase 3) + +Port pydantic → zod across `tools/*/model.py`, `*/api_models.py`, `search_models.py`, +`flow/scheduler_model.py`, `semantic/model.py`, `components/model.py`. Reuse api-client +`*/types` where API shapes already match. Confirm whether `toon-format` output encoding +must be byte-preserved (§9 Q1). + +--- + +## 6. Tests (Phase 6) + +Port `tests/` (33 files) → `__tests__/` mirroring structure. parametrize → `test.each`; +`conftest.py` → vitest setup + factories. **HTTP mocking via msw** (don't over-mock +client internals). Iterate module-by-module to green. `integtests/` ported **last**. + +--- + +## 7. CI/CD (Phase 7) — adapt existing workflows in place + +- **`ci.yml`**: replace Python matrix with Node 24; run `vitest`, `tsc`, lint, `build`, + and the TOOLS.md check. Keep Codecov. +- **`release.yml`**: keep the tag→image mapping verbatim (`v*`/`agent-v*` → + `production-`+`latest`; `canary-orion-*`; `dev-*`) so kbc-stacks routing is + unchanged. Build the Node Dockerfile. Secrets: `DOCKERHUB_PUSH_USER`, + `DOCKERHUB_PUSH_TOKEN`. +- **npm publish**: add a publish step (npm token) — publish `@keboola/mcp-server` on + `v*` tags (or via release). `bin` + `files:[dist]` + `publishConfig.access:public`. +- **`kaibench.yml`**: keep the service wiring (MCP server from branch + kai-assistant + + postgres + redis); point it at the Node build. + +--- + +## 8. Phasing (draft PR, iterate to green) + +0. Branch off `main`; scaffold TS skeleton alongside Python (keep Python running until + parity), open **draft PR** early. +1. Core infra; smoke: `tools/list` over stdio returns 39 tools with correct schemas. +2. Client adapters (+ track api-client gap additions). +3. Models for first module → 4. Tools module-by-module, porting each test file with it + (storage → jobs → sql → components → flow → project → search → semantic → oauth → + data_apps → doc). +4. Cross-cutting (filtering, auth, preview, oauth, prompts, resources, TOOLS.md gen). +5. Unit-test parity green; TOOLS.md regenerated + gated; **remove Python sources**. +6. CI/CD adapted; Docker + npm + KaiBench green. +7. Integtests ported. 9. README + SECRETS doc finalized. + +Commits start with `PSGO-268:`; PR references the issue. + +--- + +## 9. Open questions + +1. **Output encoding**: Python uses `toon-format` for some tool outputs. Preserve + byte-for-byte or is JSON acceptable for parity? +2. **Lint stack**: match the `ui` monorepo (oxlint/oxfmt) or keep an eslint/prettier setup + standalone? (Affects CI + dev ergonomics.) +3. **api-client gap timing**: can we land scheduler/AI-service endpoints in + `@keboola/api-client` in time, or ship temporary local clients and swap later? + +--- + +## 10. Out of scope / deferred + +- In Platform Agent variant (follow-up; tags kept reserved). +- Integtests ported last. +- DB drivers (SQL is HTTP via Query Service). +- No reimplementation of MCP protocol or HTTP client. diff --git a/integtests/README.md b/integtests/README.md index ff73d4691..368c3b044 100644 --- a/integtests/README.md +++ b/integtests/README.md @@ -1,332 +1,133 @@ # Integration Tests -Each test session creates and deletes real objects (buckets, tables, configurations) in a -Keboola project. The `project_lock` fixture ensures that only one session owns a given -project at a time, so concurrent CI jobs or overlapping local runs do not corrupt each -other's data. +Integration tests run the shipped MCP tools against **real** Keboola projects. Each test case +leases a project from a shared pool, runs against it, and releases it — so many CI runners +(and many test cases) can proceed in parallel without corrupting each other's data. -> **Important**: The integration tests are **not** tied to specific projects. Any clean -> Keboola project with a workspace can be used. For local development you only need **two** -> projects in the pool (one Snowflake, one BigQuery) plus one dedicated project for branch -> storage tests (with the `storage-branches` feature). +Design rationale and the full protocol are in +[`feature_spec/integration-tests/RFC.md`](../feature_spec/integration-tests/RFC.md). This file +is the operational guide. ---- - -## 1. Setting Up Your Own Test Environment - -### Do NOT reuse the CI projects - -The CI pool projects are shared across all CI runners. If you point your local `.env` at -them, you will interfere with CI runs: the pool lock mechanism will block or — worse — your -local run may acquire a CI project, crash mid-way, and leave it in a dirty state that -causes random CI failures. - -**Always create your own dedicated test projects for local development.** - -### Creating test projects - -You need: - -1. **Two pool projects** — one on Snowflake backend, one on BigQuery (or two of the same - backend if you only need one). These must be **completely empty** (no buckets, no tables, - no configurations) when a test session starts. The lock mechanism cleans up after each - session, but if a session crashes, leftovers may remain and cause the next run to fail. +> **Status:** the harness (`integtests/testproject/`) and pool/CI wiring are in place. The +> individual test *cases* are being ported from the Python suite (`integtests/*.py`, kept as +> the parity reference) module by module. -2. **One storage-branches project** — a project **with** the `storage-branches` feature, - used by `test_storage_branches.py`. - -The test session creates its own read-only workspace in the locked pool project and deletes -it on teardown, so you do **not** need to pre-create a workspace — the project token alone is -enough. - -### `.env` file - -Create a `.env` file in the project root: +--- -```dotenv -# Required — pool projects -INTEGTEST_POOL_STORAGE_API_URL=https://connection.europe-west3.gcp.keboola.com -INTEGTEST_STORAGE_TOKENS= +## 1. The project pool (`projects.json`) -# Required — branch storage tests (dedicated project, not in pool) -INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES= +The pool is a JSON **array** of project definitions — the same schema the go monorepo uses +([`go-monorepo/build/ci/projects.json`](https://github.com/keboola/go-monorepo/blob/main/build/ci/projects.json)): -# Optional — second project for multi-client tests -INTEGTEST_STORAGE_TOKEN_PRJ2= -INTEGTEST_WORKSPACE_SCHEMA_PRJ2= +```json +[ + { "host": "connection.keboola.com", "project": 1234, "token": "", "backend": "snowflake", "stagingStorage": "s3" }, + { "host": "connection.keboola.com", "project": 1235, "token": "", "backend": "bigquery", "stagingStorage": "gcs" } +] ``` -### Running the tests +The committed `.github/ci/projects.json` is a **template** whose tokens are `$TEST_KBC_PROJECT__TOKEN` +placeholders. In CI the `export-kbc-projects` action substitutes them from `TEST_KBC_PROJECT_*` +secrets into a runtime `projects.json`. -```bash -source /bin/activate -pytest integtests/ -v --log-cli-level=INFO -``` +The harness loads the pool once per process from the absolute path in **`TEST_KBC_PROJECTS_FILE`**. -`--log-cli-level=INFO` shows lock acquisition and release messages in real time, which is -useful when diagnosing a stall. +### Environment contract -### Avoiding token leaks in test output +| Variable | Meaning | Where | +| --- | --- | --- | +| `TEST_KBC_PROJECTS_FILE` | Absolute path to the generated `projects.json` | always | +| `TEST_MCP_PROJECTS_LOCK_HOST` | redis URL (`redis://host:port`, `rediss://` or `+tls` for TLS) | CI | +| `TEST_MCP_PROJECTS_LOCK_PASSWORD` | redis password | CI | +| `TEST_MCP_PROJECTS_LOCK_DIR_NAME` | dir for the local fs-lock fallback | local (optional) | -**Important for anyone writing new integration tests or fixtures.** - -Pytest prints fixture parameter values in error tracebacks. If a fixture takes -`storage_api_token: str` as a parameter and the test fails, the raw token appears in -the CI log in plain text. This is a security risk. - -**Bad** — token leaks on failure: -```python -@pytest.fixture -def my_fixture(storage_api_token: str, storage_api_url: str): - # If this fixture or any test using it fails, pytest prints: - # storage_api_token = '2728-6118086-cHM8a1s6c1AR...' - ... -``` - -**Good** — read tokens from env vars inside the function body: -```python -@pytest.fixture -def my_fixture(keboola_project, storage_api_url: str): - # keboola_project dependency ensures env_init has run and set KBC_STORAGE_TOKEN - token = os.environ['KBC_STORAGE_TOKEN'] - # On failure, pytest only shows: keboola_project=ProjectDef(...), storage_api_url='https://...' - ... -``` - -The `env_init` fixture (triggered via `keboola_project`) copies the pool token into -`KBC_STORAGE_TOKEN` and the URL into `STORAGE_API_URL`. Reading from these env vars -inside the function body keeps the token out of pytest's traceback display. - -For tokens not in the pool (e.g. `INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES`), use -`os.getenv(...)` / `os.environ[...]` directly — never accept them as fixture parameters. +When `TEST_MCP_PROJECTS_LOCK_HOST` is set, projects are leased via **redis** (cross-runner +safe). Otherwise the harness falls back to a **host-local file lock** — fine for a single +developer machine, but it provides no cross-runner safety, so CI always uses redis. --- -## 2. How the Pool and Locking Work - -### Overview - -The pool manages a set of test projects. Each test session acquires **one** project from the -pool, runs all tests against it, then releases it. The lock ensures no two sessions use the -same project simultaneously. - -**Key behavior**: -- A session acquires one project and holds it for the entire run -- The project must be **completely clean** (no buckets, configs, tables) at the start — the - `keboola_project` fixture explicitly checks this and fails if anything exists -- After all tests complete, `keboola_project` deletes everything it created -- If a session crashes without cleaning up, the next session that acquires the same project - will detect the stale lock and run `clean_project` to wipe everything before proceeding - -### Lock storage: Keboola branch metadata - -Lock state is stored in the metadata of the default branch of the test project, accessed -via the Storage API. No external service is required. - -Each runner writes up to two metadata keys per acquisition attempt: - -| Key | Value | -|---|---| -| `KBC.integtest.lock.` | JSON: `lock_id`, `acquired_at` (ISO 8601 UTC), `runner_info` | -| `KBC.integtest.lock..released` | ISO 8601 timestamp written on release | - -An entry without a corresponding `.released` key is considered active. - -### Acquisition protocol - -1. **Write a candidate entry** with the current UTC timestamp. -2. **Anti-collision window** — sleep 3 seconds to allow any concurrent writers to finish. -3. **Read all active entries** (those without a `.released` counterpart). -4. **Oldest timestamp wins** — the runner whose `acquired_at` is earliest holds the lock. - Ties are broken by `lock_id` (lexicographic). If this runner's entry wins, it proceeds. - Otherwise it writes its own `.released` key and retries after - `INTEGTEST_LOCK_POLL_INTERVAL_SECONDS`. +## 2. How leasing works -### Stale lock detection and cleanup +A test acquires a project with the per-case helper: -If the winning entry is older than `INTEGTEST_LOCK_TTL_MINUTES` it is considered abandoned -(the previous runner crashed without releasing). The detecting runner: +```ts +import { getTestProjectForTest } from '../testproject/fixture'; -1. Writes `.released` keys for all stale entries. -2. Calls `clean_project`: deletes all buckets (with their tables) and all component - configurations from the project, restoring the clean state the tests require. -3. Re-enters the acquisition protocol from step 1. - -If the integration tests ever take close to -60 minutes to complete, raise `INTEGTEST_LOCK_TTL_MINUTES` to roughly 2x the expected -duration — otherwise a slow-but-healthy runner may have its lock stolen mid-run. - -### Workspaces - -Tests do not rely on a persistent workspace. Each session creates its own read-only -workspace in the locked project (the `workspace_schema` fixture) and deletes it on teardown. -`clean_project` deletes **all** workspaces (along with all buckets and configs), so any -workspace leaked by a crashed run is wiped when the project is next acquired. - -### Pool of projects - -`ProjectPool` holds a list of `ProjectEndpoint` objects. During each acquisition pass it -tries them in randomized order and returns the first one it can lock. If all are busy it -sleeps for `INTEGTEST_LOCK_POLL_INTERVAL_SECONDS` and retries the full list. Raising the -pool size allows multiple CI jobs to run concurrently, each against a different project. - -Before the pool is created, `verify_project_endpoint` calls `GET /v2/storage/tokens/verify` -for every configured token. A revoked or misspelled token therefore causes an immediate -`pytest.fail` rather than a confusing mid-run error. - -### Fixture dependency chain - -`project_lock` is a session-scoped fixture. `storage_api_token` and `workspace_schema` -derive their values from the acquired endpoint, so every fixture that touches the project -is automatically blocked until the lock is held: - -``` -env_file_loaded - └── storage_api_url <-- reads INTEGTEST_POOL_STORAGE_API_URL - └── project_lock <-- lock acquired at session start - ├── storage_api_token - └── workspace_schema - └── env_init - └── keboola_project (creates test buckets, tables, configs) +it('lists buckets', async () => { + const { config, backend } = await getTestProjectForTest(); // or { backend: 'snowflake' } + // build an in-memory MCP client from `config`, call tools, assert… + // lease is released automatically when this test finishes +}); ``` -The lock is released in `project_lock`'s teardown, after every session-scoped fixture that -depends on it has been torn down and the project has been cleaned up by `keboola_project`. - -### Lock tuning - -The defaults work for both local and CI use. Override only if you have a reason to. - -| Variable | Default | Meaning | -|---|---|---| -| `INTEGTEST_LOCK_TTL_MINUTES` | `60` | A lock older than this is considered abandoned and cleaned up by the next runner that detects it | -| `INTEGTEST_LOCK_POLL_INTERVAL_SECONDS` | `30` | How long to wait between retries when all projects in the pool are busy | -| `INTEGTEST_LOCK_MAX_WAIT_MINUTES` | `90` | Raise `TimeoutError` after this many minutes of waiting | +- **Per case, not per run.** The lease is held only for the duration of the calling test + (released via vitest `onTestFinished`). +- **Redis lease** keyed by `-`: `SET NX PX` to obtain, auto-extended at + `TTL/4`, compare-and-delete to release. A crashed worker's lease self-expires after `TTL` + (2 min). Port of go-utils [`redislocker.go`](https://github.com/keboola/go-utils/blob/main/pkg/testproject/redislocker.go). +- **Exhaustion → wait, never error.** If every compatible project is busy, the pool sleeps + briefly and retries the whole pool **forever** (matching go-utils `GetTestProject`). The + only hard error is asking for a backend the pool has none of. +- **Clean on acquire.** Before handing the project over, `cleanProject` resets it (deletes + buckets/configs/workspaces + MCP branch metadata). A guard refuses to wipe a project that + holds any non-`*.c-test*` bucket — protection against a misconfigured pool pointing at a + real project. --- -## 3. Test-Specific Projects +## 3. Running locally -### Branch storage tests (one dedicated project) +Create a one- or two-project `projects.json` for **your own dedicated** test projects (never +the CI pool — you would interfere with CI), then: -The branch storage tests (`test_storage_branches.py`) validate the deference mechanism on -`storage-branches` projects. They use **one dedicated project outside the pool**: - -```dotenv -INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES= -``` - -The variable is required. The tests fail if it is missing or points to a project without -the `storage-branches` feature. - -This project is **not** in the pool and has no lock mechanism. Concurrent access is safe -because production data (`in.c-test_bucket_01` with `test_table_01`) is created -idempotently and each session only manages its own branches (with unique names). - -No workspace schemas are needed — these tests only exercise bucket/table listing, not -`query_data`. - -**Concurrency rules for branch tests**: These projects have no lock mechanism, so any new -tests added to `test_storage_branches.py` must be safe for concurrent execution: -- **Use unique names** for branches (the current tests use a UUID suffix). Never use - hardcoded branch names. -- **Production data must be idempotent** — use `_ensure_bucket` / `_ensure_table` which - create only if not already present. Never delete production data in teardown. -- **Only clean up your own branches** — teardown must only delete the branches created by - the current session (tracked via the `BranchTestProject` dataclass). -- **Do not modify or delete production buckets/tables** — they are shared across sessions. - -### Second project (two-project tests) - -A small number of tests exercise simultaneous access to two separate projects. They need: - -```dotenv -INTEGTEST_STORAGE_TOKEN_PRJ2= -INTEGTEST_WORKSPACE_SCHEMA_PRJ2= +```bash +export TEST_KBC_PROJECTS_FILE="$(pwd)/projects.local.json" +# no redis → host-local fs lock is used automatically +npm run test:integ ``` -These are read directly without locking and can be omitted if you are not running those -tests. No lock is needed because the only test that uses PRJ2 -(`test_http_multiple_clients_with_different_headers`) is strictly read-only against it — -it calls `list_tools`, `list_resources`, and `get_project_info`, and never creates, -modifies, or deletes any object. Any future test that writes to PRJ2 must acquire a lock -for it first. +Add `TEST_MCP_PROJECTS_LOCK_HOST` / `_PASSWORD` only if you want to exercise the redis path +locally (e.g. `docker run -p 6379:6379 redis`). -### Metastore tests - -A subset of tests exercises the Metastore API. The Metastore URL is derived automatically -from `INTEGTEST_POOL_STORAGE_API_URL` by replacing the `connection.` prefix with `metastore.` -(e.g. `https://connection.north-europe.azure.keboola.com` -> `https://metastore.north-europe.azure.keboola.com`). -Authentication reuses the storage API token — no additional environment variables are needed. +Harness logic itself (pool selection, parsing, retry) is covered by ordinary unit tests in +`__tests__/testproject.test.ts` and runs in the normal `npm test` — no projects or redis +needed. --- -## 4. Migrating from the Old Setup - -The pool is configured by `INTEGTEST_STORAGE_TOKENS` (space-separated project tokens). -Earlier single-value (`INTEGTEST_STORAGE_TOKEN`) and per-project workspace-schema -(`INTEGTEST_WORKSPACE_SCHEMA` / `INTEGTEST_WORKSPACE_SCHEMAS`) variables are no longer read — -each session now creates its own read-only workspace, so no workspace schema is configured -for the pool. +## 4. Security: don't leak tokens in test output -`INTEGTEST_STORAGE_TOKEN_PRJ2` and `INTEGTEST_WORKSPACE_SCHEMA_PRJ2` (the optional second -project) are unchanged. +Vitest prints values in failure output. **Never** put a raw token where a failed assertion or +thrown error would render it. Read the token from the leased `config`/`TestProject` object and +pass it into clients — do not interpolate it into assertion messages or `console.log`. Tokens +in `projects.json` come from CI secrets; keep them out of logs and snapshots. --- -## 5. GitHub CI Setup - -### Pool projects - -The CI pool consists of four Keboola projects, all on -`https://connection.europe-west3.gcp.keboola.com`: +## 5. CI -| Project ID | Dashboard URL | Backend | Notes | -|------------|-------------------------------------------------------------------------------|-----------|--------------| -| 2728 | https://connection.europe-west3.gcp.keboola.com/admin/projects/2728/dashboard | Snowflake | | -| 2729 | https://connection.europe-west3.gcp.keboola.com/admin/projects/2729/dashboard | Snowflake | | -| 2731 | https://connection.europe-west3.gcp.keboola.com/admin/projects/2731/dashboard | BigQuery | | -| 2732 | https://connection.europe-west3.gcp.keboola.com/admin/projects/2732/dashboard | BigQuery | | +The `integration_tests` job (see the RFC's `ci-job.yml` sketch) runs: -Having four slots means up to four CI jobs can run concurrently — each acquires a -different project from the pool and they do not block each other. +1. `export-kbc-projects` → generates `projects.json` from `TEST_KBC_PROJECT_*` secrets and + exports its absolute path as `TEST_KBC_PROJECTS_FILE`. +2. Starts a redis service and sets `TEST_MCP_PROJECTS_LOCK_HOST` / `_PASSWORD`. +3. `npm run test:integ`. -### Branch storage test project (not in pool) +Pool size = max concurrent runners. Add projects to `.github/ci/projects.json` (and a matching +`TEST_KBC_PROJECT__TOKEN` secret) to raise the ceiling. Integration tests are skipped for +fork PRs (no access to secrets). -| Project ID | Dashboard URL | Backend | Notes | -|------------|-------------------------------------------------------------------------------|-----------|------------------------------------| -| 2908 | https://connection.europe-west3.gcp.keboola.com/admin/projects/2908/dashboard | Snowflake | Has `storage-branches` feature | +### The committed pool (`.github/ci/projects.json`) -This project is used by `test_storage_branches.py` via `INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES`. -It is **not** part of the pool and has no lock mechanism — concurrent access is safe because -production data is created idempotently and each session only manages its own branches. +All on the `connection.europe-west3.gcp.keboola.com` stack (GCP → `gcs` staging): -### Secrets and variables - -The `integration_tests` job in `.github/workflows/ci.yml` reads the following from the -repository's GitHub Secrets/Variables: - -| Name | Kind | Purpose | +| Project ID | Backend | Role | |---|---|---| -| `INTEGTEST_STORAGE_TOKENS` | Secret | Space-separated master tokens for all four pool projects | -| `INTEGTEST_POOL_STORAGE_API_URL` | Variable | `https://connection.europe-west3.gcp.keboola.com` | -| `INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES` | Secret | Master token for a project **with** the `storage-branches` feature (used by `test_storage_branches.py`) | - -### Concurrency - -- **Within a single CI run** — the matrix covers Python 3.10, 3.11, and 3.12 with - `max-parallel: 1`, so the three versions run sequentially. This is intentional: a - single run only needs one project slot, not three. -- **Across concurrent CI runs** — the project-pool locking protocol (described in - section 2) handles collisions. Each run acquires a different project, so up to four - runs can proceed in parallel without interfering. -- **Duplicate-run prevention** — a workflow-level concurrency key - (`ci-${{ github.ref }}`) cancels any in-progress run on the same branch when a new - push arrives. - -### Fork behaviour - -Integration tests are skipped for pull requests from forks -(`github.repository != github.event.repository.full_name`) because forks do not have -access to the repository secrets. The dependencies are still installed so the -environment setup can be validated. \ No newline at end of file +| 3053, 3054 | Snowflake | pool | +| 3056, 3057 | BigQuery | pool | +| 3055 | Snowflake | has the `storage-branches` feature (used by the branch-storage tests) | + +Each needs a `TEST_KBC_PROJECT__TOKEN` GitHub secret (a Storage API master token). The +redis lease comes from `vars.TEST_MCP_PROJECTS_LOCK_HOST` + `secrets.TEST_MCP_PROJECTS_LOCK_PASSWORD`, +and the pool file path from `vars.TEST_KBC_PROJECTS_FILE` — same convention as keboola/go-monorepo. diff --git a/integtests/clients/test_client.py b/integtests/clients/test_client.py deleted file mode 100644 index ca4f71dbd..000000000 --- a/integtests/clients/test_client.py +++ /dev/null @@ -1,42 +0,0 @@ -import logging - -import pytest - -from integtests.conftest import ProjectDef, TableDef -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.storage import AsyncStorageClient, GlobalSearchResponse - -LOG = logging.getLogger(__name__) - - -class TestAsyncStorageClient: - - @pytest.fixture - def storage_client(self, keboola_client: KeboolaClient, keboola_project: ProjectDef) -> AsyncStorageClient: - return keboola_client.storage_client - - @pytest.mark.asyncio - @pytest.mark.skip(reason='Unstable') - async def test_global_search(self, storage_client: AsyncStorageClient): - not_existing_id = 'not-existing-id' - ret = await storage_client.global_search(query=not_existing_id) - assert isinstance(ret, GlobalSearchResponse) - assert ret.all == 0 - assert ret.items == [] - assert ret.by_type == {'total': 0} - assert ret.by_project == {} - - @pytest.mark.asyncio - @pytest.mark.skip(reason='Unstable') - async def test_global_search_with_results(self, storage_client: AsyncStorageClient, tables: list[TableDef]): - search_for_name = 'test' - is_global_search_enabled = await storage_client.is_enabled('global-search') - if not is_global_search_enabled: - LOG.warning('Global search is not enabled in the project. Skipping test. Please enable it in the project.') - pytest.skip('Global search is not enabled in the project. Skipping test.') - - ret = await storage_client.global_search(query=search_for_name, types=['table']) - assert isinstance(ret, GlobalSearchResponse) - assert ret.all == len(tables) - assert len(ret.items) == len(tables) - assert all(item.type == 'table' for item in ret.items) diff --git a/integtests/clients/test_data_science.py b/integtests/clients/test_data_science.py deleted file mode 100644 index 9a780312f..000000000 --- a/integtests/clients/test_data_science.py +++ /dev/null @@ -1,124 +0,0 @@ -import logging -from typing import AsyncGenerator - -import pytest -import pytest_asyncio - -from keboola_mcp_server.clients.client import DATA_APP_COMPONENT_ID, KeboolaClient -from keboola_mcp_server.clients.data_science import DataAppConfig, DataAppResponse, DataScienceClient - -LOG = logging.getLogger(__name__) - - -def _minimal_parameters(slug: str) -> dict[str, object]: - """Build minimal valid parameters for a code-based Streamlit data app.""" - return { - 'size': 'tiny', - 'autoSuspendAfterSeconds': 600, - 'dataApp': { - 'slug': slug, - 'streamlit': { - 'config.toml': '[theme]\nbase = "light"', - }, - }, - 'script': [ - 'import streamlit as st', - "st.write('Hello from integration test')", - ], - } - - -def _public_access_authorization() -> dict[str, object]: - """Allow public access to all paths; no providers required.""" - return { - 'app_proxy': { - 'auth_providers': [], - 'auth_rules': [ - {'type': 'pathPrefix', 'value': '/', 'auth_required': False}, - ], - } - } - - -@pytest.fixture -def ds_client(keboola_client: KeboolaClient) -> DataScienceClient: - return keboola_client.data_science_client - - -@pytest_asyncio.fixture -async def initial_data_app(ds_client: DataScienceClient, unique_id: str) -> AsyncGenerator[DataAppResponse, None]: - data_app: DataAppResponse | None = None - try: - slug = f'test-app-{unique_id}' - config = DataAppConfig.model_validate( - {'parameters': _minimal_parameters(slug), 'authorization': _public_access_authorization()} - ) - data_app = await ds_client.create_data_app( - name=f'IntegTest {slug}', - description='Created by integration tests', - configuration=config, - ) - assert isinstance(data_app, DataAppResponse) - yield data_app - finally: - if data_app: - try: - # The DSAPI delete endpoint removes a data app only if its desired and current states match. - # Otherwise, it returns a 400 Bad Request. - # When deploying/deleting/suspending/updating/etc. the data app, the desired state is set according to - # the action. Then there is a background task that runs for the given action and after it finishes, - # the current state is updated to match the desired state. - await ds_client.delete_data_app(data_app.id) - except Exception as e: - LOG.exception(f'Failed to delete data app: {e}') - raise - - -@pytest.mark.asyncio -async def test_create_and_fetch_data_app( - ds_client: DataScienceClient, initial_data_app: DataAppResponse, keboola_client: KeboolaClient -) -> None: - """Test creating a data app and fetching it from detail and list endpoints""" - # Check if the created data app is valid - created = initial_data_app - assert isinstance(created, DataAppResponse) - assert created.id - assert created.state == 'created' - assert created.type == 'streamlit' - assert created.component_id == DATA_APP_COMPONENT_ID - - # Fetch the data app from data science - fetched_ds = await ds_client.get_data_app(created.id) - assert fetched_ds.id == created.id - assert fetched_ds.type == created.type - assert fetched_ds.component_id == created.component_id - assert fetched_ds.project_id == created.project_id - assert fetched_ds.config_id == created.config_id - assert fetched_ds.config_version == created.config_version - - # Fetch the data app config from storage - fetched_s = await keboola_client.storage_client.configuration_detail( - component_id=DATA_APP_COMPONENT_ID, - configuration_id=created.config_id, - ) - - # check if the data app ids are the same (data app from data science and config from storage) - assert 'configuration' in fetched_s - assert isinstance(fetched_s['configuration'], dict) - assert 'parameters' in fetched_s['configuration'] - assert isinstance(fetched_s['configuration']['parameters'], dict) - assert 'id' in fetched_s['configuration']['parameters'] - assert fetched_ds.id == fetched_s['configuration']['parameters']['id'] - - # Fetch the all data apps and check if the created data app is in the list - # TODO: Remove this limit once DSAPI is fixed. - # The limit is temporarily increased to 500 to prevent leftover data apps from previous tests. - # These apps cannot be deleted because their configurations were removed in SAPI first, - # causing the DSAPI delete endpoint to return a 500 error afterward. - data_apps = await ds_client.list_data_apps(limit=500) - assert isinstance(data_apps, list) - assert len(data_apps) > 0 - assert any(app.id == created.id for app in data_apps) - # TODO(REMOVE): Remove this assertion once DSAPI is fixed. This only checks that we do not leave any data apps - # in the CI project after test executions except those which are already there and cannot be deleted. - assert len(data_apps) < 110 diff --git a/integtests/clients/test_encryption.py b/integtests/clients/test_encryption.py deleted file mode 100644 index 00a822a23..000000000 --- a/integtests/clients/test_encryption.py +++ /dev/null @@ -1,43 +0,0 @@ -from typing import Any - -import pytest - -from keboola_mcp_server.clients.client import DATA_APP_COMPONENT_ID, KeboolaClient - - -def test_client_does_not_send_authorization_headers(keboola_client: KeboolaClient) -> None: - """Check that the encryption client does not send any authorization headers.""" - assert 'Authorization' not in keboola_client.encryption_client.raw_client.headers - assert 'X-StorageAPI-Token' not in keboola_client.encryption_client.raw_client.headers - - -@pytest.mark.asyncio -async def test_encrypt_string_not_equal(keboola_client: KeboolaClient) -> None: - project_id = await keboola_client.storage_client.project_id() - plaintext = 'my-plain-text' - encrypted = await keboola_client.encryption_client.encrypt( - value=plaintext, - project_id=str(project_id), - component_id=DATA_APP_COMPONENT_ID, - ) - assert isinstance(encrypted, str) - assert encrypted != plaintext - - -@pytest.mark.asyncio -async def test_encrypt_dict_hash_keys_only(keboola_client: KeboolaClient) -> None: - project_id = await keboola_client.storage_client.project_id() - payload: dict[str, Any] = { - '#secret': 'sensitive-value', - 'public': 'visible-value', - } - result = await keboola_client.encryption_client.encrypt( - value=payload, - project_id=str(project_id), - component_id=DATA_APP_COMPONENT_ID, - ) - assert isinstance(result, dict) - # Values under keys beginning with '#' should be encrypted (changed) - assert result['#secret'] != payload['#secret'] - # Non-secret values should remain the same - assert result['public'] == payload['public'] diff --git a/integtests/clients/test_metastore.py b/integtests/clients/test_metastore.py deleted file mode 100644 index dfb8cdb18..000000000 --- a/integtests/clients/test_metastore.py +++ /dev/null @@ -1,193 +0,0 @@ -from __future__ import annotations - -import logging -from urllib.parse import urljoin -from uuid import uuid4 - -import httpx -import pytest - -from keboola_mcp_server.clients.metastore import MetastoreClient - -LOG = logging.getLogger(__name__) - - -async def delete_metastore_object(client: MetastoreClient, object_type: str, uuid: str) -> None: - try: - await client.delete_object(object_type, uuid) - # Delete the soft deleted object - await client.delete_object(object_type, uuid) - except httpx.HTTPStatusError as exc: - if exc.response.status_code not in (401, 403, 404): - raise - - -@pytest.fixture(scope='session') -def metastore_url(storage_api_url: str) -> str: - """Derive metastore URL from storage API URL by replacing 'connection.' prefix.""" - return storage_api_url.replace('connection.', 'metastore.', 1) - - -@pytest.fixture -def metastore_client(storage_api_token: str, metastore_url: str) -> MetastoreClient: - return MetastoreClient.create(root_url=metastore_url, token=storage_api_token) - - -@pytest.fixture(scope='module', autouse=True) -def _require_metastore_available( - storage_api_token: str, - metastore_url: str, -) -> None: - try: - probe_url = urljoin(metastore_url, '/health-check') - with httpx.Client( - headers={'X-StorageApi-Token': storage_api_token}, - timeout=httpx.Timeout(3.0, connect=1.0), - ) as client: - response = client.get(probe_url) - response.raise_for_status() - except httpx.HTTPStatusError as exc: - _skip_unauthorized(exc) - raise - except httpx.ConnectError as exc: - pytest.skip(f'Metastore endpoint is not reachable in this environment: {exc}') - except httpx.TimeoutException as exc: - pytest.skip(f'Metastore endpoint timed out in this environment: {exc}') - - -def _skip_unauthorized(exc: httpx.HTTPStatusError) -> None: - if exc.response.status_code == 401: - details = '' - try: - details = exc.response.text[:300] - except Exception: - details = '' - LOG.warning(f'Metastore unauthorized (401) for {exc.request.url}: {details}') - pytest.skip('Token is not authorized for configured Metastore.') - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - 'object_type', - [ - 'semantic-model', - 'semantic-dataset', - 'semantic-metric', - 'semantic-relationship', - 'semantic-glossary', - 'semantic-constraint', - ], -) -async def test_get_schema_for_semantic_types(metastore_client: MetastoreClient, object_type: str) -> None: - schema_doc = await metastore_client.get_schema(object_type) - assert isinstance(schema_doc, dict) - - -@pytest.mark.asyncio -async def test_get_schema_specific_version(metastore_client: MetastoreClient) -> None: - latest = await metastore_client.get_schema('semantic-model') - version = latest.get('version') - if not version: - pytest.skip('Schema version not available in metastore response.') - specific = await metastore_client.get_schema('semantic-model', version=version) - assert specific.get('title') == 'semantic-model' - assert specific.get('version') == version - - -@pytest.mark.asyncio -async def test_list_semantic_models_and_optional_detail(metastore_client: MetastoreClient) -> None: - try: - models = await metastore_client.list_objects('semantic-model') - except httpx.HTTPStatusError as exc: - _skip_unauthorized(exc) - raise - assert isinstance(models, list) - - if not models: - pytest.skip('No semantic-model objects available in project scope for this token.') - - model = await metastore_client.get_object('semantic-model', models[0].id) - assert model.id - assert model.type == 'semantic-model' - assert isinstance(model.attributes, dict) - - -@pytest.mark.asyncio -async def test_get_organization_models_list(metastore_client: MetastoreClient) -> None: - try: - models = await metastore_client.list_objects('semantic-model', organization_scope=True) - except httpx.HTTPStatusError as exc: - if exc.response.status_code in (401, 403, 404): - pytest.skip('Organization scope endpoint is not accessible for this token/environment.') - raise - assert isinstance(models, list) - - -@pytest.mark.asyncio -async def test_crud_walkthrough_post_get_put_delete_and_revisions(metastore_client: MetastoreClient) -> None: - object_type = 'semantic-model' - model_name = f'ai2607-it-model-{uuid4().hex[:8]}' - model_uuid: str | None = None - - try: - try: - created = await metastore_client.create_object( - object_type, - name=model_name, - data={ - 'name': model_name, - 'sql_dialect': 'Snowflake', - 'description': 'Integration test model', - }, - ) - except httpx.HTTPStatusError as exc: - _skip_unauthorized(exc) - raise - - model_uuid = created.id - assert model_uuid - assert created.type == object_type - assert created.attributes.get('name') == model_name - - fetched = await metastore_client.get_object(object_type, model_uuid) - assert fetched.id == model_uuid - assert fetched.attributes.get('sql_dialect') == 'Snowflake' - - replaced = await metastore_client.put_object( - object_type, - model_uuid, - name=model_name, - data={ - 'name': model_name, - 'sql_dialect': 'BigQuery', - 'description': 'Replaced via PUT', - }, - ) - assert replaced.id == model_uuid - - fetched_after_put = await metastore_client.get_object(object_type, model_uuid) - assert fetched_after_put.attributes.get('sql_dialect') == 'BigQuery' - assert fetched_after_put.meta.revision >= 2 - - revisions = await metastore_client.list_revisions(object_type, filter_by=f'id={model_uuid}') - assert isinstance(revisions, list) - assert len(revisions) >= 1 - - rev1 = await metastore_client.get_revision(object_type, model_uuid, 1) - assert rev1.id == model_uuid - assert rev1.attributes.get('sql_dialect') == 'Snowflake' - - if fetched_after_put.meta.revision >= 2: - rev2 = await metastore_client.get_revision(object_type, model_uuid, 2) - assert rev2.id == model_uuid - assert rev2.attributes.get('sql_dialect') == 'BigQuery' - - await delete_metastore_object(metastore_client, object_type, model_uuid) - - with pytest.raises(httpx.HTTPStatusError) as exc_info: - await metastore_client.get_object(object_type, model_uuid) - assert exc_info.value.response.status_code == 404 - - finally: - if model_uuid: - await delete_metastore_object(metastore_client, object_type, model_uuid) diff --git a/integtests/conftest.py b/integtests/conftest.py deleted file mode 100644 index 4626fb9dc..000000000 --- a/integtests/conftest.py +++ /dev/null @@ -1,636 +0,0 @@ -import dataclasses -import importlib.metadata -import json -import logging -import os -import time -import uuid -from collections.abc import AsyncGenerator -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Generator - -import pytest -import pytest_asyncio -import requests -from dotenv import load_dotenv -from fastmcp import Client, Context, FastMCP -from kbcstorage.client import Client as SyncStorageClient -from mcp.server.session import ServerSession -from mcp.shared.context import RequestContext -from mcp.types import ClientCapabilities, Implementation, InitializeRequestParams - -from integtests.project_lock import ( - DEFAULT_MAX_WAIT_MINUTES, - DEFAULT_POLL_INTERVAL_SECONDS, - DEFAULT_TTL_MINUTES, - AcquiredProject, - ProjectPool, - verify_project_endpoint, -) -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.mcp import ServerState, SessionStateMiddleware -from keboola_mcp_server.server import create_server -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - -_project_pool: ProjectPool | None = None -_acquired_project: AcquiredProject | None = None -_project_info: str | None = None -_project_info_printed: bool = False - -POOL_STORAGE_API_URL_ENV_VAR = 'INTEGTEST_POOL_STORAGE_API_URL' -STORAGE_API_TOKENS_ENV_VAR = 'INTEGTEST_STORAGE_TOKENS' # space-separated pool of tokens -# The second pair of token/schema for testing simultaneous access to two different projects. -STORAGE_API_TOKEN_ENV_VAR_2 = 'INTEGTEST_STORAGE_TOKEN_PRJ2' -WORKSPACE_SCHEMA_ENV_VAR_2 = 'INTEGTEST_WORKSPACE_SCHEMA_PRJ2' -# We reset dev environment variables to integtest values to ensure tests run locally using .env settings. -DEV_STORAGE_API_URL_ENV_VAR = 'STORAGE_API_URL' -DEV_STORAGE_TOKEN_ENV_VAR = 'KBC_STORAGE_TOKEN' -DEV_WORKSPACE_SCHEMA_ENV_VAR = 'KBC_WORKSPACE_SCHEMA' -INTEGTEST_CLIENT_INFO = Implementation( - name='kbc-mcp-integtests', version=importlib.metadata.version('keboola_mcp_server') -) -INTEGTEST_USER_AGENT = f'{INTEGTEST_CLIENT_INFO.name}/{INTEGTEST_CLIENT_INFO.version}' - - -@dataclass(frozen=True) -class BucketDef: - bucket_id: str - display_name: str - - -@dataclass(frozen=True) -class TableDef: - bucket_id: str - table_name: str - table_id: str - - @property - def file_path(self) -> Path: - """ - Path to the CSV file containing the table data. - """ - return _data_dir() / 'proj' / 'buckets' / self.bucket_id / f'{self.table_name}.csv' - - -@dataclass(frozen=True) -class ConfigDef: - component_id: str - configuration_id: str | None # Will be generated by Storage API - internal_id: str - - @property - def file_path(self) -> Path: - """ - Path to the JSON file containing the configuration. - """ - return _data_dir() / 'proj' / 'configs' / self.component_id / f'{self.internal_id}.json' - - -@dataclass(frozen=True) -class ProjectDef: - project_id: str - buckets: list[BucketDef] - tables: list[TableDef] - configs: list[ConfigDef] - - -@pytest.fixture(scope='session') -def env_file_loaded() -> bool: - return load_dotenv() - - -@pytest.fixture(autouse=True) -def _patch_fastmcp_client_default_info(mocker) -> None: - # Ensure all fastmcp.Client instances in integration tests use a distinct identity - # unless a test intentionally provides a different client_info. - original_init = Client.__init__ - - def _init_with_integtest_client_info(self, *args: Any, **kwargs: Any) -> None: - kwargs.setdefault('client_info', INTEGTEST_CLIENT_INFO) - original_init(self, *args, **kwargs) - - mocker.patch.object(Client, '__init__', _init_with_integtest_client_info) - - -@pytest.fixture(scope='session', autouse=True) -def _patch_session_middleware_user_agent() -> Generator[None, None, None]: - # Force a distinct User-Agent for outbound Keboola API requests during integration tests. - monkeypatch = pytest.MonkeyPatch() - original_get_headers = SessionStateMiddleware._get_headers.__func__ - - def _get_headers_with_integtest_ua( - cls: type[SessionStateMiddleware], runtime_info: ServerRuntimeInfo - ) -> dict[str, Any]: - headers = original_get_headers(cls, runtime_info) - headers['User-Agent'] = INTEGTEST_USER_AGENT - return headers - - monkeypatch.setattr(SessionStateMiddleware, '_get_headers', classmethod(_get_headers_with_integtest_ua)) - try: - yield - finally: - monkeypatch.undo() - - -@pytest.fixture(scope='session') -def env_init(env_file_loaded: bool, storage_api_token: str, storage_api_url: str, workspace_schema: str) -> bool: - # We reset the development environment variables to the values of the integtest environment variables. - os.environ[DEV_STORAGE_API_URL_ENV_VAR] = storage_api_url - os.environ[DEV_STORAGE_TOKEN_ENV_VAR] = storage_api_token - os.environ[DEV_WORKSPACE_SCHEMA_ENV_VAR] = workspace_schema - return env_file_loaded - - -def _data_dir() -> Path: - return Path(__file__).parent / 'data' - - -@pytest.fixture(scope='session') -def storage_api_url(env_file_loaded: bool) -> str: - storage_api_url = os.getenv(POOL_STORAGE_API_URL_ENV_VAR) - assert storage_api_url, f'{POOL_STORAGE_API_URL_ENV_VAR} must be set' - return storage_api_url - - -@pytest.fixture(scope='session') -def storage_api_token(project_lock: AcquiredProject) -> str: - return project_lock.endpoint.storage_api_token - - -@pytest.fixture(scope='session') -def mcp_config(storage_api_token: str, storage_api_url: str) -> Config: - return Config(storage_api_url=storage_api_url, storage_token=storage_api_token) - - -@pytest.fixture(scope='session') -def workspace_schema(_clean_project: None, storage_api_token: str, storage_api_url: str) -> Generator[str, Any, None]: - """ - Create one read-only workspace for the whole test session and yield its schema. - - The workspace is created once (session scope), reused across all tests, and deleted - on teardown. - """ - # sync_storage_client is function-scoped; build a session-scoped client here. - storage_client = _sync_storage_client(storage_api_token, storage_api_url) - token_info = storage_client.tokens.verify() - backend = token_info['owner'].get('defaultBackend') - # Mirror the MCP server's own per-backend workspace login type (see WorkspaceManager._create_ws). - if backend == 'snowflake': - login_type = 'snowflake-person-sso' - elif backend == 'bigquery': - login_type = 'default' - else: - raise RuntimeError(f'Unexpected project default backend: {backend!r} (expected snowflake or bigquery)') - - LOG.info(f'Creating a read-only {backend} workspace for the test session') - workspace = storage_client.workspaces.create( - backend=backend, - login_type=login_type, - # kbcstorage maps read_all_objects to the readOnlyStorageAccess flag (read-only storage access). - read_all_objects=True, - ) - workspace_id = workspace['id'] - schema = workspace['connection']['schema'] - LOG.info(f'Created read-only workspace {workspace_id} (schema {schema})') - try: - yield schema - finally: - LOG.info(f'Deleting test-session workspace {workspace_id}') - try: - storage_client.workspaces.delete(workspace_id) - except Exception: - LOG.exception(f'Failed to delete test-session workspace {workspace_id}') - - -@pytest.fixture(scope='session') -def storage_api_token_2(env_file_loaded: bool) -> str | None: - return os.getenv(STORAGE_API_TOKEN_ENV_VAR_2) - - -@pytest.fixture(scope='session') -def workspace_schema_2(env_file_loaded: bool) -> str | None: - return os.getenv(WORKSPACE_SCHEMA_ENV_VAR_2) - - -@pytest.fixture(scope='session') -def shared_datadir_ro() -> Path: - """ - Session-scoped access to shared data directory for integration tests. - Do not modify the data in this directory. - For function-scoped access to the data, use `shared_datadir` fixture provided by `pytest-datadir`, - which creates a temporary copy of the data which can therefore be modified. - """ - return _data_dir() - - -_BUCKET_DEFS_IN = [ - BucketDef(bucket_id='in.c-test_bucket_01', display_name='test_bucket_01'), - BucketDef(bucket_id='in.c-test_bucket_02', display_name='test_bucket_02'), -] - - -def _create_buckets(storage_client: SyncStorageClient) -> list[BucketDef]: - for bucket in _BUCKET_DEFS_IN: - LOG.info(f'Creating bucket with display name={bucket.display_name}') - created_bucket = storage_client.buckets.create(bucket.display_name) - assert created_bucket['id'] == bucket.bucket_id - assert created_bucket['displayName'] == bucket.display_name - - return _BUCKET_DEFS_IN - - -_TABLE_DEFS_IN = [ - TableDef( - bucket_id='in.c-test_bucket_01', - table_name='test_table_01', - table_id='in.c-test_bucket_01.test_table_01', - ), -] - - -def _create_tables(storage_client: SyncStorageClient) -> list[TableDef]: - for table in _TABLE_DEFS_IN: - LOG.info(f'Creating table with name={table.table_name}') - created_table_id = storage_client.tables.create( - bucket_id=table.bucket_id, - name=table.table_name, - file_path=str(table.file_path), - ) - assert created_table_id == table.table_id - - return _TABLE_DEFS_IN - - -_CONFIG_DEFS_IN = [ - ConfigDef( - component_id='ex-generic-v2', - configuration_id=None, - internal_id='test_config1', - ), - ConfigDef( - component_id='keboola.snowflake-transformation', - configuration_id=None, - internal_id='test_config2', - ), -] - - -def _create_configs(storage_client: SyncStorageClient) -> list[ConfigDef]: - configs = [] - for config in _CONFIG_DEFS_IN: - LOG.info(f'Creating config with internal ID={config.internal_id}') - with config.file_path.open('r', encoding='utf-8') as cfg_file: - created_config = storage_client.configurations.create( - component_id=config.component_id, - name=config.internal_id, - configuration_id=None, - configuration=json.load(cfg_file), - ) - config = dataclasses.replace(config, configuration_id=created_config['id']) - configs.append(config) - LOG.info(f'Created config with component ID={config.component_id} and config ID={config.configuration_id}') - - return configs - - -def _sync_storage_client(storage_api_token: str, storage_api_url: str) -> SyncStorageClient: - client = SyncStorageClient(storage_api_url, storage_api_token) - token_info = client.tokens.verify() - LOG.info( - f'Authorized as "{token_info["description"]}" ({token_info["id"]}) ' - f'to project "{token_info["owner"]["name"]}" ({token_info["owner"]["id"]}) ' - f'at "{client.root_url}" stack.' - ) - return client - - -# Buckets the integration fixtures/tests create are all stage-prefixed "c-test*". A real -# (non-dedicated) project's buckets won't match, so this doubles as a safety guard against a -# misconfigured INTEGTEST_STORAGE_TOKENS pointing at a project whose data must not be wiped. -_TEST_BUCKET_ID_PREFIXES = ('in.c-test', 'out.c-test', 'sys.c-test') - -# Workspaces not created by the tests that must never be deleted (matched on the creator token). -_STATIC_WORKSPACE_CREATORS = frozenset({'Background Indexing Token'}) - - -def _guard_dedicated_test_project(storage_client: SyncStorageClient, project_id: str) -> None: - """Refuse to reset the project unless it looks like a dedicated integtest project. - - The integ fixtures/tests only ever create stage-prefixed ``*.c-test*`` buckets, so a project - holding any other bucket is almost certainly not a dedicated test project. Failing here (instead - of deleting) protects against a misconfigured ``INTEGTEST_STORAGE_TOKENS`` that points at a - project whose data must not be wiped. - """ - foreign = [b['id'] for b in storage_client.buckets.list() if not b['id'].startswith(_TEST_BUCKET_ID_PREFIXES)] - if foreign: - pytest.fail( - f'Refusing to reset project {project_id}: found non-test buckets {foreign}. ' - f'INTEGTEST_STORAGE_TOKENS may be pointing at a non-dedicated project.' - ) - - -def _purge_project(storage_client: SyncStorageClient, storage_api_url: str, project_id: str) -> None: - """Reset a dedicated integtest project to a clean state. - - Integration tests run against a shared pool of projects acquired per run. A prior session that - was cancelled (a new push/rebase cancels the in-flight CI run) or timed out before its teardown - leaves buckets, configurations, workspaces and branch metadata behind. The next run that locks - the project would then fail an empty-project assertion *before* its own teardown runs, so the - project stays dirty and wedges every subsequent run. Purging here lets the shared pool self-heal. - """ - _guard_dedicated_test_project(storage_client, project_id) - - buckets = storage_client.buckets.list() - workspaces = [ - w - for w in storage_client.workspaces.list() - if w.get('creatorToken', {}).get('description') not in _STATIC_WORKSPACE_CREATORS - ] - components = storage_client.components.list(include=['configuration']) - branch_meta = [ - m for m in storage_client.branches.metadata('default') if m.get('key') == WorkspaceManager.MCP_META_KEY - ] - - if buckets or workspaces or any(c.get('configurations') for c in components) or branch_meta: - LOG.warning( - f'Project {project_id} was not clean: {len(buckets)} bucket(s), {len(workspaces)} workspace(s), ' - f'{sum(len(c.get("configurations", [])) for c in components)} config(s), ' - f'{len(branch_meta)} workspace-metadata entr(ies) — likely an interrupted prior run. Purging.' - ) - - for bucket in buckets: - storage_client.buckets.delete(bucket['id'], force=True) - - for workspace in workspaces: - storage_client.workspaces.delete(workspace['id']) - - for component in components: - component_id = component['id'] - for config in component.get('configurations', []): - # Double delete because the first delete only moves the configuration to the trash. - storage_client.configurations.delete(component_id, config['id']) - storage_client.configurations.delete(component_id, config['id']) - - # kbcstorage exposes no branch-metadata delete; the MCP WorkspaceManager stamps MCP_META_KEY on - # the default branch when it creates a workspace, so remove leftovers with a raw request. - for meta in branch_meta: - resp = requests.delete( - f'{storage_api_url.rstrip("/")}/v2/storage/branch/default/metadata/{meta["id"]}', - headers={'X-StorageApi-Token': storage_client.token}, - ) - resp.raise_for_status() - - -@pytest.fixture(scope='session') -def _clean_project(storage_api_token: str, storage_api_url: str) -> None: - """Reset the acquired pool project before any other fixture creates resources, so an interrupted - prior run can't wedge the shared pool. Other session fixtures depend on this to order it first. - - Side-effect only (no return value), so the leading underscore is kept per pytest-style PT005. - """ - storage_client = _sync_storage_client(storage_api_token, storage_api_url) - project_id: str = storage_client.tokens.verify()['owner']['id'] - _purge_project(storage_client, storage_api_url, project_id) - - -@pytest.fixture(scope='session') -def keboola_project( - _clean_project: None, env_init: bool, storage_api_token: str, storage_api_url: str -) -> Generator[ProjectDef, Any, None]: - """ - Sets up a Keboola project with items needed for integration tests, - such as buckets, tables and configurations. - The project is reset to a clean state first (see _clean_project) and cleaned up after the tests. - """ - # Cannot use keboola_client fixture because it is function-scoped - storage_client = _sync_storage_client(storage_api_token, storage_api_url) - token_info = storage_client.tokens.verify() - project_id: str = token_info['owner']['id'] - - buckets = _create_buckets(storage_client) - tables = _create_tables(storage_client) - configs = _create_configs(storage_client) - - if 'global-search' in token_info['owner'].get('features', []): - # Give the global search time to catch up on the changes done in the testing project. - # See https://help.keboola.com/management/global-search/#limitations for more info. - time.sleep(10) - - LOG.info(f'Test setup for project {project_id} complete') - yield ProjectDef(project_id=project_id, buckets=buckets, tables=tables, configs=configs) - - LOG.info(f'Cleaning up Keboola project with ID={project_id}') - current_buckets = storage_client.buckets.list() - for bucket in current_buckets: - bucket_id = bucket['id'] - LOG.info(f'Deleting bucket with ID={bucket_id}') - storage_client.buckets.delete(bucket_id, force=True) - - for config in configs: - LOG.info(f'Deleting config with component ID={config.component_id} and config ID={config.configuration_id}') - storage_client.configurations.delete(config.component_id, config.configuration_id) - # Double delete because the first delete moves the configuration to the trash - storage_client.configurations.delete(config.component_id, config.configuration_id) - - -def _setup_pool(storage_api_url: str) -> tuple[ProjectPool, AcquiredProject]: - tokens_raw = os.getenv(STORAGE_API_TOKENS_ENV_VAR, '').strip() - if not tokens_raw: - raise RuntimeError( - f'{STORAGE_API_TOKENS_ENV_VAR} must be set to a non-empty space-separated list of project tokens' - ) - tokens = tokens_raw.split() - - endpoints = [verify_project_endpoint(storage_api_url, t) for t in tokens] - - pool = ProjectPool( - endpoints=endpoints, - ttl_minutes=int(os.getenv('INTEGTEST_LOCK_TTL_MINUTES', str(DEFAULT_TTL_MINUTES))), - poll_interval_seconds=int( - os.getenv('INTEGTEST_LOCK_POLL_INTERVAL_SECONDS', str(DEFAULT_POLL_INTERVAL_SECONDS)) - ), - max_wait_minutes=int(os.getenv('INTEGTEST_LOCK_MAX_WAIT_MINUTES', str(DEFAULT_MAX_WAIT_MINUTES))), - ) - return pool, pool.acquire() - - -def pytest_collection_finish(session: pytest.Session) -> None: - """Pytest hook called after collection is finished, before any tests run. - - Eagerly acquires the project lock and prints project/token info to the terminal - so it appears before test progress output — avoiding the ANSI overwrite issue that - occurs when writing during fixture setup (which runs mid-progress-line in compact mode). - - If acquisition fails (e.g. missing env vars), the hook logs a warning and returns - without raising; the ``project_lock`` fixture will then acquire lazily and call - ``pytest.fail()`` with a proper error message. - - See also: - https://docs.pytest.org/en/stable/reference/reference.html#pytest.hookspec.pytest_collection_finish - """ - global _project_pool, _acquired_project, _project_info, _project_info_printed - needs_lock = any('project_lock' in getattr(item, 'fixturenames', []) for item in session.items) - if not needs_lock: - return - load_dotenv() - storage_api_url = os.getenv(POOL_STORAGE_API_URL_ENV_VAR) - if not storage_api_url: - return - try: - _project_pool, _acquired_project = _setup_pool(storage_api_url) - except Exception as exc: - LOG.warning(f'[integtest] Eager project lock acquisition failed: {exc}') - return - ep = _acquired_project.endpoint - _project_info = ep.describe() - reporter = session.config.pluginmanager.get_plugin('terminalreporter') - if reporter is not None: - reporter.write_sep('-', 'integtest project') - reporter.write_line(_project_info) - _project_info_printed = True - - -def pytest_terminal_summary(terminalreporter: Any, exitstatus: int, config: pytest.Config) -> None: - if _project_info is not None and not _project_info_printed: - terminalreporter.write_sep('-', 'integtest project') - terminalreporter.write_line(_project_info) - - -@pytest.fixture(scope='session') -def project_lock(env_file_loaded: bool, storage_api_url: str) -> Generator[AcquiredProject, Any, None]: - """ - Acquires a distributed lock on a Keboola test project via branch metadata. - - Requires INTEGTEST_STORAGE_TOKENS to be set to a space-separated list of project - tokens, one per project in the pool. The Storage API URL is provided by the - storage_api_url fixture (INTEGTEST_POOL_STORAGE_API_URL). - - Yields AcquiredProject(endpoint, lock_info) so callers know which project was - selected. The storage_api_token fixture reads its value from the acquired endpoint. - - The project lock is normally acquired eagerly in pytest_collection_finish before - tests start. This fixture falls back to acquiring it lazily if that hook did not run. - """ - global _project_pool, _acquired_project, _project_info - if _acquired_project is None: - try: - _project_pool, _acquired_project = _setup_pool(storage_api_url) - except RuntimeError as exc: - pytest.fail(str(exc)) - ep = _acquired_project.endpoint - _project_info = ep.describe() - try: - yield _acquired_project - finally: - if _project_pool is not None and _acquired_project is not None: - _project_pool.release(_acquired_project) - _acquired_project = None - _project_pool = None - - -@pytest.fixture(scope='session') -def buckets(keboola_project: ProjectDef) -> list[BucketDef]: - return keboola_project.buckets - - -@pytest.fixture(scope='session') -def tables(keboola_project: ProjectDef) -> list[TableDef]: - return keboola_project.tables - - -@pytest.fixture(scope='session') -def configs(keboola_project: ProjectDef) -> list[ConfigDef]: - return keboola_project.configs - - -@pytest.fixture -def sync_storage_client(storage_api_token: str, storage_api_url: str) -> SyncStorageClient: - """Gets the ordinary (synchronous) client from the official Keboola SDK (i.e. `kbcstorage` package).""" - return _sync_storage_client(storage_api_token, storage_api_url) - - -@pytest.fixture -def keboola_client(sync_storage_client: SyncStorageClient) -> KeboolaClient: - return KeboolaClient( - storage_api_token=sync_storage_client.token, - storage_api_url=sync_storage_client.root_url, - headers={'User-Agent': INTEGTEST_USER_AGENT}, - ) - - -@pytest.fixture -def unique_id() -> str: - """Generates a unique ID string for test resources.""" - return str(uuid.uuid4())[:8] - - -@pytest_asyncio.fixture -async def workspace_manager(keboola_client: KeboolaClient, workspace_schema: str) -> WorkspaceManager: - return await WorkspaceManager.create(keboola_client, workspace_schema) - - -@pytest_asyncio.fixture() -async def require_snowflake(workspace_manager: WorkspaceManager) -> None: - sql_dialect = await workspace_manager.get_sql_dialect() - if sql_dialect != 'Snowflake': - pytest.skip(f'Snowflake backend required, got: {sql_dialect}') - - -@pytest_asyncio.fixture() -async def require_bigquery(workspace_manager: WorkspaceManager) -> None: - sql_dialect = await workspace_manager.get_sql_dialect() - if sql_dialect != 'BigQuery': - pytest.skip(f'BigQuery backend required, got: {sql_dialect}') - - -@pytest.fixture -def mcp_context( - mocker, - keboola_client: KeboolaClient, - workspace_manager: WorkspaceManager, - keboola_project: ProjectDef, - mcp_config: Config, -) -> Context: - """ - MCP context containing the Keboola client and workspace manager. - """ - client_context = mocker.MagicMock(Context) - client_context.session = mocker.MagicMock(ServerSession) - # We set the user session state as it is done in the @with_session_state decorator - client_context.session.state = { - KeboolaClient.STATE_KEY: keboola_client, - WorkspaceManager.STATE_KEY: workspace_manager, - } - client_context.session.client_params = InitializeRequestParams( - protocolVersion='1', - capabilities=ClientCapabilities(), - clientInfo=INTEGTEST_CLIENT_INFO, - ) - client_context.client_id = INTEGTEST_USER_AGENT - client_context.session_id = None - client_context.request_context = mocker.MagicMock(RequestContext) - client_context.request_context.lifespan_context = ServerState(mcp_config, ServerRuntimeInfo(transport='stdio')) - # `meta` is an instance attribute of RequestContext (set in __init__), not a class attribute, - # so MagicMock(spec=RequestContext) doesn't expose it. Default it to None so tools that read - # the progressToken (e.g. query_data) don't trip AttributeError; individual tests can override. - client_context.request_context.meta = None - - return client_context - - -@pytest.fixture -def mcp_server(storage_api_url: str, storage_api_token: str, workspace_schema: str) -> FastMCP: - config = Config(storage_api_url=storage_api_url, storage_token=storage_api_token, workspace_schema=workspace_schema) - server = create_server(config, runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - return server - - -@pytest_asyncio.fixture -async def mcp_client(mcp_server: FastMCP) -> AsyncGenerator[Client, None]: - async with Client(mcp_server, client_info=INTEGTEST_CLIENT_INFO) as client: - yield client diff --git a/integtests/errors.test.ts b/integtests/errors.test.ts new file mode 100644 index 000000000..24990f209 --- /dev/null +++ b/integtests/errors.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolRaw, connectMcp } from './helpers/mcp'; +import { getTestProjectForTest } from './testproject/fixture'; + +// Ported from integtests/test_errors.py. +// +// The Python tests call the in-process tool functions directly and assert on raised +// exceptions / typed outputs. Here we exercise the same error paths end-to-end through +// the MCP client: tools that swallow "not found" into a structured output return text +// (no isError), while tools that propagate an upstream HTTP/SQL error surface it as a +// CallToolResult with `isError: true` and the formatted message in the text content. + +const errorText = (result: unknown): string => + (result as { content: { text: string }[] }).content[0]!.text; + +describe('error handling (integration)', () => { + // test_storage_api_404_error_maintains_standard_behavior: a non-existent bucket is + // reported via the structured `buckets_not_found` field, not as a tool error. + it('get_buckets reports a missing bucket without erroring', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const result = await callToolRaw(session.client, 'get_buckets', { + bucket_ids: ['non.existent.bucket'], + }); + expect(result.isError).toBeFalsy(); + expect(errorText(result)).toContain('non.existent.bucket'); + } finally { + await session.close(); + } + }); + + // test_concurrent_error_handling: many concurrent not-found lookups are each handled + // consistently (no error, each reported in buckets_not_found). + it('handles concurrent missing-bucket lookups consistently', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const results = await Promise.all( + Array.from({ length: 5 }, (_, i) => + callToolRaw(session.client, 'get_buckets', { + bucket_ids: [`non.existent.bucket.${i}`], + }), + ), + ); + for (let i = 0; i < results.length; i++) { + const result = results[i]!; + expect(result.isError).toBeFalsy(); + expect(errorText(result)).toContain(`non.existent.bucket.${i}`); + } + } finally { + await session.close(); + } + }); + + // test_jobs_api_404_error_: requesting a non-existent job id propagates the upstream + // 404 as a tool error mentioning the job id. + it('get_jobs surfaces a 404 for a non-existent job id as a tool error', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const result = await callToolRaw(session.client, 'get_jobs', { job_ids: ['999999999'] }); + expect(result.isError).toBeTruthy(); + // The TS queue client surfaces the upstream 404 as a generic "Not Found" message + // (it does not echo the job id / URL the Python HTTPStatusError carried), so we + // assert only that the not-found error reached the tool layer. + expect(errorText(result)).toMatch(/404|not found/i); + } finally { + await session.close(); + } + }); + + // NOTE: the Python test_docs_api_empty_query_error (a 422 from the AI docs service on an + // empty query) was dropped: docs_query is now served by the pgvector docs-search index + // (RFC: feature_spec/docs-search-pgvector/), where an empty query returns no results + // rather than erroring. The docs happy-path + index gating are covered by + // integtests/tools/doc.test.ts. + + // test_sql_api_invalid_query_error_(snowflake|bigquery): an invalid SQL query is + // surfaced as a tool error with the "Failed to run SQL query" prefix, regardless of + // the backend dialect. + it('query_data surfaces an invalid SQL query as a tool error', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const result = await callToolRaw(session.client, 'query_data', { + sql_query: 'INVALID SQL SYNTAX HERE', + query_name: 'Invalid SQL query.', + }); + expect(result.isError).toBeTruthy(); + expect(errorText(result)).toMatch(/Failed to run SQL query/i); + } finally { + await session.close(); + } + }); + + // Bad tool input (schema violation) is rejected by the server before the handler runs + // and is reported as a tool error — the structured-error end-to-end contract. + it('rejects a tool call with invalid input as a structured error', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + // get_jobs.limit is an int in [1, 500]; an out-of-range value must be rejected. + const result = await callToolRaw(session.client, 'get_jobs', { limit: 99999 }); + expect(result.isError).toBeTruthy(); + } finally { + await session.close(); + } + }); + + // test_event_emitted / TestStorageEvents: the Python suite verifies that a SAPI Storage + // event is emitted (and its mcpServerContext payload) for every tool call. That is an + // internal telemetry concern asserted by polling the SAPI events endpoint, not a + // tool-output contract reachable through the MCP client, so it is not ported here. + it.skip('emits a SAPI storage event per tool call (telemetry; not a tool-layer concern)', () => { + // Intentionally skipped — see comment above. Would require polling client.storage + // events and is out of scope for the in-memory server/middleware-level suite. + }); +}); diff --git a/integtests/helpers/mcp.ts b/integtests/helpers/mcp.ts new file mode 100644 index 000000000..fc0492c2e --- /dev/null +++ b/integtests/helpers/mcp.ts @@ -0,0 +1,47 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; + +import type { Config } from '@/config'; +import { createServer } from '@/server'; + +// Shared integration-test harness: connect a real MCP server (built from a leased project's +// Config) over an in-memory transport and call its tools — the same shape as the unit tests +// (__tests__/*), but hitting the real Keboola stack instead of msw mocks. + +export type McpSession = { + client: Client; + close: () => Promise; +}; + +export const connectMcp = async (config: Config): Promise => { + const [clientT, serverT] = InMemoryTransport.createLinkedPair(); + const server = createServer(config); + await server.connect(serverT); + const client = new Client({ name: 'kbc-mcp-integtests', version: '0.0.0' }); + await client.connect(clientT); + return { + client, + close: async () => { + await client.close(); + await server.close(); + }, + }; +}; + +/** Calls a tool and returns its text content, asserting the call did not error. */ +export const callToolText = async ( + client: Client, + name: string, + args: Record = {}, +): Promise => { + const result = await client.callTool({ name, arguments: args }); + if (result.isError) { + const text = (result.content as { text?: string }[])[0]?.text ?? ''; + throw new Error(`Tool "${name}" returned an error: ${text}`); + } + return (result.content as { text: string }[])[0]!.text; +}; + +/** Calls a tool and returns the raw CallToolResult (for negative-path / isError assertions). */ +export const callToolRaw = (client: Client, name: string, args: Record = {}) => + client.callTool({ name, arguments: args }); diff --git a/integtests/helpers/seed.ts b/integtests/helpers/seed.ts new file mode 100644 index 000000000..2ae41042b --- /dev/null +++ b/integtests/helpers/seed.ts @@ -0,0 +1,98 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import type { TestProject } from '../testproject/fixture'; + +// Seeds a leased project with the same fixtures the Python integ suite creates +// (integtests/conftest.py: _create_buckets/_create_tables/_create_configs): two input +// buckets, one CSV table, and two component configurations. Uses the raw Storage API +// (form-encoded, matching the kbcstorage SDK the Python tests used) so it does not depend +// on api-client method shapes. + +const DATA_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', 'data', 'proj'); + +export type SeedBucket = { id: string; displayName: string }; +export type SeedTable = { id: string; bucketId: string; name: string }; +export type SeedConfig = { componentId: string; configurationId: string; internalId: string }; +export type SeededProject = { + buckets: SeedBucket[]; + tables: SeedTable[]; + configs: SeedConfig[]; +}; + +const BUCKETS: { displayName: string; id: string }[] = [ + { displayName: 'test_bucket_01', id: 'in.c-test_bucket_01' }, + { displayName: 'test_bucket_02', id: 'in.c-test_bucket_02' }, +]; +const TABLES: { bucketId: string; name: string; id: string }[] = [ + { bucketId: 'in.c-test_bucket_01', name: 'test_table_01', id: 'in.c-test_bucket_01.test_table_01' }, +]; +const CONFIGS: { componentId: string; internalId: string; file: string }[] = [ + { componentId: 'ex-generic-v2', internalId: 'test_config1', file: 'ex-generic-v2/test_config1.json' }, + { + componentId: 'keboola.snowflake-transformation', + internalId: 'test_config2', + file: 'keboola.snowflake-transformation/test_config2.json', + }, +]; + +/** Form-encoded POST to the Storage API (the shape kbcstorage write endpoints expect). */ +const form = async ( + base: string, + token: string, + path: string, + fields: Record, +): Promise> => { + const body = new URLSearchParams(fields); + const res = await fetch(`${base}/v2/storage/${path}`, { + method: 'POST', + headers: { + 'X-StorageApi-Token': token, + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body, + }); + const text = await res.text(); + if (!res.ok) throw new Error(`Seed POST ${path} failed: ${res.status} ${text}`); + return text ? (JSON.parse(text) as Record) : {}; +}; + +export const seedProject = async (project: TestProject): Promise => { + const base = project.storageApiUrl; + const token = project.storageApiToken; + + const buckets: SeedBucket[] = []; + for (const b of BUCKETS) { + const created = await form(base, token, 'buckets', { name: b.displayName, stage: 'in' }); + buckets.push({ id: String(created.id ?? b.id), displayName: b.displayName }); + } + + const tables: SeedTable[] = []; + for (const t of TABLES) { + const csv = readFileSync(join(DATA_DIR, 'buckets', t.bucketId, `${t.name}.csv`), 'utf-8'); + // Synchronous create-from-string: the simplest way to seed a small table without the + // file-upload + async-import dance. + const created = await form(base, token, `buckets/${t.bucketId}/tables`, { + name: t.name, + dataString: csv, + }); + tables.push({ id: String(created.id ?? t.id), bucketId: t.bucketId, name: t.name }); + } + + const configs: SeedConfig[] = []; + for (const c of CONFIGS) { + const configuration = readFileSync(join(DATA_DIR, 'configs', c.file), 'utf-8'); + const created = await form(base, token, `branch/default/components/${c.componentId}/configs`, { + name: c.internalId, + configuration, + }); + configs.push({ + componentId: c.componentId, + configurationId: String(created.id ?? ''), + internalId: c.internalId, + }); + } + + return { buckets, tables, configs }; +}; diff --git a/integtests/mcp_server.test.ts b/integtests/mcp_server.test.ts new file mode 100644 index 000000000..e2cddf417 --- /dev/null +++ b/integtests/mcp_server.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolText, connectMcp } from './helpers/mcp'; +import { getTestProjectForTest } from './testproject/fixture'; + +// Ported from integtests/test_mcp_server.py. +// +// The Python suite spins up the server over stdio / streamable-http subprocesses and +// asserts: the expected tool set is present (tools/list), 6 prompts (prompts/list), +// 0 resources (resources/list), and a get_configs round-trip returns the seeded +// component config. Here we connect the in-process MCP server (built from a leased +// project's Config) over the in-memory transport and make the same assertions through +// the MCP client. The transport-matrix / multi-client / different-header cases are +// stdio/HTTP-subprocess concerns that do not apply to the in-memory harness, so they +// are not ported. + +// The tool set the server must expose. Tools gated on project features (conditional / +// classic flows, search, semantic) are excluded from the strict comparison because +// whether they appear depends on the leased project's enabled features — mirroring the +// Python `exclude` set. +const EXCLUDE = new Set([ + 'create_conditional_flow', + 'create_flow', + 'search', + 'update_flow', + 'modify_flow', + 'get_semantic_context', + 'get_semantic_schema', + 'search_semantic_context', + 'validate_semantic_query', + // Docs-search tools are gated on a configured pgvector index (DATABASE_URL); whether they + // appear depends on the run's env, so exclude them from the strict set (covered by + // integtests/tools/doc.test.ts). + 'docs_query', + 'find_component_id', +]); + +const EXPECTED_TOOLS = new Set( + [ + 'add_config_row', + 'create_conditional_flow', + 'create_config', + 'create_flow', + 'create_oauth_url', + 'create_python_js_data_app_git_credential', + 'create_sql_transformation', + 'delete_python_js_data_app_draft', + 'deploy_data_app', + 'docs_query', + 'find_component_id', + 'get_buckets', + 'get_components', + 'get_config_examples', + 'get_configs', + 'get_data_apps', + 'get_flow_examples', + 'get_flow_schema', + 'get_flows', + 'get_jobs', + 'get_project_info', + 'get_tables', + 'modify_flow', + 'modify_python_js_data_app', + 'modify_streamlit_data_app', + 'query_data', + 'run_job', + 'run_sync_action', + 'search', + 'update_config', + 'update_config_row', + 'update_descriptions', + 'update_flow', + 'update_project_description', + 'update_sql_transformation', + // The TS port carries a `get_server_info` scaffold tool that has no Python equivalent; + // it is expected to be present until the scaffold is removed. + 'get_server_info', + ].filter((name) => !EXCLUDE.has(name)), +); + +describe('MCP server wiring (integration)', () => { + it('exposes the expected tool set, 6 prompts, and no resources', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const { tools } = await session.client.listTools(); + const actual = new Set(tools.map((t) => t.name).filter((name) => !EXCLUDE.has(name))); + + const missing = [...EXPECTED_TOOLS].filter((name) => !actual.has(name)); + expect(missing, `Missing tools: ${missing.join(', ')}`).toEqual([]); + + const unexpected = [...actual].filter((name) => !EXPECTED_TOOLS.has(name)); + expect(unexpected, `Unexpected new tools: ${unexpected.join(', ')}`).toEqual([]); + + const { prompts } = await session.client.listPrompts(); + expect(prompts.length).toBe(6); + + // No resources are exposed. The SDK only registers a `resources/list` handler when + // at least one resource is registered, so with none the call raises "Method not + // found" (-32601) — which is itself proof the server exposes zero resources, the + // same fact the Python `len(resources) == 0` assertion checked. + const resourceCount = await session.client + .listResources() + .then((r) => r.resources.length) + .catch((err: { code?: number }) => { + if (err?.code === -32601) return 0; + throw err; + }); + expect(resourceCount).toBe(0); + } finally { + await session.close(); + } + }); + + it('round-trips a component configuration through create_config + get_configs', async () => { + // Port of _assert_get_component_details_tool_call: the Python test seeds a component + // config and fetches its detail through get_configs. We create the config through the + // tool layer instead (no dependency on the clean+seed wipe path, which is gated on a + // dedicated project), then fetch it back — the same create→read round-trip through the + // in-process MCP server. + const COMPONENT_ID = 'ex-generic-v2'; + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const created = await callToolText(session.client, 'create_config', { + name: `integ_roundtrip_${Date.now()}`, + description: 'Created by the mcp_server integration round-trip test.', + component_id: COMPONENT_ID, + parameters: { api: { baseUrl: 'https://example.com' } }, + }); + // The create output carries the new configuration_id. + const configurationId = created.match(/configuration_id:\s*"?([^\s"]+)"?/)?.[1]; + expect(configurationId, `create_config output had no configuration_id:\n${created}`).toBeTruthy(); + + const detail = await callToolText(session.client, 'get_configs', { + configs: [{ component_id: COMPONENT_ID, configuration_id: configurationId }], + }); + + // The detail output names the requested component + configuration. (Python validated + // the GetConfigsDetailOutput model fields; we assert on the TOON text shape.) + expect(detail).toContain(COMPONENT_ID); + expect(detail).toContain(configurationId!); + // Component metadata (type/name) is resolved and present. + expect(detail).toMatch(/component_type|component_name/); + } finally { + await session.close(); + } + }); +}); diff --git a/integtests/project_lock.py b/integtests/project_lock.py deleted file mode 100644 index 7d713c188..000000000 --- a/integtests/project_lock.py +++ /dev/null @@ -1,523 +0,0 @@ -""" -Distributed project lock backed by Keboola branch metadata. - -Prevents concurrent CI runners from corrupting shared integration-test data via -a write-and-verify window + oldest-timestamp-wins protocol. - -Lock state is represented by two metadata keys per runner: - Active : KBC.integtest.lock. (JSON payload) - Released : KBC.integtest.lock..released (ISO timestamp string) - -For a pool of projects, use ProjectPool, which tries each ProjectEndpoint in -order and returns the first one that can be locked. - -This file has no dependency on the keboola_mcp_server package — it only uses -httpx (already a project dependency) for direct Storage API calls. -""" - -import json -import logging -import os -import random -import socket -import time -import uuid -from dataclasses import dataclass -from datetime import datetime, timezone -from typing import Any - -import httpx - -LOG = logging.getLogger(__name__) - -LOCK_KEY_PREFIX = 'KBC.integtest.lock.' -DEFAULT_TTL_MINUTES = 60 -DEFAULT_POLL_INTERVAL_SECONDS = 30 -DEFAULT_MAX_WAIT_MINUTES = 90 -DEFAULT_ANTI_COLLISION_SECONDS = 3 - - -@dataclass(frozen=True) -class LockInfo: - lock_id: str - acquired_at: datetime # UTC, timezone-aware - runner_info: str - metadata_key: str # 'KBC.integtest.lock.' - - -@dataclass(frozen=True) -class ProjectEndpoint: - """A Keboola project identified by its Storage API URL, token, and metadata.""" - - storage_api_url: str - storage_api_token: str - project_id: str - project_name: str - token_id: str = '' - token_description: str = '' - - def describe(self) -> str: - """Return a human-readable summary of the project and token.""" - return ( - f'Authorized as "{self.token_description}" ({self.token_id}, ...{self.storage_api_token[-4:]}) ' - f'to project "{self.project_name}" ({self.project_id}) ' - f'at "{self.storage_api_url}" stack.' - ) - - -@dataclass(frozen=True) -class AcquiredProject: - """Result of acquiring a lock from a pool: which project was selected and its lock.""" - - endpoint: ProjectEndpoint - lock_info: LockInfo - - -def verify_project_endpoint( - storage_api_url: str, - storage_api_token: str, -) -> ProjectEndpoint: - """ - Verify a Storage API token and return a fully populated ProjectEndpoint. - - Calls GET /v2/storage/tokens/verify to confirm the token is valid and to - fetch the project name and ID. Raises httpx.HTTPStatusError if the token - is invalid or the request fails. - """ - base_url = storage_api_url.rstrip('/') - with httpx.Client(headers={'X-StorageApi-Token': storage_api_token}, timeout=30.0) as client: - resp = client.get(f'{base_url}/v2/storage/tokens/verify') - resp.raise_for_status() - token_info = resp.json() - token_id = str(token_info['id']) - token_description = str(token_info['description']) - project_id = str(token_info['owner']['id']) - project_name = token_info['owner']['name'] - LOG.info(f'[project_lock] Verified token ...{storage_api_token[-4:]} — project "{project_name}" ({project_id})') - return ProjectEndpoint( - storage_api_url=storage_api_url, - storage_api_token=storage_api_token, - project_id=project_id, - project_name=project_name, - token_id=token_id, - token_description=token_description, - ) - - -class ProjectLock: - def __init__( - self, - storage_api_url: str, - storage_api_token: str, - ttl_minutes: int = DEFAULT_TTL_MINUTES, - poll_interval_seconds: int = DEFAULT_POLL_INTERVAL_SECONDS, - max_wait_minutes: int = DEFAULT_MAX_WAIT_MINUTES, - anti_collision_seconds: int = DEFAULT_ANTI_COLLISION_SECONDS, - ) -> None: - self._base_url = storage_api_url.rstrip('/') - self._token = storage_api_token - self._ttl_minutes = ttl_minutes - self._poll_interval_seconds = poll_interval_seconds - self._max_wait_minutes = max_wait_minutes - self._anti_collision_seconds = anti_collision_seconds - - # ------------------------------------------------------------------ - # Public API - # ------------------------------------------------------------------ - - def acquire(self) -> LockInfo: - """Block until this runner owns the project lock; return LockInfo.""" - deadline = datetime.now(timezone.utc).timestamp() + self._max_wait_minutes * 60 - - while True: - if datetime.now(timezone.utc).timestamp() > deadline: - raise TimeoutError(f'Could not acquire project lock within {self._max_wait_minutes} minutes') - - result = self._try_acquire_once() - if result is not None: - return result - - LOG.info(f'[project_lock] Waiting {self._poll_interval_seconds}s before retrying.') - time.sleep(self._poll_interval_seconds) - - def release(self, lock: LockInfo) -> None: - """Mark the lock as released by writing the .released key.""" - released_key = lock.metadata_key + '.released' - LOG.info(f'[project_lock] Releasing lock {lock.lock_id}') - self._write_metadata({released_key: datetime.now(timezone.utc).isoformat()}) - - # ------------------------------------------------------------------ - # Internal helpers - # ------------------------------------------------------------------ - - def _try_acquire_once(self) -> LockInfo | None: - """ - Single non-looping acquisition attempt. - - Returns LockInfo if this runner wins the lock (including after cleaning - a stale lock). Returns None if another runner holds an active non-stale - lock — signals the caller to try a different project or wait and retry. - """ - runner_info = self._runner_info() - lock_id = str(uuid.uuid4()) - acquired_at = datetime.now(timezone.utc) - key = LOCK_KEY_PREFIX + lock_id - payload = json.dumps( - { - 'lock_id': lock_id, - 'acquired_at': acquired_at.isoformat(), - 'runner_info': runner_info, - } - ) - LOG.info(f'[project_lock] Writing candidate lock {lock_id} (runner: {runner_info})') - self._write_metadata({key: payload}) - - # Anti-collision window: let concurrent writers finish their writes - time.sleep(self._anti_collision_seconds) - - active = self._read_active_locks() - winner = min(active, key=lambda li: (li.acquired_at, li.lock_id)) if active else None - - # Case 1: We are the winner - if winner is not None and winner.lock_id == lock_id: - LOG.info(f'[project_lock] Acquired lock {lock_id}') - self._cleanup_old_locks(lock_id) - self.clean_project() - return LockInfo( - lock_id=lock_id, - acquired_at=acquired_at, - runner_info=runner_info, - metadata_key=key, - ) - - # Case 2: The winner is stale — clean up and retry immediately once - if winner is not None and self._is_stale(winner): - LOG.warning( - f'[project_lock] Stale lock detected: {winner.lock_id} ' - f'(acquired_at={winner.acquired_at.isoformat()}). ' - 'Releasing stale entries and cleaning project.' - ) - for stale in active: - if self._is_stale(stale): - self._release_lock_entry(stale.lock_id) - # Release our pending entry and write a fresh candidate - self._release_lock_entry(lock_id) - time.sleep(2) - - lock_id2 = str(uuid.uuid4()) - acquired_at2 = datetime.now(timezone.utc) - key2 = LOCK_KEY_PREFIX + lock_id2 - payload2 = json.dumps( - { - 'lock_id': lock_id2, - 'acquired_at': acquired_at2.isoformat(), - 'runner_info': runner_info, - } - ) - LOG.info(f'[project_lock] Post-stale-clean: writing candidate lock {lock_id2}') - self._write_metadata({key2: payload2}) - time.sleep(self._anti_collision_seconds) - - active2 = self._read_active_locks() - winner2 = min(active2, key=lambda li: (li.acquired_at, li.lock_id)) if active2 else None - if winner2 is not None and winner2.lock_id == lock_id2: - LOG.info(f'[project_lock] Acquired lock {lock_id2} after stale cleanup') - self._cleanup_old_locks(lock_id2) - return LockInfo( - lock_id=lock_id2, - acquired_at=acquired_at2, - runner_info=runner_info, - metadata_key=key2, - ) - # Another runner raced us after the cleanup — withdraw and signal the caller - self._release_lock_entry(lock_id2) - return None - - # Case 3: Another runner holds an active (non-stale) lock - if winner is not None: - expires_approx = winner.acquired_at.timestamp() + self._ttl_minutes * 60 - expires_str = datetime.fromtimestamp(expires_approx, tz=timezone.utc).isoformat() - LOG.info( - f'[project_lock] Lock held by {winner.runner_info} ' - f'(id={winner.lock_id}), expires ~{expires_str}. ' - 'Releasing our candidate.' - ) - else: - LOG.info('[project_lock] No winner determined yet. Releasing candidate.') - self._release_lock_entry(lock_id) - return None - - def _write_metadata(self, kv: dict[str, str]) -> None: - payload = {'metadata': [{'key': k, 'value': v} for k, v in kv.items()]} - self._post('/v2/storage/branch/default/metadata', data=payload) - - def _read_metadata(self) -> list[dict[str, Any]]: - return self._get('/v2/storage/branch/default/metadata') - - def _read_active_locks(self) -> list[LockInfo]: - """Return all active (not-released) lock entries, sorted by acquired_at ASC.""" - entries = self._read_metadata() - - # Build sets of known lock UUIDs and released UUIDs - lock_entries: dict[str, dict[str, Any]] = {} # lock_id -> raw metadata entry - released_ids: set[str] = set() - - for entry in entries: - key: str = entry.get('key', '') - if not key.startswith(LOCK_KEY_PREFIX): - continue - suffix = key[len(LOCK_KEY_PREFIX) :] - if suffix.endswith('.released'): - released_ids.add(suffix[: -len('.released')]) - else: - lock_entries[suffix] = entry - - active: list[LockInfo] = [] - for lock_id, entry in lock_entries.items(): - if lock_id in released_ids: - continue - try: - data = json.loads(entry['value']) - acquired_at = datetime.fromisoformat(data['acquired_at']) - if acquired_at.tzinfo is None: - acquired_at = acquired_at.replace(tzinfo=timezone.utc) - active.append( - LockInfo( - lock_id=lock_id, - acquired_at=acquired_at, - runner_info=data.get('runner_info', ''), - metadata_key=LOCK_KEY_PREFIX + lock_id, - ) - ) - except (KeyError, ValueError, json.JSONDecodeError) as exc: - LOG.warning(f'[project_lock] Skipping malformed lock entry {lock_id!r}: {exc}') - - return sorted(active, key=lambda li: (li.acquired_at, li.lock_id)) - - def _is_stale(self, lock: LockInfo) -> bool: - expiry = lock.acquired_at.timestamp() + self._ttl_minutes * 60 - return expiry <= datetime.now(timezone.utc).timestamp() - - def _release_lock_entry(self, lock_id: str) -> None: - released_key = LOCK_KEY_PREFIX + lock_id + '.released' - self._write_metadata({released_key: datetime.now(timezone.utc).isoformat()}) - - def _delete_metadata_by_id(self, metadata_id: str) -> None: - """Delete a single branch metadata entry by its numeric Storage API id.""" - self._delete(f'/v2/storage/branch/default/metadata/{metadata_id}') - - def _cleanup_old_locks(self, current_lock_id: str) -> None: - """ - Delete all released lock metadata entries to prevent metadata accumulation. - - Deletion order per pair: main entry first, then .released entry. - This prevents transient reappearance of a released lock as active. - Errors are swallowed so cleanup never breaks lock acquisition. - """ - try: - entries = self._read_metadata() - except Exception as exc: - LOG.warning(f'[project_lock] _cleanup_old_locks: failed to read metadata: {exc}') - return - - main_entries: dict[str, dict[str, Any]] = {} # lock_id -> raw entry - released_entries: dict[str, dict[str, Any]] = {} # lock_id -> raw entry - - for entry in entries: - key: str = entry.get('key', '') - if not key.startswith(LOCK_KEY_PREFIX): - continue - suffix = key[len(LOCK_KEY_PREFIX) :] - if suffix.endswith('.released'): - lock_id = suffix[: -len('.released')] - released_entries[lock_id] = entry - else: - main_entries[suffix] = entry - - for lock_id, released_entry in released_entries.items(): - if lock_id == current_lock_id: - continue # never touch the active lock - main_entry = main_entries.get(lock_id) - if main_entry is not None: - # Delete main entry FIRST (safety: prevents transient reactivation) - try: - LOG.info(f'[project_lock] Cleaning up released lock {lock_id} (main)') - self._delete_metadata_by_id(str(main_entry['id'])) - except Exception as exc: - LOG.warning( - f'[project_lock] Failed to delete main lock entry {lock_id}: {exc}. ' - 'Skipping .released deletion to avoid reactivating the lock.' - ) - continue - # Delete .released entry SECOND — only if main was successfully deleted - try: - LOG.info(f'[project_lock] Cleaning up released lock {lock_id} (.released)') - self._delete_metadata_by_id(str(released_entry['id'])) - except Exception as exc: - LOG.warning(f'[project_lock] Failed to delete .released entry {lock_id}: {exc}') - else: - # Orphaned .released entry — main was already deleted or never written - try: - LOG.info(f'[project_lock] Cleaning up orphaned .released entry {lock_id}') - self._delete_metadata_by_id(str(released_entry['id'])) - except Exception as exc: - LOG.warning(f'[project_lock] Failed to delete orphaned .released entry {lock_id}: {exc}') - - def clean_project(self) -> None: - """Delete all buckets, component configurations, and workspaces from the project. - - Integration tests no longer rely on a persistent workspace — each session creates its - own read-only workspace and deletes it on teardown — so cleaning wipes everything, - including any workspace leaked by a crashed run. - """ - LOG.info('[project_lock] Cleaning project (deleting all buckets, configs and workspaces)') - - # Delete all buckets (force=True also removes tables inside them) - buckets = self._get('/v2/storage/buckets') - for bucket in buckets: - bucket_id = bucket['id'] - LOG.info(f'[project_lock] Deleting bucket {bucket_id}') - self._delete(f'/v2/storage/buckets/{bucket_id}', force='true') - - # Delete all component configurations - components = self._get('/v2/storage/components', include='configuration') - for component in components: - comp_id = component['id'] - for cfg in component.get('configurations', []): - cfg_id = cfg['id'] - LOG.info(f'[project_lock] Deleting config {comp_id}/{cfg_id}') - # First delete moves to trash; second delete removes from trash - self._delete(f'/v2/storage/components/{comp_id}/configs/{cfg_id}') - self._delete(f'/v2/storage/components/{comp_id}/configs/{cfg_id}') - - # Delete all workspaces (none are persistent anymore) - workspaces = self._get('/v2/storage/branch/default/workspaces') - for workspace in workspaces: - workspace_id = workspace['id'] - LOG.info(f'[project_lock] Deleting workspace {workspace_id}') - try: - self._delete(f'/v2/storage/workspaces/{workspace_id}') - except httpx.HTTPStatusError as exc: - # A workspace backed by a sandbox config may already be gone after config deletion. - LOG.warning(f'[project_lock] Failed to delete workspace {workspace_id}: {exc}') - - @staticmethod - def _runner_info() -> str: - hostname = socket.gethostname() - pid = os.getpid() - base = f'{hostname}/{pid}' - run_id = os.getenv('GITHUB_RUN_ID') - if run_id: - return f'CI={run_id} {base}' - return base - - # ------------------------------------------------------------------ - # HTTP helpers - # ------------------------------------------------------------------ - - def _client(self) -> httpx.Client: - return httpx.Client( - headers={'X-StorageApi-Token': self._token}, - timeout=30.0, - ) - - def _get(self, path: str, **params: Any) -> Any: - with self._client() as client: - resp = client.get(self._base_url + path, params=params or None) - resp.raise_for_status() - return resp.json() - - def _post(self, path: str, data: dict[str, Any]) -> Any: - with self._client() as client: - resp = client.post(self._base_url + path, json=data) - resp.raise_for_status() - return resp.json() - - def _delete(self, path: str, **params: Any) -> None: - with self._client() as client: - resp = client.delete(self._base_url + path, params=params or None) - resp.raise_for_status() - - -class ProjectPool: - """ - Manages a pool of Keboola projects for integration tests. - - Tries each endpoint in order on every acquisition pass; returns the first - project whose lock can be acquired. If all projects are held by active - runners, sleeps poll_interval_seconds and retries the whole pool. - Raises TimeoutError after max_wait_minutes. - - Stale locks on any project are detected and cleaned automatically before - claiming that project (handled inside ProjectLock._try_acquire_once). - """ - - def __init__( - self, - endpoints: list[ProjectEndpoint], - ttl_minutes: int = DEFAULT_TTL_MINUTES, - poll_interval_seconds: int = DEFAULT_POLL_INTERVAL_SECONDS, - max_wait_minutes: int = DEFAULT_MAX_WAIT_MINUTES, - anti_collision_seconds: int = DEFAULT_ANTI_COLLISION_SECONDS, - ) -> None: - if not endpoints: - raise ValueError('ProjectPool requires at least one endpoint') - self._endpoints = endpoints - self._ttl_minutes = ttl_minutes - self._poll_interval_seconds = poll_interval_seconds - self._max_wait_minutes = max_wait_minutes - self._anti_collision_seconds = anti_collision_seconds - - def acquire(self) -> AcquiredProject: - """ - Try each endpoint in order; return the first one successfully locked. - Retries the whole pool until max_wait_minutes is exceeded. - """ - deadline = datetime.now(timezone.utc).timestamp() + self._max_wait_minutes * 60 - - while True: - if datetime.now(timezone.utc).timestamp() > deadline: - raise TimeoutError( - f'Could not acquire any project lock within {self._max_wait_minutes} minutes ' - f'(pool size: {len(self._endpoints)})' - ) - - start = random.randrange(len(self._endpoints)) - rotated = self._endpoints[start:] + self._endpoints[:start] - for endpoint in rotated: - LOG.info( - f'[project_pool] Trying to acquire lock for ' - f'"{endpoint.project_name}" ({endpoint.project_id}) (...{endpoint.storage_api_token[-4:]})' - ) - lock_info = self._make_lock(endpoint)._try_acquire_once() - if lock_info is not None: - LOG.info( - f'[project_pool] Acquired project ' - f'"{endpoint.project_name}" ({endpoint.project_id}) (...{endpoint.storage_api_token[-4:]})' - ) - return AcquiredProject(endpoint=endpoint, lock_info=lock_info) - - LOG.info( - f'[project_pool] All {len(self._endpoints)} projects busy. ' - f'Sleeping {self._poll_interval_seconds}s before retry.' - ) - time.sleep(self._poll_interval_seconds) - - def release(self, acquired: AcquiredProject) -> None: - """Release the lock held on acquired.endpoint.""" - LOG.info( - f'[project_pool] Releasing lock on ' - f'"{acquired.endpoint.project_name}" ({acquired.endpoint.project_id})' - f' (...{acquired.endpoint.storage_api_token[-4:]})' - ) - self._make_lock(acquired.endpoint).release(acquired.lock_info) - - def _make_lock(self, endpoint: ProjectEndpoint) -> ProjectLock: - return ProjectLock( - storage_api_url=endpoint.storage_api_url, - storage_api_token=endpoint.storage_api_token, - ttl_minutes=self._ttl_minutes, - poll_interval_seconds=self._poll_interval_seconds, - max_wait_minutes=self._max_wait_minutes, - anti_collision_seconds=self._anti_collision_seconds, - ) diff --git a/integtests/test_errors.py b/integtests/test_errors.py deleted file mode 100644 index f797fe499..000000000 --- a/integtests/test_errors.py +++ /dev/null @@ -1,179 +0,0 @@ -import asyncio -import json -import math -import re -import uuid -from importlib.metadata import distribution -from typing import Any, Mapping -from urllib.parse import urlparse - -import httpx -import pytest -from fastmcp import Context -from mcp.types import ClientCapabilities, Implementation, InitializeRequestParams - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.mcp import CONVERSATION_ID, AggregateError -from keboola_mcp_server.tools.doc import docs_query -from keboola_mcp_server.tools.jobs import get_jobs -from keboola_mcp_server.tools.sql import query_data -from keboola_mcp_server.tools.storage.tools import GetBucketsOutput, get_buckets - - -class TestHttpErrors: - """Test different HTTP error scenarios to ensure enhanced error handling works correctly.""" - - @pytest.mark.asyncio - async def test_storage_api_404_error_maintains_standard_behavior(self, mcp_context: Context): - result = await get_buckets(mcp_context, ['non.existent.bucket']) - assert 'non.existent.bucket' in result.buckets_not_found - - @pytest.mark.asyncio - async def test_jobs_api_404_error_(self, mcp_context: Context, storage_api_url: str): - hostname_suffix = urlparse(storage_api_url).hostname.split('connection.')[1] - queue_url = f'https://queue.{hostname_suffix}' - match = re.compile( - r"Client error '404 [^']+' " - rf"for url '{re.escape(queue_url)}/jobs/999999999'\n" - r'For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n' - r'API error: Job "999999999" not found\n' - r'Exception ID: .+\n' - r'When contacting Keboola support please provide the exception ID\.', - re.IGNORECASE, - ) - with pytest.raises(AggregateError) as exc_info: - await get_jobs(ctx=mcp_context, job_ids=('999999999',)) - - # Verify AggregateError contains the HTTPStatusError - err = exc_info.value - assert len(err.exceptions) == 1 - assert isinstance(err.exceptions[0], httpx.HTTPStatusError) - assert match.search(str(err.exceptions[0])) is not None - - @pytest.mark.asyncio - async def test_docs_api_empty_query_error(self, mcp_context: Context, storage_api_url: str): - """Test that docs_query raises 422 error for empty queries.""" - hostname_suffix = urlparse(storage_api_url).hostname.split('connection.')[1] - ai_url = f'https://ai.{hostname_suffix}' - match = re.compile( - r"Client error '422 [^']+' " - rf"for url '{re.escape(ai_url)}/docs/question'\n" - r'For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422\n' - r'API error: Request contents is not valid\n' - r'Exception ID: .+\n' - r'When contacting Keboola support please provide the exception ID\.', - re.IGNORECASE, - ) - with pytest.raises(httpx.HTTPStatusError, match=match): - await docs_query(ctx=mcp_context, query='') - - @pytest.mark.asyncio - async def test_sql_api_invalid_query_error_snowflake(self, mcp_context: Context, require_snowflake: None): - match = re.compile( - r'Failed to run SQL query, error: SQL compilation error:\n' - r"syntax error line 1 at position 0 unexpected 'INVALID'\.", - re.IGNORECASE, - ) - with pytest.raises(ValueError, match=match): - await query_data('INVALID SQL SYNTAX HERE', 'Invalid SQL query.', mcp_context) - - @pytest.mark.asyncio - async def test_sql_api_invalid_query_error_bigquery(self, mcp_context: Context, require_bigquery: None): - match = re.compile( - r'Failed to run SQL query, error: Syntax error: Unexpected identifier "INVALID" at \[1:1]', - re.IGNORECASE, - ) - with pytest.raises(ValueError, match=match): - await query_data('INVALID SQL SYNTAX HERE', 'Invalid SQL query.', mcp_context) - - @pytest.mark.asyncio - async def test_concurrent_error_handling(self, mcp_context: Context): - # Run multiple concurrent operations that will trigger 404 errors - tasks = [get_buckets(mcp_context, [f'non.existent.bucket.{i}']) for i in range(5)] - results = await asyncio.gather(*tasks, return_exceptions=True) - - # Verify all errors are handled consistently - pattern = re.compile(r'non\.existent\.bucket\.\d+', re.IGNORECASE) - - for result in results: - assert isinstance(result, GetBucketsOutput) - assert result.buckets_not_found - assert len(result.buckets_not_found) == 1 - assert pattern.fullmatch(result.buckets_not_found[0]) - - -class TestStorageEvents: - @staticmethod - @tool_errors() - async def foo(unique: str, ctx: Context): - """A fake MCP tool to test events emitting.""" - await asyncio.sleep(0.1) - - @staticmethod - @tool_errors() - async def bar(unique: str, ctx: Context): - """A fake MCP tool that fails by raising an error to test events emitting.""" - await asyncio.sleep(0.1) - raise ValueError('Intentional error in bar tool.') - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('tool_name', 'event_message', 'event_type'), - [ - ('foo', 'MCP tool "foo" call succeeded.', 'success'), - ('bar', 'MCP tool "bar" call failed. ValueError: Intentional error in bar tool.', 'error'), - ], - ) - async def test_event_emitted(self, tool_name: str, event_message: str, event_type: str, mcp_context: Context): - mcp_context.session_id = 'deadbee' - mcp_context.session.client_params = InitializeRequestParams( - protocolVersion='1', - capabilities=ClientCapabilities(), - clientInfo=Implementation(name='kbc-mcp-integtests-test-events-emitted', version='1.2.3'), - ) - mcp_context.session.state[CONVERSATION_ID] = '#987654321' - unique = uuid.uuid4().hex - tool_func = getattr(self, tool_name) - try: - await tool_func(unique, mcp_context) - except ValueError: - pass # ignore - await asyncio.sleep(1) # give SAPI time to digest the event - - client = KeboolaClient.from_state(mcp_context.session.state) - events = await client.storage_client.get( - endpoint='events', - params={ - 'component': 'keboola.mcp-server-tool', - 'q': f'message:"MCP tool "{tool_name}" call*"', - 'limit': 10, - }, - ) - emitted_event = self._find_event(events, tool_name=tool_name, param_name='unique', param_value=unique) - assert emitted_event is not None - assert emitted_event['message'] == event_message - assert emitted_event['type'] == event_type - # SAPI events don't support float durations, so the duration is rounded up to the nearest second. - assert math.fabs(emitted_event['performance']['duration'] - 0.1) < 1 - assert emitted_event['params']['mcpServerContext'] == { - 'appEnv': 'DEV', - 'version': distribution('keboola_mcp_server').version, - 'userAgent': 'kbc-mcp-integtests-test-events-emitted/1.2.3', - 'sessionId': 'deadbee', - 'serverTransport': 'stdio', - 'conversationId': '#987654321', - } - - @staticmethod - def _find_event( - events: list[Mapping[str, Any]], *, tool_name: str, param_name: str, param_value: str - ) -> Mapping[str, Any] | None: - for event in events: - event_tool = event['params']['tool'] - if event_tool['name'] != tool_name: - continue - for argument in event_tool['arguments']: - if argument['key'] == param_name and json.loads(argument['value']) == param_value: - return event - return None diff --git a/integtests/test_mcp_server.py b/integtests/test_mcp_server.py deleted file mode 100644 index 9ea7ff3c8..000000000 --- a/integtests/test_mcp_server.py +++ /dev/null @@ -1,330 +0,0 @@ -import logging -import os -import random -import subprocess -import time -from contextlib import asynccontextmanager -from typing import Any, AsyncGenerator, Iterable, Literal, cast - -import pytest -from fastmcp import Client -from fastmcp.client import StdioTransport, StreamableHttpTransport -from mcp.types import TextContent - -from integtests.conftest import ( - DEV_STORAGE_API_URL_ENV_VAR, - DEV_STORAGE_TOKEN_ENV_VAR, - DEV_WORKSPACE_SCHEMA_ENV_VAR, - ConfigDef, -) -from keboola_mcp_server.tools.components.model import GetConfigsDetailOutput -from keboola_mcp_server.tools.project import ProjectInfo - -LOG = logging.getLogger(__name__) -HttpTransportStr = Literal['streamable-http', 'http-compat'] - - -@pytest.mark.asyncio -async def test_stdio_setup( - configs: list[ConfigDef], - storage_api_token: str, - workspace_schema: str, - storage_api_url: str, -): - assert storage_api_token is not None - assert workspace_schema is not None - assert storage_api_url is not None - - transport = StdioTransport( - command='python', - args=[ - '-m', - 'keboola_mcp_server', - '--api-url', - storage_api_url, - '--storage-token', - storage_api_token, - '--workspace-schema', - workspace_schema, - ], - env={}, # make sure no env vars are passed from the test environment - ) - component_config = configs[0] - async with Client(transport) as client: - await _assert_basic_setup(client) - await _assert_get_component_details_tool_call(client, component_config) - - -@pytest.mark.asyncio -@pytest.mark.parametrize('transport', ['streamable-http', 'http-compat']) -async def test_remote_setup( - transport: HttpTransportStr, - configs: list[ConfigDef], - storage_api_token: str, - workspace_schema: str, - storage_api_url: str, -): - assert storage_api_token is not None - assert workspace_schema is not None - assert storage_api_url is not None - - component_config = configs[0] - for url in _run_server_remote(storage_api_url, transport): - # test both cases: with headers and without headers using query params - headers = {'storage_token': storage_api_token, 'workspace_schema': workspace_schema} - async with _run_client(url, headers) as client: - await _assert_basic_setup(client) - await _assert_get_component_details_tool_call(client, component_config) - - -@pytest.mark.asyncio -async def test_http_multiple_clients( - configs: list[ConfigDef], - storage_api_token: str, - workspace_schema: str, - storage_api_url: str, -): - transport: HttpTransportStr = 'streamable-http' - component_config = configs[0] - for url in _run_server_remote(storage_api_url, transport): - headers = { - 'storage_token': storage_api_token, - 'workspace_schema': workspace_schema, - 'storage_api_url': storage_api_url, - } - async with ( - _run_client(url, headers) as client_1, - _run_client(url, headers) as client_2, - _run_client(url, headers) as client_3, - ): - await _assert_basic_setup(client_1) - await _assert_basic_setup(client_2) - await _assert_basic_setup(client_3) - await _assert_get_component_details_tool_call(client_1, component_config) - await _assert_get_component_details_tool_call(client_2, component_config) - await _assert_get_component_details_tool_call(client_3, component_config) - - -@pytest.mark.asyncio -async def test_http_multiple_clients_with_different_headers( - storage_api_url: str, - storage_api_token: str, - workspace_schema: str, - storage_api_token_2: str | None, - workspace_schema_2: str | None, -): - """ - Test that the server can handle multiple clients with different headers and checks the values of the headers. - - This test accesses the second project (PRJ2) without a lock. That is safe only because - it is strictly read-only against PRJ2 (list_tools, list_resources, get_project_info). - Do not add any writes to PRJ2 here without acquiring a lock for it first. - """ - if not storage_api_token_2 or not workspace_schema_2: - pytest.skip('No SAPI token or workspace schema for the second client. Skipping test.') - - headers = { - 'client_1': {'storage_token': storage_api_token, 'workspace_schema': workspace_schema}, - 'client_2': {'storage_token': storage_api_token_2, 'workspace_schema': workspace_schema_2}, - } - - transport: HttpTransportStr = 'streamable-http' - for url in _run_server_remote(storage_api_url, transport): - async with ( - _run_client(url, headers['client_1']) as client_1, - _run_client(url, headers['client_2']) as client_2, - ): - await _assert_basic_setup(client_1) - await _assert_basic_setup(client_2) - - response_1 = await client_1.call_tool('get_project_info') - project_info_1 = ProjectInfo.model_validate(response_1.structured_content) - project_info_1.project_id = storage_api_token.split(sep='-')[0] - LOG.info(f'project_info_1={project_info_1}') - - response_2 = await client_2.call_tool('get_project_info') - project_info_2 = ProjectInfo.model_validate(response_2.structured_content) - project_info_2.project_id = storage_api_token_2.split(sep='-')[0] - LOG.info(f'project_info_2={project_info_2}') - - -async def _assert_basic_setup(client: Client): - tools = await client.list_tools() - # the create_conditional_flow, create_flow, search, and semantic tools may not be present - # based on the testing project features - exclude = { - 'create_conditional_flow', - 'create_flow', - 'search', - 'update_flow', - 'modify_flow', - 'get_semantic_context', - 'get_semantic_schema', - 'search_semantic_context', - 'validate_semantic_query', - } - expected_tools = { - 'add_config_row', - 'create_conditional_flow', - 'create_config', - 'create_flow', - 'create_oauth_url', - 'create_python_js_data_app_git_credential', - 'create_sql_transformation', - 'delete_python_js_data_app_draft', - 'deploy_data_app', - 'docs_query', - 'find_component_id', - 'get_buckets', - 'get_components', - 'get_config_examples', - 'get_configs', - 'get_data_apps', - 'get_flow_examples', - 'get_flow_schema', - 'get_flows', - 'get_jobs', - 'get_project_info', - 'get_tables', - 'modify_flow', - 'modify_python_js_data_app', - 'modify_streamlit_data_app', - 'query_data', - 'run_job', - 'run_sync_action', - 'search', - 'update_config', - 'update_config_row', - 'update_descriptions', - 'update_flow', - 'update_project_description', - 'update_sql_transformation', - } - expected_tools = expected_tools - exclude - - actual_tools = {tool.name for tool in tools} - actual_tools = actual_tools - exclude - - missing_tools = expected_tools - actual_tools - assert not missing_tools, f'Missing tools: {missing_tools}' - - unexpected_tools = actual_tools - expected_tools - assert not unexpected_tools, f'Unexpected new tools: {unexpected_tools}' - - prompts = await client.list_prompts() - assert len(prompts) == 6 - - # there are no resources exposed in the MCP server; just check that the call succeeds - resources = await client.list_resources() - assert len(resources) == 0 - - -async def _assert_get_component_details_tool_call(client: Client, config: ConfigDef): - assert config.configuration_id is not None - - tool_result = await client.call_tool( - 'get_configs', - {'configs': [{'configuration_id': config.configuration_id, 'component_id': config.component_id}]}, - ) - - assert tool_result is not None - assert len(tool_result.content) == 1 - tool_result_content = tool_result.content[0] - assert isinstance(tool_result_content, TextContent) # only one tool call is executed - - component_configs = GetConfigsDetailOutput.model_validate( - cast(dict[str, Any], tool_result.structured_content)['result'] - ) - assert len(component_configs.configs) == 1 - component_config = component_configs.configs[0] - assert component_config.component is not None - assert component_config.component.component_id == config.component_id - assert component_config.component.component_type is not None - assert component_config.component.component_name is not None - - assert component_config.configuration_root is not None - assert component_config.configuration_root.configuration_id == config.configuration_id - - assert component_config.configuration_rows is None - - -def _run_server_remote(storage_api_url: str, transport: HttpTransportStr) -> Iterable[str]: - """ - Run the server in a subprocess. - :param storage_api_url: The Storage API URL to use. - :param transport: The transport to use. - :return: The url of the remote server. - """ - - port = random.randint(8000, 9000) - p = subprocess.Popen( - [ - 'python', - '-m', - 'keboola_mcp_server', - '--transport', - transport, - '--api-url', - storage_api_url, - '--port', - str(port), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env={ - name: val - for name, val in os.environ.items() - if name not in [DEV_STORAGE_API_URL_ENV_VAR, DEV_STORAGE_TOKEN_ENV_VAR, DEV_WORKSPACE_SCHEMA_ENV_VAR] - }, - ) - try: - urls: list[str] = [] - if transport in ['streamable-http', 'http-compat']: - urls.append(f'http://127.0.0.1:{port}/mcp') - if not urls: - raise ValueError(f'Unknown transport: {transport}') - - LOG.info(f'Running MCP server in subprocess with {transport} transport, listening on: {urls}') - time.sleep(5) # wait for the server to start - yield from urls - finally: - LOG.info('Terminating MCP server subprocess.') - p.terminate() - stdout, stderr = p.communicate() - LOG.info(f'-- MCP server stdout --\n{stdout}\n-- end stdout --') - LOG.info(f'-- MCP server stderr --\n{stderr}\n-- end stderr --') - - -@asynccontextmanager -async def _run_client(url: str, headers: dict[str, str] | None = None) -> AsyncGenerator[Client, None]: - """ - Run the client in an async context manager which will ensure that the client is properly closed after the test. - The client is created with the given transport and connected to the url of the remote server with which it - communicates. - :param url: The url of the remote server to which the client will be connected. - :param headers: The headers to use for the client. - :return: The Client connected to the remote server. - """ - if url.endswith('/mcp'): - transport = StreamableHttpTransport(url=url, headers=headers) - else: - raise ValueError(f'Unknown transport: {url}') - - client_explicit = Client(transport) - exception_from_client = None - - LOG.info(f'Running MCP client connecting to {url} and expecting `{transport}` server transport.') - try: - async with client_explicit: - try: - yield client_explicit - except Exception as e: - LOG.error(f'Error in client TaskGroup: {e}') - exception_from_client = e - # we need to keep an exception from the client TaskGroup and raise it - # outside the context manager, otherwise it will inform only about task group error - finally: - del client_explicit - if isinstance(exception_from_client, Exception): - raise exception_from_client diff --git a/integtests/test_project_lock.py b/integtests/test_project_lock.py deleted file mode 100644 index 2d6f77336..000000000 --- a/integtests/test_project_lock.py +++ /dev/null @@ -1,1138 +0,0 @@ -""" -Unit tests for integtests/project_lock.py. - -All HTTP calls are mocked via pytest-mock — no real Storage API or Keboola project is -ever touched. These tests do not require any INTEGTEST_* environment variables. -""" - -import json -import os -import socket -from datetime import datetime, timedelta, timezone -from typing import Any -from unittest.mock import MagicMock, call - -import httpx -import pytest - -from integtests.project_lock import ( - LOCK_KEY_PREFIX, - AcquiredProject, - LockInfo, - ProjectEndpoint, - ProjectLock, - ProjectPool, - verify_project_endpoint, -) - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _make_lock( - *, - lock_id: str = 'test-uuid', - minutes_ago: float = 0, - runner_info: str = 'host/1', - meta_id: int | None = None, -) -> dict: - """Build a raw branch-metadata dict as returned by the Storage API.""" - acquired_at = datetime.now(timezone.utc) - timedelta(minutes=minutes_ago) - payload = json.dumps( - { - 'lock_id': lock_id, - 'acquired_at': acquired_at.isoformat(), - 'runner_info': runner_info, - } - ) - result = {'key': LOCK_KEY_PREFIX + lock_id, 'value': payload} - if meta_id is not None: - result['id'] = meta_id - return result - - -def _released_entry(lock_id: str, meta_id: int | None = None) -> dict: - result = { - 'key': LOCK_KEY_PREFIX + lock_id + '.released', - 'value': datetime.now(timezone.utc).isoformat(), - } - if meta_id is not None: - result['id'] = meta_id - return result - - -def _make_project_lock(**kwargs) -> ProjectLock: - defaults = dict( - storage_api_url='https://connection.keboola.com', - storage_api_token='test-token', - ttl_minutes=60, - poll_interval_seconds=1, - max_wait_minutes=2, - anti_collision_seconds=0, - ) - defaults.update(kwargs) - return ProjectLock(**defaults) - - -# --------------------------------------------------------------------------- -# test_acquire_happy_path -# --------------------------------------------------------------------------- - - -def test_acquire_happy_path(mocker): - """Single runner: writes lock, reads it back as winner, returns LockInfo.""" - lock = _make_project_lock() - my_lock_id = 'aaaaaaaa-0000-0000-0000-000000000001' - - post_mock = mocker.patch.object(lock, '_post', return_value=[]) - mocker.patch('uuid.uuid4', return_value=MagicMock(__str__=lambda _: my_lock_id)) - mocker.patch.object(lock, 'clean_project') - - # _read_metadata returns only our own entry after the anti-collision sleep - my_entry = _make_lock(lock_id=my_lock_id, minutes_ago=0) - mocker.patch.object(lock, '_read_metadata', return_value=[my_entry]) - mocker.patch('time.sleep') - - result = lock.acquire() - - assert isinstance(result, LockInfo) - assert result.lock_id == my_lock_id - assert result.metadata_key == LOCK_KEY_PREFIX + my_lock_id - # Verify that the lock key was written - written_keys = [ - entry['key'] - for call_args in post_mock.call_args_list - for entry in call_args.kwargs.get('data', {}).get('metadata', []) - ] - assert LOCK_KEY_PREFIX + my_lock_id in written_keys - - -# --------------------------------------------------------------------------- -# test_acquire_anti_collision_waits -# --------------------------------------------------------------------------- - - -def test_acquire_anti_collision_waits(mocker): - """time.sleep is called with anti_collision_seconds before reading back.""" - anti_collision = 3 - lock = _make_project_lock(anti_collision_seconds=anti_collision) - my_lock_id = 'aaaaaaaa-0000-0000-0000-000000000002' - - mocker.patch.object(lock, '_post', return_value=[]) - mocker.patch('uuid.uuid4', return_value=MagicMock(__str__=lambda _: my_lock_id)) - mocker.patch.object(lock, 'clean_project') - - my_entry = _make_lock(lock_id=my_lock_id, minutes_ago=0) - mocker.patch.object(lock, '_read_metadata', return_value=[my_entry]) - sleep_mock = mocker.patch('time.sleep') - - lock.acquire() - - # At minimum one sleep with the anti-collision value must have occurred - assert any(c == call(anti_collision) for c in sleep_mock.call_args_list) - - -# --------------------------------------------------------------------------- -# test_acquire_win_oldest_timestamp -# --------------------------------------------------------------------------- - - -def test_acquire_win_oldest_timestamp(mocker): - """Two active entries; ours is the oldest → we win.""" - lock = _make_project_lock() - my_lock_id = 'aaaaaaaa-0000-0000-0000-aaaaaaaaaaaa' - other_lock_id = 'aaaaaaaa-0000-0000-0000-bbbbbbbbbbbb' - - mocker.patch.object(lock, '_post', return_value=[]) - mocker.patch('uuid.uuid4', return_value=MagicMock(__str__=lambda _: my_lock_id)) - mocker.patch.object(lock, 'clean_project') - - # Our entry is 5 minutes older than the other - my_entry = _make_lock(lock_id=my_lock_id, minutes_ago=5) - other_entry = _make_lock(lock_id=other_lock_id, minutes_ago=0) - mocker.patch.object(lock, '_read_metadata', return_value=[my_entry, other_entry]) - mocker.patch('time.sleep') - - result = lock.acquire() - assert result.lock_id == my_lock_id - - -# --------------------------------------------------------------------------- -# test_acquire_lose_to_older -# --------------------------------------------------------------------------- - - -def test_acquire_lose_to_older(mocker): - """ - Two active entries; theirs is older → we release our candidate, sleep, then - on the second iteration ours is oldest and we acquire. - """ - lock = _make_project_lock(poll_interval_seconds=1, max_wait_minutes=5) - - my_lock_id_1 = 'my-lock-id-0001' - my_lock_id_2 = 'my-lock-id-0002' - other_lock_id = 'other-lock-id-00' - - uuid_iter = iter([my_lock_id_1, my_lock_id_2]) - mocker.patch('uuid.uuid4', side_effect=lambda: MagicMock(__str__=lambda _: next(uuid_iter))) - - post_mock = mocker.patch.object(lock, '_post', return_value=[]) - mocker.patch.object(lock, 'clean_project') - - other_entry = _make_lock(lock_id=other_lock_id, minutes_ago=10) - - def metadata_side_effect(): - # Check how many times _post was called to determine which iteration we're in - release_calls = [ - c - for c in post_mock.call_args_list - if any( - k.endswith('.released') - for entry in c.kwargs.get('data', {}).get('metadata', []) - for k in [entry['key']] - ) - ] - if not release_calls: - # First read: both entries active, other is older - my_entry = _make_lock(lock_id=my_lock_id_1, minutes_ago=0) - return [other_entry, my_entry] - else: - # Second iteration: only our new entry (other runner released theirs) - my_entry2 = _make_lock(lock_id=my_lock_id_2, minutes_ago=0) - return [my_entry2] - - mocker.patch.object(lock, '_read_metadata', side_effect=metadata_side_effect) - mocker.patch('time.sleep') - - result = lock.acquire() - assert result.lock_id == my_lock_id_2 - - # We should have released the first candidate - released_keys = [ - entry['key'] - for c in post_mock.call_args_list - for entry in c.kwargs.get('data', {}).get('metadata', []) - if entry['key'].endswith('.released') - ] - assert LOCK_KEY_PREFIX + my_lock_id_1 + '.released' in released_keys - - -# --------------------------------------------------------------------------- -# test_acquire_stale_detected -# --------------------------------------------------------------------------- - - -def test_acquire_stale_detected(mocker): - """Stale entry detected → releases it, cleans project, re-acquires.""" - lock = _make_project_lock(ttl_minutes=60) - - stale_lock_id = 'stale-lock-id-001' - my_lock_id = 'my-fresh-lock-001' - - uuid_iter = iter([my_lock_id, my_lock_id]) # second acquire returns same id - mocker.patch('uuid.uuid4', side_effect=lambda: MagicMock(__str__=lambda _: next(uuid_iter))) - - post_mock = mocker.patch.object(lock, '_post', return_value=[]) - clean_mock = mocker.patch.object(lock, 'clean_project') - - # Build a stale entry (acquired 120 minutes ago, TTL=60) - stale_entry = _make_lock(lock_id=stale_lock_id, minutes_ago=120) - - read_call_count = [0] - - def metadata_side_effect(): - read_call_count[0] += 1 - if read_call_count[0] == 1: - # First read: stale entry + our pending entry - my_entry = _make_lock(lock_id=my_lock_id, minutes_ago=0) - return [stale_entry, my_entry] - else: - # After cleanup: only our fresh entry - my_entry = _make_lock(lock_id=my_lock_id, minutes_ago=0) - return [my_entry] - - mocker.patch.object(lock, '_read_metadata', side_effect=metadata_side_effect) - mocker.patch('time.sleep') - - result = lock.acquire() - - # _clean_project must have been called - clean_mock.assert_not_called() - - # The stale lock must have been released - released_keys = [ - entry['key'] - for c in post_mock.call_args_list - for entry in c.kwargs.get('data', {}).get('metadata', []) - if entry['key'].endswith('.released') - ] - assert LOCK_KEY_PREFIX + stale_lock_id + '.released' in released_keys - - assert result.lock_id == my_lock_id - - -# --------------------------------------------------------------------------- -# test_clean_project_deletes_buckets -# --------------------------------------------------------------------------- - - -def test_clean_project_deletes_buckets(mocker): - """_clean_project calls DELETE for each bucket with force=true.""" - lock = _make_project_lock() - - mocker.patch.object( - lock, - '_get', - side_effect=lambda path, **params: ( - [{'id': 'in.c-bucket1'}, {'id': 'in.c-bucket2'}] if path.endswith('/buckets') else [] - ), - ) - delete_mock = mocker.patch.object(lock, '_delete') - - lock.clean_project() - - delete_calls = [c for c in delete_mock.call_args_list if 'buckets' in c.args[0]] - deleted_bucket_paths = {c.args[0] for c in delete_calls} - assert '/v2/storage/buckets/in.c-bucket1' in deleted_bucket_paths - assert '/v2/storage/buckets/in.c-bucket2' in deleted_bucket_paths - # Each bucket deleted with force=true - for c in delete_calls: - assert c.kwargs.get('force') == 'true' - - -# --------------------------------------------------------------------------- -# test_clean_project_deletes_configs -# --------------------------------------------------------------------------- - - -def test_clean_project_deletes_configs(mocker): - """_clean_project calls DELETE twice for each config (move to trash + purge).""" - lock = _make_project_lock() - - components = [ - { - 'id': 'ex-generic-v2', - 'configurations': [{'id': '123'}, {'id': '456'}], - } - ] - - mocker.patch.object( - lock, - '_get', - side_effect=lambda path, **params: ([] if path.endswith('/buckets') else components), - ) - delete_mock = mocker.patch.object(lock, '_delete') - - lock.clean_project() - - config_delete_paths = [c.args[0] for c in delete_mock.call_args_list if 'configs' in c.args[0]] - # Each config deleted twice - assert config_delete_paths.count('/v2/storage/components/ex-generic-v2/configs/123') == 2 - assert config_delete_paths.count('/v2/storage/components/ex-generic-v2/configs/456') == 2 - - -# --------------------------------------------------------------------------- -# test_clean_project_deletes_workspaces -# --------------------------------------------------------------------------- - - -def test_clean_project_deletes_workspaces(mocker): - """clean_project deletes every workspace — none are persistent anymore.""" - lock = _make_project_lock() - - def _get(path: str, **params: Any) -> list[dict]: - if path.endswith('/workspaces'): - return [{'id': 9001}, {'id': 9002}] - return [] - - mocker.patch.object(lock, '_get', side_effect=_get) - delete_mock = mocker.patch.object(lock, '_delete') - - lock.clean_project() - - workspace_delete_paths = [c.args[0] for c in delete_mock.call_args_list if '/workspaces/' in c.args[0]] - assert '/v2/storage/workspaces/9001' in workspace_delete_paths - assert '/v2/storage/workspaces/9002' in workspace_delete_paths - - -def test_clean_project_tolerates_workspace_delete_failure(mocker): - """A workspace already removed via its sandbox config must not abort cleanup.""" - lock = _make_project_lock() - - def _get(path: str, **params: Any) -> list[dict]: - if path.endswith('/workspaces'): - return [{'id': 9001}] - return [] - - mocker.patch.object(lock, '_get', side_effect=_get) - mocker.patch.object( - lock, - '_delete', - side_effect=httpx.HTTPStatusError('404', request=mocker.MagicMock(), response=mocker.MagicMock()), - ) - - # Should not raise. - lock.clean_project() - - -# --------------------------------------------------------------------------- -# test_release_writes_released_key -# --------------------------------------------------------------------------- - - -def test_release_writes_released_key(mocker): - """release() writes the .released metadata key for the given lock_id.""" - lock = _make_project_lock() - post_mock = mocker.patch.object(lock, '_post', return_value=[]) - - lock_info = LockInfo( - lock_id='release-test-id', - acquired_at=datetime.now(timezone.utc), - runner_info='host/99', - metadata_key=LOCK_KEY_PREFIX + 'release-test-id', - ) - lock.release(lock_info) - - written_keys = [ - entry['key'] for c in post_mock.call_args_list for entry in c.kwargs.get('data', {}).get('metadata', []) - ] - assert LOCK_KEY_PREFIX + 'release-test-id.released' in written_keys - - -# --------------------------------------------------------------------------- -# test_runner_info_includes_hostname_pid -# --------------------------------------------------------------------------- - - -def test_runner_info_includes_hostname_pid(monkeypatch): - """runner_info contains hostname and PID.""" - monkeypatch.delenv('GITHUB_RUN_ID', raising=False) - info = ProjectLock._runner_info() - assert socket.gethostname() in info - assert str(os.getpid()) in info - - -# --------------------------------------------------------------------------- -# test_runner_info_includes_ci_job -# --------------------------------------------------------------------------- - - -def test_runner_info_includes_ci_job(monkeypatch): - """runner_info includes GITHUB_RUN_ID when set.""" - monkeypatch.setenv('GITHUB_RUN_ID', '987654321') - info = ProjectLock._runner_info() - assert 'CI=987654321' in info - - -# --------------------------------------------------------------------------- -# test_max_wait_exceeded_raises -# --------------------------------------------------------------------------- - - -def test_max_wait_exceeded_raises(mocker): - """Raises TimeoutError after max_wait_minutes is exhausted.""" - lock = _make_project_lock( - poll_interval_seconds=1, - max_wait_minutes=0, # expire immediately - anti_collision_seconds=0, - ) - - other_lock_id = 'other-runner-lock' - other_entry = _make_lock(lock_id=other_lock_id, minutes_ago=0) - my_lock_id = 'my-candidate-lock' - my_entry = _make_lock(lock_id=my_lock_id, minutes_ago=0) - - mocker.patch('uuid.uuid4', return_value=MagicMock(__str__=lambda _: my_lock_id)) - mocker.patch.object(lock, '_post', return_value=[]) - # The other runner always holds the lock; their entry is older - mocker.patch.object(lock, '_read_metadata', return_value=[other_entry, my_entry]) - mocker.patch('time.sleep') - - # Patch datetime.now to return a time past the deadline on the second call - original_now = datetime.now - - call_count = [0] - - def fake_now(tz=None): - call_count[0] += 1 - if call_count[0] <= 2: - return original_now(tz) - # Return a time far in the future to trigger the deadline - return datetime(2099, 1, 1, tzinfo=timezone.utc) - - mocker.patch('integtests.project_lock.datetime', wraps=datetime) - mocker.patch('integtests.project_lock.datetime.now', side_effect=fake_now) - - with pytest.raises(TimeoutError, match='Could not acquire project lock'): - lock.acquire() - - -# =========================================================================== -# Helpers for ProjectPool / _try_acquire_once tests -# =========================================================================== - - -def _make_endpoint( - url: str = 'https://connection.keboola.com', - token: str = 'test-token', - project_id: str = 'proj-001', - project_name: str = 'Test Project', -) -> ProjectEndpoint: - return ProjectEndpoint( - storage_api_url=url, - storage_api_token=token, - project_id=project_id, - project_name=project_name, - ) - - -def _make_pool(**kwargs) -> ProjectPool: - defaults = dict( - endpoints=[_make_endpoint()], - ttl_minutes=60, - poll_interval_seconds=1, - max_wait_minutes=5, - anti_collision_seconds=0, - ) - defaults.update(kwargs) - return ProjectPool(**defaults) - - -def _make_lock_info(lock_id: str = 'test-lock-id') -> LockInfo: - return LockInfo( - lock_id=lock_id, - acquired_at=datetime.now(timezone.utc), - runner_info='host/1', - metadata_key=LOCK_KEY_PREFIX + lock_id, - ) - - -# =========================================================================== -# _try_acquire_once tests -# =========================================================================== - - -# --------------------------------------------------------------------------- -# test_try_acquire_once_happy_path -# --------------------------------------------------------------------------- - - -def test_try_acquire_once_happy_path(mocker): - """Single candidate, we are oldest → returns LockInfo.""" - lock = _make_project_lock() - my_lock_id = 'try-once-happy-01' - - mocker.patch('uuid.uuid4', return_value=MagicMock(__str__=lambda _: my_lock_id)) - mocker.patch.object(lock, '_post', return_value=[]) - my_entry = _make_lock(lock_id=my_lock_id, minutes_ago=0) - mocker.patch.object(lock, '_read_metadata', return_value=[my_entry]) - sleep_mock = mocker.patch('time.sleep') - cleanup_mock = mocker.patch.object(lock, '_cleanup_old_locks') - clean_mock = mocker.patch.object(lock, 'clean_project') - - result = lock._try_acquire_once() - - assert isinstance(result, LockInfo) - assert result.lock_id == my_lock_id - assert result.metadata_key == LOCK_KEY_PREFIX + my_lock_id - assert any(c == call(0) for c in sleep_mock.call_args_list) # anti_collision=0 - cleanup_mock.assert_called_once_with(my_lock_id) - clean_mock.assert_called_once() - - -# --------------------------------------------------------------------------- -# test_try_acquire_once_loses_to_active_runner -# --------------------------------------------------------------------------- - - -def test_try_acquire_once_loses_to_active_runner(mocker): - """Other runner is older → returns None; our candidate is released.""" - lock = _make_project_lock() - my_lock_id = 'try-once-lose-001' - other_lock_id = 'try-once-other-01' - - mocker.patch('uuid.uuid4', return_value=MagicMock(__str__=lambda _: my_lock_id)) - post_mock = mocker.patch.object(lock, '_post', return_value=[]) - clean_mock = mocker.patch.object(lock, 'clean_project') - - other_entry = _make_lock(lock_id=other_lock_id, minutes_ago=5) # older - my_entry = _make_lock(lock_id=my_lock_id, minutes_ago=0) - mocker.patch.object(lock, '_read_metadata', return_value=[other_entry, my_entry]) - mocker.patch('time.sleep') - - result = lock._try_acquire_once() - - assert result is None - # Our candidate must be released - released_keys = [ - entry['key'] - for c in post_mock.call_args_list - for entry in c.kwargs.get('data', {}).get('metadata', []) - if entry['key'].endswith('.released') - ] - assert LOCK_KEY_PREFIX + my_lock_id + '.released' in released_keys - clean_mock.assert_not_called() - - -# --------------------------------------------------------------------------- -# test_try_acquire_once_stale_then_wins -# --------------------------------------------------------------------------- - - -def test_try_acquire_once_stale_then_wins(mocker): - """Stale detected → cleans project, second candidate wins → returns LockInfo.""" - lock = _make_project_lock(ttl_minutes=60) - - stale_id = 'stale-entry-0001' - my_id_1 = 'my-first-cand-001' - my_id_2 = 'my-second-cand-01' - - uuid_iter = iter([my_id_1, my_id_2]) - mocker.patch('uuid.uuid4', side_effect=lambda: MagicMock(__str__=lambda _: next(uuid_iter))) - - post_mock = mocker.patch.object(lock, '_post', return_value=[]) - clean_mock = mocker.patch.object(lock, 'clean_project') - cleanup_mock = mocker.patch.object(lock, '_cleanup_old_locks') - - stale_entry = _make_lock(lock_id=stale_id, minutes_ago=120) - - read_call = [0] - - def metadata_side_effect(): - read_call[0] += 1 - if read_call[0] == 1: - return [stale_entry, _make_lock(lock_id=my_id_1, minutes_ago=0)] - else: - return [_make_lock(lock_id=my_id_2, minutes_ago=0)] - - mocker.patch.object(lock, '_read_metadata', side_effect=metadata_side_effect) - mocker.patch('time.sleep') - - result = lock._try_acquire_once() - - assert isinstance(result, LockInfo) - assert result.lock_id == my_id_2 - clean_mock.assert_not_called() - cleanup_mock.assert_called_once_with(my_id_2) - - released_keys = [ - entry['key'] - for c in post_mock.call_args_list - for entry in c.kwargs.get('data', {}).get('metadata', []) - if entry['key'].endswith('.released') - ] - assert LOCK_KEY_PREFIX + stale_id + '.released' in released_keys - assert LOCK_KEY_PREFIX + my_id_1 + '.released' in released_keys - - -# --------------------------------------------------------------------------- -# test_try_acquire_once_stale_then_loses -# --------------------------------------------------------------------------- - - -def test_try_acquire_once_stale_then_loses(mocker): - """Stale cleaned but second attempt loses to a racing runner → returns None.""" - lock = _make_project_lock(ttl_minutes=60) - - stale_id = 'stale-entry-0002' - my_id_1 = 'my-first-cand-002' - my_id_2 = 'my-second-cand-02' - other_id = 'other-racer-0001' - - uuid_iter = iter([my_id_1, my_id_2]) - mocker.patch('uuid.uuid4', side_effect=lambda: MagicMock(__str__=lambda _: next(uuid_iter))) - - post_mock = mocker.patch.object(lock, '_post', return_value=[]) - clean_mock = mocker.patch.object(lock, 'clean_project') - - stale_entry = _make_lock(lock_id=stale_id, minutes_ago=120) - - read_call = [0] - - def metadata_side_effect(): - read_call[0] += 1 - if read_call[0] == 1: - return [stale_entry, _make_lock(lock_id=my_id_1, minutes_ago=0)] - else: - # Another runner snuck in and is older than our second candidate - other_entry = _make_lock(lock_id=other_id, minutes_ago=1) - return [other_entry, _make_lock(lock_id=my_id_2, minutes_ago=0)] - - mocker.patch.object(lock, '_read_metadata', side_effect=metadata_side_effect) - mocker.patch('time.sleep') - - result = lock._try_acquire_once() - - assert result is None - clean_mock.assert_not_called() - - released_keys = [ - entry['key'] - for c in post_mock.call_args_list - for entry in c.kwargs.get('data', {}).get('metadata', []) - if entry['key'].endswith('.released') - ] - # Both candidates must be released - assert LOCK_KEY_PREFIX + my_id_1 + '.released' in released_keys - assert LOCK_KEY_PREFIX + my_id_2 + '.released' in released_keys - - -# --------------------------------------------------------------------------- -# test_acquire_still_works_via_try_acquire_once -# --------------------------------------------------------------------------- - - -def test_acquire_still_works_via_try_acquire_once(mocker): - """acquire() loops _try_acquire_once(); None on first call, LockInfo on second.""" - lock = _make_project_lock(poll_interval_seconds=7) - - lock_info = _make_lock_info('final-lock-0001') - mocker.patch.object(lock, '_try_acquire_once', side_effect=[None, lock_info]) - sleep_mock = mocker.patch('time.sleep') - - result = lock.acquire() - - assert result == lock_info - assert call(7) in sleep_mock.call_args_list - - -# =========================================================================== -# ProjectPool tests -# =========================================================================== - - -# --------------------------------------------------------------------------- -# test_pool_empty_endpoints_raises -# --------------------------------------------------------------------------- - - -def test_pool_empty_endpoints_raises(): - """ProjectPool(endpoints=[]) raises ValueError.""" - with pytest.raises(ValueError, match='at least one endpoint'): - ProjectPool(endpoints=[]) - - -# --------------------------------------------------------------------------- -# test_pool_single_endpoint_acquires -# --------------------------------------------------------------------------- - - -def test_pool_single_endpoint_acquires(mocker): - """Pool of one endpoint: _try_acquire_once wins on first try → AcquiredProject.""" - endpoint = _make_endpoint() - pool = _make_pool(endpoints=[endpoint]) - - lock_info = _make_lock_info('pool-single-001') - mock_lock = mocker.MagicMock() - mock_lock._try_acquire_once.return_value = lock_info - mocker.patch.object(pool, '_make_lock', return_value=mock_lock) - mocker.patch('time.sleep') - - result = pool.acquire() - - assert isinstance(result, AcquiredProject) - assert result.endpoint == endpoint - assert result.lock_info == lock_info - - -# --------------------------------------------------------------------------- -# test_pool_first_busy_second_free -# --------------------------------------------------------------------------- - - -def test_pool_first_busy_second_free(mocker): - """First endpoint is locked; second is free → result uses second endpoint, no poll sleep.""" - endpoint1 = _make_endpoint(token='token-aaa') - endpoint2 = _make_endpoint(token='token-bbb') - pool = _make_pool(endpoints=[endpoint1, endpoint2], poll_interval_seconds=30) - - lock_info = _make_lock_info('pool-second-001') - mock_lock1 = mocker.MagicMock() - mock_lock1._try_acquire_once.return_value = None - mock_lock2 = mocker.MagicMock() - mock_lock2._try_acquire_once.return_value = lock_info - - mocker.patch.object(pool, '_make_lock', side_effect=[mock_lock1, mock_lock2]) - mocker.patch('integtests.project_lock.random.randrange', return_value=0) - sleep_mock = mocker.patch('time.sleep') - - result = pool.acquire() - - assert result.endpoint == endpoint2 - assert result.lock_info == lock_info - # No poll sleep — a project was found within the first pass - assert call(30) not in sleep_mock.call_args_list - - -# --------------------------------------------------------------------------- -# test_pool_all_busy_then_one_frees -# --------------------------------------------------------------------------- - - -def test_pool_all_busy_then_one_frees(mocker): - """Both endpoints busy on pass 1; first frees on pass 2 → poll sleep called once.""" - endpoint1 = _make_endpoint(token='token-ccc') - endpoint2 = _make_endpoint(token='token-ddd') - pool = _make_pool(endpoints=[endpoint1, endpoint2], poll_interval_seconds=11) - - lock_info = _make_lock_info('pool-retry-001') - # Pass 1: both locked - mock_lock1a = mocker.MagicMock() - mock_lock1a._try_acquire_once.return_value = None - mock_lock2a = mocker.MagicMock() - mock_lock2a._try_acquire_once.return_value = None - # Pass 2: endpoint1 succeeds - mock_lock1b = mocker.MagicMock() - mock_lock1b._try_acquire_once.return_value = lock_info - - mocker.patch.object(pool, '_make_lock', side_effect=[mock_lock1a, mock_lock2a, mock_lock1b]) - mocker.patch('integtests.project_lock.random.randrange', return_value=0) - sleep_mock = mocker.patch('time.sleep') - - result = pool.acquire() - - assert result.endpoint == endpoint1 - assert result.lock_info == lock_info - # Poll sleep must have been called between the two passes - assert call(11) in sleep_mock.call_args_list - - -# --------------------------------------------------------------------------- -# test_pool_stale_project_claimed_not_skipped -# --------------------------------------------------------------------------- - - -def test_pool_stale_project_claimed_not_skipped(mocker): - """ - First endpoint's _try_acquire_once returns LockInfo (stale-path internal win) - → pool claims it immediately; second endpoint is never tried. - """ - endpoint1 = _make_endpoint(token='token-eee') - endpoint2 = _make_endpoint(token='token-fff') - pool = _make_pool(endpoints=[endpoint1, endpoint2]) - - lock_info = _make_lock_info('pool-stale-win-01') - mock_lock1 = mocker.MagicMock() - mock_lock1._try_acquire_once.return_value = lock_info - - make_lock_mock = mocker.patch.object(pool, '_make_lock', return_value=mock_lock1) - mocker.patch('integtests.project_lock.random.randrange', return_value=0) - mocker.patch('time.sleep') - - result = pool.acquire() - - assert result.endpoint == endpoint1 - assert result.lock_info == lock_info - # _make_lock called only once (endpoint2 was never tried) - assert make_lock_mock.call_count == 1 - make_lock_mock.assert_called_once_with(endpoint1) - - -# --------------------------------------------------------------------------- -# test_pool_timeout_raises -# --------------------------------------------------------------------------- - - -def test_pool_timeout_raises(mocker): - """Raises TimeoutError when no project can be acquired within max_wait_minutes.""" - pool = _make_pool(max_wait_minutes=0) - mocker.patch('time.sleep') - - with pytest.raises(TimeoutError, match='Could not acquire any project lock'): - pool.acquire() - - -# --------------------------------------------------------------------------- -# test_pool_release_delegates_to_lock -# --------------------------------------------------------------------------- - - -def test_pool_release_delegates_to_lock(mocker): - """pool.release(acquired) delegates to _make_lock(endpoint).release(lock_info).""" - endpoint = _make_endpoint() - pool = _make_pool(endpoints=[endpoint]) - - lock_info = _make_lock_info('pool-release-001') - acquired = AcquiredProject(endpoint=endpoint, lock_info=lock_info) - - mock_lock = mocker.MagicMock() - make_lock_mock = mocker.patch.object(pool, '_make_lock', return_value=mock_lock) - - pool.release(acquired) - - make_lock_mock.assert_called_once_with(endpoint) - mock_lock.release.assert_called_once_with(lock_info) - - -# =========================================================================== -# ProjectEndpoint / pool acquisition tests -# =========================================================================== - - -# --------------------------------------------------------------------------- -# test_project_endpoint_stores_all_fields -# --------------------------------------------------------------------------- - - -def test_project_endpoint_stores_all_fields(): - """ProjectEndpoint stores all fields correctly.""" - ep = ProjectEndpoint( - storage_api_url='https://connection.keboola.com', - storage_api_token='my-token', - project_id='99', - project_name='My Project', - token_id='5', - token_description='My test token', - ) - assert ep.storage_api_token == 'my-token' - assert ep.project_id == '99' - assert ep.project_name == 'My Project' - assert ep.token_id == '5' - assert ep.token_description == 'My test token' - - -# --------------------------------------------------------------------------- -# test_pool_acquired_project_carries_endpoint -# --------------------------------------------------------------------------- - - -def test_pool_acquired_project_carries_endpoint(mocker): - """AcquiredProject.endpoint carries the acquired endpoint.""" - endpoint = _make_endpoint(token='token-a') - pool = _make_pool(endpoints=[endpoint]) - - lock_info = _make_lock_info('ws-endpoint-test-001') - mock_lock = mocker.MagicMock() - mock_lock._try_acquire_once.return_value = lock_info - mocker.patch.object(pool, '_make_lock', return_value=mock_lock) - mocker.patch('time.sleep') - - result = pool.acquire() - - assert result.endpoint is endpoint - assert result.endpoint.storage_api_token == 'token-a' - - -# --------------------------------------------------------------------------- -# test_pool_selects_correct_endpoint_when_acquired -# --------------------------------------------------------------------------- - - -def test_pool_selects_correct_endpoint_when_acquired(mocker): - """When the second endpoint is acquired, that endpoint is returned, not the first's.""" - endpoint1 = _make_endpoint(token='token-aaa') - endpoint2 = _make_endpoint(token='token-bbb') - pool = _make_pool(endpoints=[endpoint1, endpoint2], poll_interval_seconds=30) - - lock_info = _make_lock_info('ws-endpoint-second-001') - mock_lock1 = mocker.MagicMock() - mock_lock1._try_acquire_once.return_value = None - mock_lock2 = mocker.MagicMock() - mock_lock2._try_acquire_once.return_value = lock_info - mocker.patch.object(pool, '_make_lock', side_effect=[mock_lock1, mock_lock2]) - mocker.patch('integtests.project_lock.random.randrange', return_value=0) - mocker.patch('time.sleep') - - result = pool.acquire() - - assert result.endpoint is endpoint2 - assert result.endpoint.storage_api_token == 'token-bbb' - - -# --------------------------------------------------------------------------- -# test_pool_acquire_randomizes_start_per_pass -# --------------------------------------------------------------------------- - - -def test_pool_acquire_randomizes_start_per_pass(mocker): - """random.randrange is called once per pool pass.""" - endpoints = [ - _make_endpoint(token='token-aaa'), - _make_endpoint(token='token-bbb'), - _make_endpoint(token='token-ccc'), - ] - pool = _make_pool(endpoints=endpoints, poll_interval_seconds=1) - - lock_info = _make_lock_info('rand-start-001') - # Pass 1: all busy (None). Pass 2: first tried endpoint succeeds. - try_once_results = iter([None, None, None, lock_info]) - mock_lock = mocker.MagicMock() - mock_lock._try_acquire_once.side_effect = lambda: next(try_once_results) - mocker.patch.object(pool, '_make_lock', return_value=mock_lock) - mocker.patch('time.sleep') - - randrange_mock = mocker.patch('integtests.project_lock.random.randrange', return_value=0) - - pool.acquire() - - # randrange must be called once per pass (2 passes: one all-busy + one with a winner) - assert randrange_mock.call_count == 2 - # Each call must pass the pool size as the upper bound - for c in randrange_mock.call_args_list: - assert c == call(len(endpoints)) - - -# =========================================================================== -# verify_project_endpoint tests -# =========================================================================== - - -# --------------------------------------------------------------------------- -# test_verify_project_endpoint_happy_path -# --------------------------------------------------------------------------- - - -def test_verify_project_endpoint_happy_path(mocker): - """verify_project_endpoint returns a fully populated ProjectEndpoint on success.""" - token_info = { - 'id': 7, - 'description': 'CI integration test token', - 'owner': {'id': 42, 'name': 'My CI Project'}, - } - mock_resp = mocker.MagicMock() - mock_resp.json.return_value = token_info - mock_client = mocker.MagicMock() - mock_client.__enter__ = mocker.MagicMock(return_value=mock_client) - mock_client.__exit__ = mocker.MagicMock(return_value=False) - mock_client.get.return_value = mock_resp - mocker.patch('integtests.project_lock.httpx.Client', return_value=mock_client) - - ep = verify_project_endpoint( - storage_api_url='https://connection.keboola.com', - storage_api_token='my-secret-token', - ) - - assert ep.project_id == '42' - assert ep.project_name == 'My CI Project' - assert ep.storage_api_token == 'my-secret-token' - assert ep.token_id == '7' - assert ep.token_description == 'CI integration test token' - mock_client.get.assert_called_once_with('https://connection.keboola.com/v2/storage/tokens/verify') - mock_resp.raise_for_status.assert_called_once() - - -# --------------------------------------------------------------------------- -# test_verify_project_endpoint_bad_token_raises -# --------------------------------------------------------------------------- - - -def test_verify_project_endpoint_bad_token_raises(mocker): - """verify_project_endpoint propagates HTTPStatusError on a bad token.""" - mock_resp = mocker.MagicMock() - mock_resp.raise_for_status.side_effect = httpx.HTTPStatusError( - '401 Unauthorized', request=mocker.MagicMock(), response=mocker.MagicMock() - ) - mock_client = mocker.MagicMock() - mock_client.__enter__ = mocker.MagicMock(return_value=mock_client) - mock_client.__exit__ = mocker.MagicMock(return_value=False) - mock_client.get.return_value = mock_resp - mocker.patch('integtests.project_lock.httpx.Client', return_value=mock_client) - - with pytest.raises(httpx.HTTPStatusError): - verify_project_endpoint( - storage_api_url='https://connection.keboola.com', - storage_api_token='bad-token', - ) - - -# =========================================================================== -# _delete_metadata_by_id / _cleanup_old_locks tests -# =========================================================================== - - -# --------------------------------------------------------------------------- -# test_delete_metadata_by_id -# --------------------------------------------------------------------------- - - -def test_delete_metadata_by_id(mocker): - """_delete_metadata_by_id calls DELETE on the correct metadata path.""" - lock = _make_project_lock() - delete_mock = mocker.patch.object(lock, '_delete') - lock._delete_metadata_by_id('9876') - delete_mock.assert_called_once_with('/v2/storage/branch/default/metadata/9876') - - -# --------------------------------------------------------------------------- -# test_cleanup_old_locks -# --------------------------------------------------------------------------- - -_OLD_ID = 'old-lock-id-0001' -_CUR_ID = 'cur-lock-id-0001' - - -@pytest.mark.parametrize( - ('scenario', 'entries', 'current_lock_id', 'expected_delete_calls'), - [ - ( - 'no_released_entries', - lambda: [_make_lock(lock_id=_OLD_ID, meta_id=1)], - _CUR_ID, - [], - ), - ( - 'full_released_pair', - lambda: [ - _make_lock(lock_id=_OLD_ID, meta_id=10), - _released_entry(_OLD_ID, meta_id=11), - ], - _CUR_ID, - ['10', '11'], - ), - ( - 'orphaned_released_only', - lambda: [_released_entry(_OLD_ID, meta_id=20)], - _CUR_ID, - ['20'], - ), - ( - 'skip_current_lock', - lambda: [ - _make_lock(lock_id=_CUR_ID, meta_id=30), - _released_entry(_CUR_ID, meta_id=31), - ], - _CUR_ID, - [], - ), - ( - 'mixed_old_and_current', - lambda: [ - _make_lock(lock_id=_OLD_ID, meta_id=40), - _released_entry(_OLD_ID, meta_id=41), - _make_lock(lock_id=_CUR_ID, meta_id=42), - _released_entry(_CUR_ID, meta_id=43), - ], - _CUR_ID, - ['40', '41'], - ), - ( - 'main_deletion_error_skips_released', - lambda: [ - _make_lock(lock_id=_OLD_ID, meta_id=50), - _released_entry(_OLD_ID, meta_id=51), - ], - _CUR_ID, - 'error_on_main', - ), - ], -) -def test_cleanup_old_locks(mocker, scenario, entries, current_lock_id, expected_delete_calls): - lock = _make_project_lock() - mocker.patch.object(lock, '_read_metadata', side_effect=entries) - - if expected_delete_calls == 'error_on_main': - delete_mock = mocker.patch.object(lock, '_delete_metadata_by_id', side_effect=RuntimeError('fail')) - # Should not raise; the error is swallowed - lock._cleanup_old_locks(current_lock_id) - # Only one call attempted (the main entry); .released is skipped - delete_mock.assert_called_once_with('50') - else: - delete_mock = mocker.patch.object(lock, '_delete_metadata_by_id') - lock._cleanup_old_locks(current_lock_id) - if not expected_delete_calls: - delete_mock.assert_not_called() - else: - assert delete_mock.call_count == len(expected_delete_calls) - # Verify the calls were made in the correct order - assert delete_mock.call_args_list == [call(mid) for mid in expected_delete_calls] diff --git a/integtests/test_validate.py b/integtests/test_validate.py deleted file mode 100644 index 31ec29f33..000000000 --- a/integtests/test_validate.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -This test is used to validate both row and root parameter schemas of all components. -- Serves as a sanity check for the schemas validation, identifying invalid schemas and proposing two possible solutions: - - Fix the json schema to be valid - - Improve the KeboolaParametersValidator to accept the schema -- Ensures that all parameter schemas are valid and that the MCP server will them correctly to validate the parameters -received from the LLM Agent. -- In case the schema is invalid, we skip the validation, log the schema error but continue with the action (creation or -update of the component) assuming that the validation of json object against the schema had been correct. That is the -reason why we are having those tests. -""" - -import logging -from typing import cast - -import httpx -import jsonschema -import pytest - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.storage import ComponentAPIResponse -from keboola_mcp_server.tools.components.model import Component -from keboola_mcp_server.tools.validation import KeboolaParametersValidator - -LOG = logging.getLogger(__name__) - - -def _check_schema(schema: JsonDict, dummy_parameters: JsonDict) -> None: - try: - KeboolaParametersValidator.validate(dummy_parameters, schema) - except jsonschema.ValidationError: - # We care only about schema errors, ignore validation errors since we are using dummy parameters. - # The schema itself is checked just before we validate JSON object against it. Hence, we can ignore - # ValidationError because the schema is valid which is our objective, but our dummy_parameters violates - # the schema - we are not interested in the dummy parameters. - pass - - -@pytest.mark.asyncio -async def test_validate_parameters(keboola_client: KeboolaClient): - # Fetch the storage stack index directly to avoid the trailing-slash 301 that - # `storage_client.get('')` triggers on some stacks (it builds `/`). - raw = keboola_client.storage_client.raw_client - async with httpx.AsyncClient(timeout=raw.timeout, transport=raw._create_transport()) as client: - response = await client.get(raw.base_api_url, headers=raw.headers) - response.raise_for_status() - data = cast(JsonDict, response.json()) - LOG.info(f'Fetched information: {data.keys()}') - components = cast(list[JsonDict], data['components']) - components.sort(key=lambda x: (x['type'], x['name'])) # sort by type and then by name - LOG.info(f'Fetched total of {len(components)} components') - - row_counts, root_counts = 0, 0 - invalid_row_schemas, invalid_root_schemas = [], [] - for raw_component in components: - api_component = ComponentAPIResponse.model_validate(raw_component) - component = Component.from_api_response(api_component) - if component.configuration_schema: - try: - root_counts += 1 - _check_schema(component.configuration_schema, dummy_parameters={}) - except jsonschema.SchemaError as e: - LOG.exception(f'Root schema error for {raw_component["id"]}: {e}') - invalid_root_schemas.append(raw_component['id']) - if component.configuration_row_schema: - try: - row_counts += 1 - _check_schema(component.configuration_row_schema, dummy_parameters={}) - except jsonschema.SchemaError as e: - LOG.exception(f'Row schema error for {raw_component["id"]}: {e}') - invalid_row_schemas.append(raw_component['id']) - - if invalid_root_schemas: - pytest.fail(f'Invalid root schemas({len(invalid_root_schemas)}): {invalid_root_schemas}') - if invalid_row_schemas: - pytest.fail(f'Invalid row schemas({len(invalid_row_schemas)}): {invalid_row_schemas}') - - LOG.info( - f'Total components: {len(components)}, from which {root_counts} have root configuration schema and ' - f'{row_counts} have row configuration schema. All schemas are valid.' - ) diff --git a/integtests/test_workspace.py b/integtests/test_workspace.py deleted file mode 100644 index 3b5de0bca..000000000 --- a/integtests/test_workspace.py +++ /dev/null @@ -1,128 +0,0 @@ -import logging -from collections.abc import AsyncGenerator, Mapping -from typing import Any - -import pytest -import pytest_asyncio -import requests -from kbcstorage.client import Client as SyncStorageClient - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - - -@pytest_asyncio.fixture -async def dynamic_manager( - keboola_client: KeboolaClient, sync_storage_client: SyncStorageClient, workspace_schema: str -) -> AsyncGenerator[WorkspaceManager, Any]: - storage_client = sync_storage_client - token_info = storage_client.tokens.verify() - project_id: str = token_info['owner']['id'] - - def _get_workspace_meta() -> list[Mapping[str, Any]]: - metadata: list[Mapping[str, Any]] = [] - for m in storage_client.branches.metadata('default'): - if m.get('key') == WorkspaceManager.MCP_META_KEY: - metadata.append(m) - return metadata - - metas = _get_workspace_meta() - if metas: - pytest.fail(f'Expecting empty Keboola project {project_id}, but found {metas} in the default branch') - - workspaces = storage_client.workspaces.list() - # ignore the static workspaces - workspaces = [ - w - for w in workspaces - if all( - [ - w['connection']['schema'] != workspace_schema, - w.get('creatorToken', {}).get('description') != 'Background Indexing Token', - ] - ) - ] - if workspaces: - pytest.fail( - f'Expecting empty Keboola project {project_id}, but found {len(workspaces)} extra workspaces: ' - f'{[{"id": w["id"], "name": w["name"]} for w in workspaces]}' - ) - - component_id = WorkspaceManager.MCP_WORKSPACE_COMPONENT_ID - existing_configs = list(storage_client.configurations.list(component_id=component_id)) - if existing_configs: - pytest.fail( - f'Expecting no MCP workspace configs in project {project_id}, ' - f'but found: {[c.get("id") for c in existing_configs]}' - ) - - yield await WorkspaceManager.create(keboola_client) - - LOG.info(f'Cleaning up workspaces in Keboola project with ID={project_id}') - metas = _get_workspace_meta() - if len(metas) > 1: - LOG.info(f'Multiple metadata entries found: {metas}') - for meta in metas: - try: - storage_client.workspaces.delete(meta['value']) - LOG.info(f'Deleted workspaces: {meta["value"]}') - except requests.HTTPError: - LOG.exception(f'Failed to delete workspace {meta["value"]}') - try: - url = storage_client.branches.base_url.rstrip('/') - storage_client.branches._delete(f'{url}/branch/default/metadata/{meta["id"]}') - LOG.info(f'Deleted workspaces metadata: {meta["id"]}') - except requests.HTTPError as e: - LOG.exception(f'Failed to delete workspace metadata {meta["id"]}: {e}') - - # Clean up configurations created under the MCP workspace component - component_id = WorkspaceManager.MCP_WORKSPACE_COMPONENT_ID - try: - configs = storage_client.configurations.list(component_id=component_id) - for cfg in configs: - cfg_id = cfg.get('id') - if cfg_id: - try: - storage_client.configurations.delete(component_id, cfg_id) - # Double delete to skip trash - storage_client.configurations.delete(component_id, cfg_id) - LOG.info(f'Deleted component config: {component_id}/{cfg_id}') - except requests.HTTPError: - LOG.exception(f'Failed to delete component config {component_id}/{cfg_id}') - except requests.HTTPError: - LOG.exception(f'Failed to list configs for {component_id}') - - -class TestWorkspaceManager: - - @pytest.mark.asyncio - async def test_static_workspace(self, workspace_manager: WorkspaceManager, workspace_schema: str): - assert workspace_manager._workspace_schema == workspace_schema - - info = await workspace_manager._find_ws_by_schema(workspace_schema) - assert info is not None - assert info.schema == workspace_schema - assert info.backend in ['snowflake', 'bigquery'] - - workspace = await workspace_manager._get_workspace() - assert workspace is not None - assert workspace.id == info.id - - @pytest.mark.asyncio - async def test_dynamic_workspace(self, dynamic_manager: WorkspaceManager): - assert dynamic_manager._workspace_schema is None - - # check that there is no workspace in the branch - info = await dynamic_manager._find_ws_in_branch() - assert info is None - - # create workspace - workspace = await dynamic_manager._get_workspace() - assert workspace is not None - - # check that the new workspace is recorded in the branch - info = await dynamic_manager._find_ws_in_branch() - assert info is not None - assert workspace.id == info.id diff --git a/integtests/testproject/clean.ts b/integtests/testproject/clean.ts new file mode 100644 index 000000000..f8d577a06 --- /dev/null +++ b/integtests/testproject/clean.ts @@ -0,0 +1,90 @@ +import { createRawClient } from '@/clients/raw'; +import { WorkspaceManager } from '@/workspace'; +import { type ProjectDefinition, storageApiUrl } from './types'; + +// Reset a leased project to a clean state before a test runs — port of the Python +// integtests _purge_project + _guard_dedicated_test_project. Uses the raw Storage API +// (rooted at /v2/storage) so it has no dependency on the per-request server config. + +// The integ fixtures only ever create stage-prefixed `*.c-test*` buckets. A project holding +// any other bucket is almost certainly NOT a dedicated test project — refuse to wipe it. +const TEST_BUCKET_PREFIXES = ['in.c-test', 'out.c-test', 'sys.c-test']; + +type Bucket = { id: string }; +type Component = { id: string; configurations?: { id: string }[] }; +type Workspace = { id: string | number; creatorToken?: { description?: string } }; +type Meta = { id: string | number; key?: string }; +type StorageJob = { id?: string | number; status?: string }; + +const STATIC_WORKSPACE_CREATORS = new Set(['Background Indexing Token']); + +type RawClient = ReturnType; +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +/** Polls a Storage async job to completion (used for async bucket drop). */ +const waitForStorageJob = async (raw: RawClient, job: StorageJob): Promise => { + if (!job?.id) return; // synchronous response (older stacks) — nothing to wait for + const deadline = Date.now() + 120_000; + for (;;) { + const current = await raw.get(`jobs/${job.id}`); + const status = current.status ?? ''; + if (status === 'success' || status === 'error' || status === 'cancelled') return; + if (Date.now() > deadline) throw new Error(`Storage job ${job.id} did not finish within 120s`); + await sleep(1000); + } +}; + +export const cleanProject = async (def: ProjectDefinition): Promise => { + const raw = createRawClient({ baseUrl: `${storageApiUrl(def)}/v2/storage`, token: def.token }); + + // Guard: refuse to reset a project that holds non-test buckets. + const buckets = await raw.get('buckets'); + const foreign = buckets.filter((b) => !TEST_BUCKET_PREFIXES.some((p) => b.id.startsWith(p))); + if (foreign.length > 0) { + throw new Error( + `Refusing to reset project ${def.project}: found non-test buckets ${foreign + .map((b) => b.id) + .join(', ')}. The projects.json pool may point at a non-dedicated project.`, + ); + } + + // Bucket drop must be async — the Storage API rejects a synchronous force-drop + // ("Synchronous drop is not supported, use async call"). Each delete returns a storage + // job; wait for it so a subsequent seed doesn't race a half-deleted bucket. + for (const bucket of buckets) { + const job = await raw.delete(`buckets/${bucket.id}`, { + params: { force: 'true', async: 'true' }, + }); + await waitForStorageJob(raw, job); + } + + const components = await raw.get('components', { params: { include: 'configuration' } }); + for (const component of components) { + for (const config of component.configurations ?? []) { + // First delete moves to trash; second purges it. Best-effort: some pooled projects' + // tokens return 403 on the purge ("You don't have access to the resource") — leaving a + // trashed config behind is harmless, so tolerate it rather than aborting the reset. + await raw.delete(`components/${component.id}/configs/${config.id}`).catch(() => {}); + await raw.delete(`components/${component.id}/configs/${config.id}`).catch(() => {}); + } + } + + const workspaces = await raw.get('branch/default/workspaces'); + for (const ws of workspaces) { + if (STATIC_WORKSPACE_CREATORS.has(ws.creatorToken?.description ?? '')) continue; + await raw.delete(`workspaces/${ws.id}`).catch(() => { + /* a workspace backed by a deleted sandbox config may already be gone */ + }); + } + + const metadata = await raw.get('branch/default/metadata'); + for (const meta of metadata) { + if (meta.key === WorkspaceManager.MCP_META_KEY) { + // Some pooled projects return 403 on branch-metadata delete (token scope); the stale + // workspace-id metadata is harmless to leave behind, so tolerate it. + await raw.delete(`branch/default/metadata/${meta.id}`).catch(() => { + /* best-effort: leftover MCP workspace-id metadata does not break the next run */ + }); + } + } +}; diff --git a/integtests/testproject/fixture.ts b/integtests/testproject/fixture.ts new file mode 100644 index 000000000..22ee237e4 --- /dev/null +++ b/integtests/testproject/fixture.ts @@ -0,0 +1,60 @@ +import { onTestFinished } from 'vitest'; + +import { Config } from '@/config'; +import { cleanProject } from './clean'; +import { getPool } from './pool'; +import type { AcquireOptions, Backend } from './types'; + +// Per-test-case project acquisition — port of go-utils GetTestProjectForTest. The lease is +// acquired now and released automatically when the current test finishes (vitest +// onTestFinished), so each case holds a project only for its own duration. + +export type TestProject = { + config: Config; + storageApiUrl: string; + storageApiToken: string; + backend: Backend; + projectId: number; +}; + +export type GetTestProjectOptions = AcquireOptions & { + /** + * Reset the project to a clean state before returning (default `true`). Tests that don't + * mutate project state (e.g. docs_query, get_project_info, a literal SELECT) can pass + * `false` to skip the wipe — useful against shared projects the dedicated-project guard + * would otherwise refuse to clean. + */ + clean?: boolean; +}; + +/** + * Leases a project for the calling test, resets it to a clean state, and returns a ready + * `Config`. The lease is released on test completion. If the whole pool is busy this blocks + * (and retries) until a project frees up rather than failing. + * + * Must be called from within a running test (it registers onTestFinished). + */ +export const getTestProjectForTest = async ( + opts: GetTestProjectOptions = {}, +): Promise => { + const pool = getPool(); + const leased = await pool.getTestProject(opts); + onTestFinished(async () => { + await leased.release(); + }); + + if (opts.clean !== false) { + await cleanProject(leased.definition); + } + + return { + config: new Config({ + storageApiUrl: leased.storageApiUrl, + storageToken: leased.storageApiToken, + }), + storageApiUrl: leased.storageApiUrl, + storageApiToken: leased.storageApiToken, + backend: leased.backend, + projectId: leased.definition.project, + }; +}; diff --git a/integtests/testproject/fsLocker.ts b/integtests/testproject/fsLocker.ts new file mode 100644 index 000000000..21feee63f --- /dev/null +++ b/integtests/testproject/fsLocker.ts @@ -0,0 +1,48 @@ +import { closeSync, mkdirSync, openSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import type { Locker, ProjectLocker, ReleaseFn } from './locker'; +import { lockKey, type ProjectDefinition } from './types'; + +/** + * Host-local file lock — port of go-utils pkg/testproject/fslocker.go. Used only as a + * fallback for local single-host runs when no redis is configured; it provides NO + * cross-runner safety (CI always uses the redis locker). + * + * A project is leased by exclusively creating `/-.lock` + * (O_CREAT|O_EXCL); a concurrent creator gets EEXIST and treats the project as busy. + * Release unlinks the file. + */ +export const createFsLocker = (dirName?: string): Locker => { + const dir = dirName ?? join(tmpdir(), 'kbc-mcp-testproject-locks'); + mkdirSync(dir, { recursive: true }); + + const forProject = (def: ProjectDefinition): ProjectLocker => { + const file = join(dir, `${lockKey(def)}.lock`); + return { + tryLock: async (): Promise => { + let fd: number; + try { + fd = openSync(file, 'wx'); // exclusive create; throws EEXIST if held + } catch { + return null; + } + closeSync(fd); + let released = false; + return async (): Promise => { + if (released) return; + released = true; + rmSync(file, { force: true }); + }; + }, + }; + }; + + return { + forProject, + close: async (): Promise => { + /* nothing to close */ + }, + }; +}; diff --git a/integtests/testproject/locker.ts b/integtests/testproject/locker.ts new file mode 100644 index 000000000..fa29a43cd --- /dev/null +++ b/integtests/testproject/locker.ts @@ -0,0 +1,24 @@ +import type { ProjectDefinition } from './types'; + +/** + * Locker abstraction (port of the go-utils `locker` / `projectLocker` interfaces). + * + * `tryLock` performs a single, non-blocking attempt to lease the project. It returns a + * `release` function on success, or `null` if the project is currently leased elsewhere. + * The pool (pool.ts) is responsible for the retry loop. + */ +export type ReleaseFn = () => Promise; + +export type ProjectLocker = { + tryLock: () => Promise; +}; + +export type Locker = { + forProject: (def: ProjectDefinition) => ProjectLocker; + /** Releases any locker-wide resources (e.g. the redis connection). */ + close: () => Promise; +}; + +export const LOCK_HOST_ENV = 'TEST_MCP_PROJECTS_LOCK_HOST'; +export const LOCK_PASSWORD_ENV = 'TEST_MCP_PROJECTS_LOCK_PASSWORD'; +export const LOCK_DIR_ENV = 'TEST_MCP_PROJECTS_LOCK_DIR_NAME'; diff --git a/integtests/testproject/pool.ts b/integtests/testproject/pool.ts new file mode 100644 index 000000000..14a53b13c --- /dev/null +++ b/integtests/testproject/pool.ts @@ -0,0 +1,83 @@ +import { createFsLocker } from './fsLocker'; +import { LOCK_DIR_ENV, LOCK_HOST_ENV, LOCK_PASSWORD_ENV,type Locker } from './locker'; +import { loadProjects } from './projects'; +import { createRedisLocker } from './redisLocker'; +import { + type AcquireOptions, + isCompatible, + type LockedProject, + type ProjectDefinition, + storageApiUrl, +} from './types'; + +// Port of go-utils ProjectsPool.GetTestProject: try each compatible project once; if all are +// busy, sleep briefly and retry the WHOLE pool forever. The only hard error is "no compatible +// project exists in the pool at all". + +const RETRY_DELAY_MS = 100; +const WAIT_LOG_EVERY_MS = 5000; + +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +/** Chooses the redis locker when LOCK_HOST/PASSWORD are set, else the host-local fs fallback. */ +export const newLocker = (): Locker => { + const host = process.env[LOCK_HOST_ENV]; + const password = process.env[LOCK_PASSWORD_ENV]; + if (host) { + if (!password) throw new Error(`${LOCK_PASSWORD_ENV} is required when ${LOCK_HOST_ENV} is set.`); + return createRedisLocker(host, password); + } + return createFsLocker(process.env[LOCK_DIR_ENV]); +}; + +export type Pool = { + getTestProject: (opts?: AcquireOptions) => Promise; + close: () => Promise; +}; + +export const createPool = (defs: ProjectDefinition[], locker: Locker): Pool => { + const lockers = new Map(defs.map((d) => [d, locker.forProject(d)])); + + const getTestProject = async (opts: AcquireOptions = {}): Promise => { + const compatible = defs.filter((d) => isCompatible(d, opts)); + if (compatible.length === 0) { + throw new Error(`No compatible test project in the pool (backend=${opts.backend ?? 'any'}).`); + } + + let lastLog = 0; + // Randomize the start so parallel workers don't all stampede the first project. + const start = Math.floor(Math.random() * compatible.length); + for (let attempt = 0; ; attempt++) { + for (let i = 0; i < compatible.length; i++) { + const def = compatible[(start + i) % compatible.length]!; + const release = await lockers.get(def)!.tryLock(); + if (release) { + return { + definition: def, + storageApiUrl: storageApiUrl(def), + storageApiToken: def.token, + backend: def.backend, + release, + }; + } + } + const now = Date.now(); + if (now - lastLog >= WAIT_LOG_EVERY_MS) { + // eslint-disable-next-line no-console + console.info(`[testproject] all ${compatible.length} project(s) busy; waiting…`); + lastLog = now; + } + await sleep(RETRY_DELAY_MS); + } + }; + + return { getTestProject, close: () => locker.close() }; +}; + +let singleton: Pool | undefined; + +/** Process-singleton pool built from TEST_KBC_PROJECTS_FILE + the env-selected locker. */ +export const getPool = (): Pool => { + if (!singleton) singleton = createPool(loadProjects(), newLocker()); + return singleton; +}; diff --git a/integtests/testproject/projects.ts b/integtests/testproject/projects.ts new file mode 100644 index 000000000..20bafaa7e --- /dev/null +++ b/integtests/testproject/projects.ts @@ -0,0 +1,47 @@ +import { readFileSync } from 'node:fs'; +import { isAbsolute } from 'node:path'; + +import { type ProjectDefinition, projectDefinitionSchema } from './types'; + +// Port of go-utils getProjects / GetProjectsFrom: load + validate the projects.json pool +// once per process from TEST_KBC_PROJECTS_FILE (absolute path). + +export const TEST_KBC_PROJECTS_FILE = 'TEST_KBC_PROJECTS_FILE'; + +/** Parses + validates a projects.json document (the array form). */ +export const parseProjects = (json: string): ProjectDefinition[] => { + const raw: unknown = JSON.parse(json); + if (!Array.isArray(raw) || raw.length === 0) { + throw new Error( + 'projects.json must be a non-empty array of {host, project, token, backend, stagingStorage}.', + ); + } + return raw.map((entry, i) => { + const result = projectDefinitionSchema.safeParse(entry); + if (!result.success) { + throw new Error(`projects.json[${i}] is invalid: ${result.error.issues.map((x) => x.message).join('; ')}`); + } + return result.data; + }); +}; + +let cached: ProjectDefinition[] | undefined; + +/** Loads the pool once per process. `path` overrides TEST_KBC_PROJECTS_FILE (must be absolute). */ +export const loadProjects = (path?: string): ProjectDefinition[] => { + if (cached) return cached; + const file = path ?? process.env[TEST_KBC_PROJECTS_FILE]; + if (!file) { + throw new Error(`Set ${TEST_KBC_PROJECTS_FILE} to the absolute path of the projects.json pool file.`); + } + if (!isAbsolute(file)) { + throw new Error(`${TEST_KBC_PROJECTS_FILE} must be an absolute path, got: ${file}`); + } + cached = parseProjects(readFileSync(file, 'utf-8')); + return cached; +}; + +/** Test-only: clears the process-singleton cache. */ +export const _resetProjectsCache = (): void => { + cached = undefined; +}; diff --git a/integtests/testproject/redisLocker.ts b/integtests/testproject/redisLocker.ts new file mode 100644 index 000000000..e454e469c --- /dev/null +++ b/integtests/testproject/redisLocker.ts @@ -0,0 +1,105 @@ +import Redis from 'ioredis'; +import { randomUUID } from 'node:crypto'; + +import type { Locker, ProjectLocker, ReleaseFn } from './locker'; +import { lockKey, type ProjectDefinition } from './types'; + +/** + * Redis-backed cross-runner project lease — port of go-utils + * pkg/testproject/redislocker.go (which wraps bsm/redislock). + * + * We use raw ioredis + small Lua scripts instead of a lock library so the + * compare-and-swap semantics match the go implementation exactly: + * obtain : SET key token NX PX ttl + * refresh: if GET key == token then PEXPIRE key ttl (every ttl/4) + * release: if GET key == token then DEL key + * A crashed worker's lease therefore self-expires after at most TTL. + */ +const TTL_MS = 2 * 60 * 1000; +const REFRESH_MS = TTL_MS / 4; + +const REFRESH_LUA = + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('pexpire', KEYS[1], ARGV[2]) else return 0 end"; +const RELEASE_LUA = + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end"; + +/** Parses `redis://host:port` / `rediss://...` / a `+tls` suffix into ioredis options. */ +const parseRedisUrl = (url: string): { host: string; port: number; tls: boolean } => { + const sep = url.indexOf('://'); + if (sep === -1) throw new Error(`${url}: no protocol specified (expected redis://...)`); + const scheme = url.slice(0, sep); + const hostPort = url.slice(sep + 3); + const [host, port] = hostPort.split(':'); + return { + host: host || '127.0.0.1', + port: port ? Number(port) : 6379, + tls: scheme.includes('+tls') || scheme === 'rediss', + }; +}; + +export const createRedisLocker = (redisUrl: string, password: string): Locker => { + const { host, port, tls } = parseRedisUrl(redisUrl); + const client = new Redis({ + host, + port, + password, + ...(tls ? { tls: { minVersion: 'TLSv1.2' as const } } : {}), + // Tolerate a flaky/briefly-unavailable redis: retry each command a few times and + // reconnect with capped exponential backoff instead of failing immediately. + maxRetriesPerRequest: 3, + retryStrategy: (times) => Math.min(times * 200, 2000), + }); + client.on('error', (err) => { + // ioredis emits connection errors on this channel; without a listener they become + // unhandled 'error' events that crash the process. Swallow — command-level failures + // surface where they're awaited and are handled there. + console.warn(`[testproject] redis error: ${err.message}`); + }); + + const forProject = (def: ProjectDefinition): ProjectLocker => { + const key = lockKey(def); + return { + tryLock: async (): Promise => { + const token = randomUUID(); + let ok: string | null; + try { + ok = await client.set(key, token, 'PX', TTL_MS, 'NX'); + } catch (err) { + // Transient redis failure (after the per-command retries above): treat the project + // as unavailable for now so the pool moves on and retries, rather than failing the + // whole test on a blip. A persistently-down redis is bounded by the job timeout. + console.warn( + `[testproject] redis SET failed for ${key}: ${err instanceof Error ? err.message : String(err)}`, + ); + return null; + } + if (ok !== 'OK') return null; + + // Auto-extend the lease for the (unknown) lifetime of the test. + const timer = setInterval(() => { + client.eval(REFRESH_LUA, 1, key, token, String(TTL_MS)).catch(() => { + /* best-effort; lease will expire on its own if refresh fails */ + }); + }, REFRESH_MS); + timer.unref?.(); + + let released = false; + return async (): Promise => { + if (released) return; + released = true; + clearInterval(timer); + await client.eval(RELEASE_LUA, 1, key, token).catch(() => { + /* lease will expire via TTL even if the explicit release fails */ + }); + }; + }, + }; + }; + + return { + forProject, + close: async (): Promise => { + await client.quit().catch(() => client.disconnect()); + }, + }; +}; diff --git a/integtests/testproject/types.ts b/integtests/testproject/types.ts new file mode 100644 index 000000000..f58e668c7 --- /dev/null +++ b/integtests/testproject/types.ts @@ -0,0 +1,45 @@ +import { z } from 'zod'; + +// Port of go-utils pkg/testproject Definition. The projects.json array uses this exact +// shape (same as keboola/go-monorepo build/ci/projects.json). + +export const BACKENDS = ['snowflake', 'bigquery'] as const; +export type Backend = (typeof BACKENDS)[number]; + +export const STAGING_STORAGES = ['abs', 'gcs', 's3'] as const; +export type StagingStorage = (typeof STAGING_STORAGES)[number]; + +export const projectDefinitionSchema = z.object({ + host: z.string().min(1), + project: z.number().int().positive(), + token: z.string().min(1), + backend: z.enum(BACKENDS), + stagingStorage: z.enum(STAGING_STORAGES), + legacyTransformation: z.boolean().default(false), + isGuest: z.boolean().default(false), +}); + +export type ProjectDefinition = z.infer; + +/** Unique redis/fs lock key for a project: host + numeric id (port of go `host-projectID`). */ +export const lockKey = (def: ProjectDefinition): string => `${def.host}-${def.project}`; + +/** Storage API base URL derived from the project's bare `host`. */ +export const storageApiUrl = (def: ProjectDefinition): string => `https://${def.host}`; + +/** A project leased for the duration of one test; `release` frees the lease. */ +export type LockedProject = { + definition: ProjectDefinition; + storageApiUrl: string; + storageApiToken: string; + backend: Backend; + release: () => Promise; +}; + +/** Optional selector when a test requires a specific backend. */ +export type AcquireOptions = { + backend?: Backend; +}; + +export const isCompatible = (def: ProjectDefinition, opts: AcquireOptions): boolean => + !opts.backend || def.backend === opts.backend; diff --git a/integtests/tools/__init__.py b/integtests/tools/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/integtests/tools/components.test.ts b/integtests/tools/components.test.ts new file mode 100644 index 000000000..2133289a7 --- /dev/null +++ b/integtests/tools/components.test.ts @@ -0,0 +1,503 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolText, connectMcp } from '../helpers/mcp'; +import { seedProject } from '../helpers/seed'; +import { getTestProjectForTest } from '../testproject/fixture'; + +// Ported from integtests/tools/components/test_tools.py. The Python suite calls the tool +// functions directly and asserts on typed Pydantic models; here the tools return TOON text +// via the MCP client, so we assert on substrings/regex. Read tests seed the standard fixtures +// (two configs: ex-generic-v2 extractor + keboola.snowflake-transformation transformation); +// write tests lease a clean project and create their own data. + +const EXTRACTOR_COMPONENT = 'ex-generic-v2'; +const TRANSFORMATION_COMPONENT = 'keboola.snowflake-transformation'; + +describe('components tools (integration)', () => { + // --- get_configs --------------------------------------------------------- + + it('get_configs returns detailed configuration for specific configs', async () => { + const project = await getTestProjectForTest(); + const seeded = await seedProject(project); + const session = await connectMcp(project.config); + try { + for (const cfg of seeded.configs) { + const text = await callToolText(session.client, 'get_configs', { + configs: [ + { component_id: cfg.componentId, configuration_id: cfg.configurationId }, + ], + }); + // Detailed Configuration output carries the component, the configuration_root with the + // matching IDs, and a non-empty links list. + expect(text).toContain(cfg.componentId); + expect(text).toContain(cfg.configurationId); + expect(text).toMatch(/component_type/); + expect(text).toMatch(/component_name/); + expect(text).toMatch(/links/); + } + } finally { + await session.close(); + } + }); + + it('get_configs lists components filtered by component IDs', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_configs', { + component_ids: [EXTRACTOR_COMPONENT, TRANSFORMATION_COMPONENT], + }); + expect(text).toContain(EXTRACTOR_COMPONENT); + expect(text).toContain(TRANSFORMATION_COMPONENT); + expect(text).toMatch(/components_with_configs|component/); + } finally { + await session.close(); + } + }); + + // Mirrors the Python parametrized test_get_configs_list_by_types: the seeded project has one + // extractor and one transformation config. + it.each([ + { types: ['extractor'], expected: [EXTRACTOR_COMPONENT], absent: [TRANSFORMATION_COMPONENT] }, + { + types: ['transformation'], + expected: [TRANSFORMATION_COMPONENT], + absent: [EXTRACTOR_COMPONENT], + }, + { + types: ['application', 'extractor', 'transformation'], + expected: [EXTRACTOR_COMPONENT, TRANSFORMATION_COMPONENT], + absent: [], + }, + { types: [], expected: [EXTRACTOR_COMPONENT, TRANSFORMATION_COMPONENT], absent: [] }, + ])('get_configs filters by component types %o', async ({ types, expected, absent }) => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_configs', { component_types: types }); + for (const id of expected) expect(text).toContain(id); + for (const id of absent) expect(text).not.toContain(id); + } finally { + await session.close(); + } + }); + + // --- get_components ------------------------------------------------------ + + it('get_components returns details for multiple components', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_components', { + component_ids: [EXTRACTOR_COMPONENT, TRANSFORMATION_COMPONENT], + }); + expect(text).toContain(EXTRACTOR_COMPONENT); + expect(text).toContain(TRANSFORMATION_COMPONENT); + expect(text).toMatch(/component_name/); + expect(text).toMatch(/component_type/); + // both per-component links and output-level links should be present + expect(text).toMatch(/links/); + } finally { + await session.close(); + } + }); + + // --- get_config_examples ------------------------------------------------- + + it('get_config_examples returns markdown examples for a component', async () => { + const project = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_config_examples', { + component_id: EXTRACTOR_COMPONENT, + }); + expect(text).toContain(`# Configuration Examples for \`${EXTRACTOR_COMPONENT}\``); + expect(text).toContain('parameters'); + } finally { + await session.close(); + } + }); + + it('get_config_examples returns empty string for an invalid component', async () => { + const project = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_config_examples', { + component_id: 'completely-non-existent-component-12345', + }); + expect(text).toBe(''); + } finally { + await session.close(); + } + }); + + // --- create_config ------------------------------------------------------- + + it('create_config creates a configuration with success metadata and links', async () => { + const project = await getTestProjectForTest(); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'create_config', { + name: 'Test Configuration', + description: 'Test configuration created by automated test', + component_id: EXTRACTOR_COMPONENT, + parameters: {}, + storage: {}, + }); + expect(text).toContain(EXTRACTOR_COMPONENT); + expect(text).toContain('Test configuration created by automated test'); + expect(text).toMatch(/success/); + expect(text).toMatch(/true/); + // links: ui-detail + ui-dashboard for the created configuration + expect(text).toContain(`/components/${EXTRACTOR_COMPONENT}`); + expect(text).toContain('Configuration: Test Configuration'); + } finally { + await session.close(); + } + }); + + // --- add_config_row ------------------------------------------------------ + + it('add_config_row adds a row to a configuration', async () => { + const project = await getTestProjectForTest(); + const session = await connectMcp(project.config); + try { + const rootText = await callToolText(session.client, 'create_config', { + name: 'Root Configuration for Row Test', + description: 'Root configuration created for row configuration test', + component_id: EXTRACTOR_COMPONENT, + parameters: {}, + storage: {}, + }); + const configurationId = extractConfigurationId(rootText); + + const rowText = await callToolText(session.client, 'add_config_row', { + name: 'Test Row Configuration', + description: 'Test row configuration created by automated test', + component_id: EXTRACTOR_COMPONENT, + configuration_id: configurationId, + parameters: { row_param: 'row_value' }, + storage: {}, + }); + expect(rowText).toContain(EXTRACTOR_COMPONENT); + expect(rowText).toContain(configurationId); + expect(rowText).toContain('Test row configuration created by automated test'); + expect(rowText).toMatch(/success/); + expect(rowText).toContain('Configuration: Test Row Configuration'); + } finally { + await session.close(); + } + }); + + // --- update_config ------------------------------------------------------- + + // Mirrors Python test_update_config parametrize cases (only the fields-affecting-output ones + // are asserted via text, since backend re-fetch is not available through the MCP client). + it.each([ + { + label: 'all fields', + updates: { + name: 'Updated Test Configuration', + description: 'Updated test configuration by automated test', + parameter_updates: [{ op: 'set', path: 'updated_param', value: 'updated_value' }], + storage: { output: { tables: [{ source: 'output.csv', destination: 'out.c-bucket.table' }] } }, + }, + expectName: 'Updated Test Configuration', + expectDescription: 'Updated test configuration by automated test', + }, + { + label: 'just name', + updates: { name: 'Updated just name' }, + expectName: 'Updated just name', + expectDescription: 'Initial test configuration created by automated test', + }, + { + label: 'just description', + updates: { description: 'Updated just description' }, + expectName: 'Initial Test Configuration', + expectDescription: 'Updated just description', + }, + { + label: 'just parameters', + updates: { + parameter_updates: [{ op: 'set', path: 'updated_param', value: 'Updated just parameters' }], + }, + expectName: 'Initial Test Configuration', + expectDescription: 'Initial test configuration created by automated test', + }, + { + label: 'just storage', + updates: { + storage: { output: { tables: [{ source: 'output.csv', destination: 'out.c-bucket.table' }] } }, + }, + expectName: 'Initial Test Configuration', + expectDescription: 'Initial test configuration created by automated test', + }, + ])('update_config updates a configuration ($label)', async ({ updates, expectName, expectDescription }) => { + const project = await getTestProjectForTest(); + const session = await connectMcp(project.config); + try { + const createText = await callToolText(session.client, 'create_config', { + name: 'Initial Test Configuration', + description: 'Initial test configuration created by automated test', + component_id: EXTRACTOR_COMPONENT, + parameters: { initial_param: 'initial_value' }, + storage: { input: { tables: [{ source: 'in.c-bucket.table', destination: 'input.csv' }] } }, + }); + const configurationId = extractConfigurationId(createText); + + const text = await callToolText(session.client, 'update_config', { + change_description: 'Integration test update', + component_id: EXTRACTOR_COMPONENT, + configuration_id: configurationId, + ...updates, + }); + expect(text).toContain(EXTRACTOR_COMPONENT); + expect(text).toContain(configurationId); + expect(text).toMatch(/success/); + expect(text).toContain(expectDescription); + expect(text).toContain(`Configuration: ${expectName}`); + } finally { + await session.close(); + } + }); + + // --- update_config_row --------------------------------------------------- + + it.each([ + { + label: 'all fields', + updates: { + name: 'Updated Row Configuration', + description: 'Updated row configuration by automated test', + parameter_updates: [{ op: 'set', path: '$', value: { updated_row_param: 'updated_row_value' } }], + storage: {}, + }, + expectName: 'Updated Row Configuration', + expectDescription: 'Updated row configuration by automated test', + }, + { + label: 'just name', + updates: { name: 'Updated just name' }, + expectName: 'Updated just name', + expectDescription: 'Initial row configuration for update test', + }, + { + label: 'just description', + updates: { description: 'Updated just description' }, + expectName: 'Initial Test Row Configuration', + expectDescription: 'Updated just description', + }, + { + label: 'is_disabled', + updates: { is_disabled: true }, + expectName: 'Initial Test Row Configuration', + expectDescription: 'Initial row configuration for update test', + }, + ])('update_config_row updates a row configuration ($label)', async ({ updates, expectName, expectDescription }) => { + const project = await getTestProjectForTest(); + const session = await connectMcp(project.config); + try { + const createText = await callToolText(session.client, 'create_config', { + name: 'Initial Test Configuration', + description: 'Initial test configuration created by automated test', + component_id: EXTRACTOR_COMPONENT, + parameters: { initial_param: 'initial_value' }, + storage: {}, + }); + const configurationId = extractConfigurationId(createText); + + // create the initial row + await callToolText(session.client, 'add_config_row', { + name: 'Initial Test Row Configuration', + description: 'Initial row configuration for update test', + component_id: EXTRACTOR_COMPONENT, + configuration_id: configurationId, + parameters: { initial_row_param: 'initial_row_value' }, + storage: {}, + }); + + // fetch the row id from the configuration detail (raw Storage API) + const rowId = await fetchFirstRowId(project, EXTRACTOR_COMPONENT, configurationId); + + const text = await callToolText(session.client, 'update_config_row', { + change_description: 'Integration test update', + component_id: EXTRACTOR_COMPONENT, + configuration_id: configurationId, + configuration_row_id: rowId, + ...updates, + }); + expect(text).toContain(EXTRACTOR_COMPONENT); + expect(text).toContain(configurationId); + expect(text).toMatch(/success/); + expect(text).toContain(expectDescription); + expect(text).toContain(`Configuration: ${expectName}`); + } finally { + await session.close(); + } + }); + + // --- create_sql_transformation ------------------------------------------- + + it('create_sql_transformation creates a SQL transformation', async () => { + const project = await getTestProjectForTest({ backend: 'snowflake' }); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'create_sql_transformation', { + name: 'Test SQL Transformation', + description: 'Test SQL transformation created by automated test', + sql_code_blocks: [ + { + name: 'Main transformation', + script: 'SELECT 1 as test_column; SELECT 2 as another_column;', + }, + ], + created_table_names: ['test_output_table'], + }); + expect(text).toContain('Test SQL transformation created by automated test'); + expect(text).toContain('keboola.snowflake-transformation'); + expect(text).toMatch(/success/); + expect(text).toContain('Transformation: Test SQL Transformation'); + expect(text).toContain('Transformations dashboard'); + } finally { + await session.close(); + } + }); + + // --- update_sql_transformation ------------------------------------------- + + it.each([ + { + label: 'all fields', + updates: { + name: 'Updated SQL transformation name', + description: 'Updated SQL transformation description', + parameter_updates: [ + { op: 'rename_block', block_id: 'b0', block_name: 'Updated block' }, + { op: 'rename_code', block_id: 'b0', code_id: 'b0.c0', code_name: 'Updated code' }, + { + op: 'set_code', + block_id: 'b0', + code_id: 'b0.c0', + script: + 'SELECT 1 as updated_column;\n\nSELECT 2 as additional_column;\n\nSELECT 3 as third_column;\n\n', + }, + ], + storage: { + input: { tables: [{ source: 'in.c-bucket.input_table', destination: 'input.csv' }] }, + output: { + tables: [ + { source: 'updated_output_table', destination: 'out.c-bucket.updated_output_table' }, + { source: 'second_output_table', destination: 'out.c-bucket.second_output_table' }, + ], + }, + }, + }, + expectName: 'Updated SQL transformation name', + expectDescription: 'Updated SQL transformation description', + }, + { + label: 'just name', + updates: { name: 'Updated SQL transformation name' }, + expectName: 'Updated SQL transformation name', + expectDescription: 'Initial SQL transformation for update test', + }, + { + label: 'just description', + updates: { description: 'Updated SQL transformation description' }, + expectName: 'Initial Test SQL Transformation', + expectDescription: 'Updated SQL transformation description', + }, + { + label: 'just parameters', + updates: { + parameter_updates: [ + { + op: 'str_replace', + block_id: 'b0', + code_id: 'b0.c0', + search_for: 'SELECT 1', + replace_with: 'SELECT 12', + }, + { + op: 'add_script', + block_id: 'b0', + code_id: 'b0.c0', + script: 'SELECT 2 as additional_column', + position: 'end', + }, + ], + }, + expectName: 'Initial Test SQL Transformation', + expectDescription: 'Initial SQL transformation for update test', + }, + { + label: 'just storage', + updates: { + storage: { + input: { tables: [{ source: 'in.c-bucket.input_table', destination: 'input.csv' }] }, + output: { + tables: [ + { source: 'updated_output_table', destination: 'out.c-bucket.updated_output_table' }, + { source: 'second_output_table', destination: 'out.c-bucket.second_output_table' }, + ], + }, + }, + }, + expectName: 'Initial Test SQL Transformation', + expectDescription: 'Initial SQL transformation for update test', + }, + ])('update_sql_transformation updates a transformation ($label)', async ({ updates, expectName, expectDescription }) => { + const project = await getTestProjectForTest({ backend: 'snowflake' }); + const session = await connectMcp(project.config); + try { + const createText = await callToolText(session.client, 'create_sql_transformation', { + name: 'Initial Test SQL Transformation', + description: 'Initial SQL transformation for update test', + sql_code_blocks: [{ name: 'Initial transformation', script: 'SELECT 1 as initial_column;' }], + created_table_names: ['initial_output_table'], + }); + const configurationId = extractConfigurationId(createText); + + const text = await callToolText(session.client, 'update_sql_transformation', { + change_description: 'Integration test update', + configuration_id: configurationId, + ...updates, + }); + expect(text).toContain('keboola.snowflake-transformation'); + expect(text).toContain(configurationId); + expect(text).toMatch(/success/); + expect(text).toContain(expectDescription); + expect(text).toContain(`Transformation: ${expectName}`); + } finally { + await session.close(); + } + }); +}); + +/** Pulls the configuration_id out of a TOON ConfigToolOutput text blob. */ +function extractConfigurationId(text: string): string { + const match = text.match(/configuration_id:\s*([^\s,]+)/); + if (!match) throw new Error(`Could not find configuration_id in tool output:\n${text}`); + return match[1]!.replace(/["']/g, ''); +} + +/** Reads the first row id of a configuration via the raw Storage API. */ +async function fetchFirstRowId( + project: { storageApiUrl: string; storageApiToken: string }, + componentId: string, + configurationId: string, +): Promise { + const res = await fetch( + `${project.storageApiUrl}/v2/storage/branch/default/components/${componentId}/configs/${configurationId}`, + { headers: { 'X-StorageApi-Token': project.storageApiToken } }, + ); + if (!res.ok) throw new Error(`Failed to fetch config detail: ${res.status} ${await res.text()}`); + const detail = (await res.json()) as { rows?: { id: string }[] }; + const rows = detail.rows ?? []; + if (rows.length === 0) throw new Error('No rows found in configuration'); + return String(rows[0]!.id); +} diff --git a/integtests/tools/components/__init__.py b/integtests/tools/components/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/integtests/tools/components/test_tools.py b/integtests/tools/components/test_tools.py deleted file mode 100644 index d73d4cf69..000000000 --- a/integtests/tools/components/test_tools.py +++ /dev/null @@ -1,1052 +0,0 @@ -import logging -from typing import Any, AsyncGenerator, cast - -import pytest -import pytest_asyncio -import toon_format -from fastmcp import Client, Context -from pydantic import TypeAdapter - -from integtests.conftest import ConfigDef, ProjectDef -from keboola_mcp_server.clients.client import KeboolaClient, get_metadata_property -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.components import ( - add_config_row, - create_config, - create_sql_transformation, - get_components, - get_config_examples, - get_configs, -) -from keboola_mcp_server.tools.components.model import ( - ComponentType, - ComponentWithConfigs, - ConfigParamUpdate, - ConfigToolOutput, - Configuration, - FullConfigId, - GetComponentsOutput, - GetConfigsDetailOutput, - GetConfigsListOutput, - SimplifiedTfBlocks, - TfAddScript, - TfParamUpdate, - TfRenameBlock, - TfRenameCode, - TfSetCode, - TfStrReplace, - TransformationConfiguration, -) -from keboola_mcp_server.tools.components.sql_utils import split_sql_statements -from keboola_mcp_server.tools.components.utils import ( - clean_bucket_name, - expand_component_types, - get_sql_transformation_id_from_sql_dialect, - update_params, - update_transformation_parameters, -) -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - - -@pytest.mark.asyncio -async def test_get_configs_detail(mcp_context: Context, configs: list[ConfigDef]): - """Tests that `get_configs` with specific configs returns detailed `Configuration` instances.""" - - for config in configs: - assert config.configuration_id is not None - - result = await get_configs( - ctx=mcp_context, - configs=[FullConfigId(component_id=config.component_id, configuration_id=config.configuration_id)], - ) - - assert isinstance(result, GetConfigsDetailOutput) - assert len(result.configs) == 1 - - configuration = result.configs[0] - assert isinstance(configuration, Configuration) - assert configuration.component is not None - assert configuration.component.component_id == config.component_id - assert configuration.component.component_type is not None - assert configuration.component.component_name is not None - - assert configuration.configuration_root is not None - assert configuration.configuration_root.configuration_id == config.configuration_id - assert configuration.configuration_root.component_id == config.component_id - # Check links field - assert configuration.links, 'Links list should not be empty.' - for link in configuration.links: - assert isinstance(link, Link) - - -@pytest.mark.asyncio -async def test_get_configs_list_by_ids(mcp_context: Context, configs: list[ConfigDef]): - """Tests that `get_configs` returns components filtered by component IDs.""" - - # Get unique component IDs from test configs - component_ids = list({config.component_id for config in configs}) - assert len(component_ids) > 0 - - result = await get_configs(ctx=mcp_context, component_ids=component_ids) - - # Verify result structure and content - assert isinstance(result, GetConfigsListOutput) - assert len(result.components_with_configs) == len(component_ids) - - for item in result.components_with_configs: - assert isinstance(item, ComponentWithConfigs) - assert item.component.component_id in component_ids - - # Check that configurations belong to this component - for config in item.configs: - assert config.configuration_root.component_id == item.component.component_id - - -@pytest.mark.skip(reason='bug in toon_format library') -@pytest.mark.asyncio -async def test_get_configs_output_format(mcp_client: Client, configs: list[ConfigDef]): - """Tests that `get_configs` returns the tool output in TOON format.""" - # Temporarily skip this test due to bug in the toon-format library: - # See: https://github.com/toon-format/toon-python/pull/36 - # The bug creates TOON which not valid according to the TOON specs but still readable to the agents. - component_ids = list({config.component_id for config in configs}) - assert len(component_ids) > 0 - - tool_result = await mcp_client.call_tool(name='get_configs', arguments={'component_ids': component_ids}) - - # Verify structured content - assert tool_result.structured_content is not None - result = GetConfigsListOutput.model_validate(tool_result.structured_content) - assert len(result.components_with_configs) > 0 - - # Verify TOON formatted text content matches structured content - assert len(tool_result.content) == 1 - assert tool_result.content[0].type == 'text' - toon_decoded = toon_format.decode(tool_result.content[0].text) - assert GetConfigsListOutput.model_validate(toon_decoded) == result - - -@pytest.mark.parametrize( - ('component_types', 'expected_count'), - [ - (['extractor'], 1), - (['transformation'], 1), - (['application', 'extractor', 'transformation'], 2), - ([], 2), - ], -) -@pytest.mark.asyncio -async def test_get_configs_list_by_types( - mcp_context: Context, configs: list[ConfigDef], component_types: list[ComponentType], expected_count: int -): - """Tests that `get_configs` returns components filtered by component types.""" - - result = await get_configs(ctx=mcp_context, component_types=component_types) - - assert isinstance(result, GetConfigsListOutput) - - assert sum(len(cmp.configs) for cmp in result.components_with_configs) == expected_count - - for item in result.components_with_configs: - assert isinstance(item, ComponentWithConfigs) - assert item.component.component_type in expand_component_types(component_types) - - -@pytest.mark.asyncio -async def test_create_config( - mcp_context: Context, configs: list[ConfigDef], keboola_project: ProjectDef, storage_api_url: str -): - """Tests that `create_config` creates a configuration with correct metadata.""" - - # Use the first component from configs for testing - test_config = configs[0] - component_id = test_config.component_id - - # Define test configuration data - test_name = 'Test Configuration' - test_description = 'Test configuration created by automated test' - test_parameters = {} - test_storage = {} - - client = KeboolaClient.from_state(mcp_context.session.state) - - project_id = keboola_project.project_id - - # Create the configuration - created_config = await create_config( - ctx=mcp_context, - name=test_name, - description=test_description, - component_id=component_id, - parameters=test_parameters, - storage=test_storage, - ) - try: - # Verify the response structure - assert isinstance(created_config, ConfigToolOutput) - assert created_config.component_id == component_id - assert created_config.configuration_id is not None - assert created_config.description == test_description - assert created_config.success is True - assert created_config.timestamp is not None - assert created_config.version is not None - assert frozenset(created_config.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Configuration: {test_name}', - url=( - f'{storage_api_url}/admin/projects/{project_id}/components/{component_id}/' - + f'{created_config.configuration_id}' - ), - ), - Link( - type='ui-dashboard', - title=f'Component "{component_id}" Configurations Dashboard', - url=f'{storage_api_url}/admin/projects/{project_id}/components/{component_id}', - ), - ] - ) - - # Verify the configuration exists in the backend by fetching it - config_detail = await client.storage_client.configuration_detail( - component_id=component_id, configuration_id=created_config.configuration_id - ) - - assert config_detail['name'] == test_name - assert config_detail['description'] == test_description - assert 'configuration' in config_detail - - # Verify the parameters and storage were set correctly - configuration_data = cast(dict, config_detail['configuration']) - assert configuration_data['parameters'] == test_parameters - assert configuration_data['storage'] == test_storage - - # Verify the metadata - check that KBC.MCP.createdBy is set to 'true' - metadata = await client.storage_client.configuration_metadata_get( - component_id=component_id, configuration_id=created_config.configuration_id - ) - - # Convert metadata list to dictionary for easier checking - # metadata is a list of dicts with 'key' and 'value' keys - assert isinstance(metadata, list) - metadata_dict = {item['key']: item['value'] for item in metadata if isinstance(item, dict)} - assert MetadataField.CREATED_BY_MCP in metadata_dict - assert metadata_dict[MetadataField.CREATED_BY_MCP] == 'true' - - finally: - # Clean up: Delete the configuration - await client.storage_client.configuration_delete( - component_id=component_id, - configuration_id=created_config.configuration_id, - skip_trash=True, - ) - - -@pytest_asyncio.fixture -async def initial_cmpconf( - mcp_client: Client, configs: list[ConfigDef], keboola_client: KeboolaClient -) -> AsyncGenerator[ConfigToolOutput, None]: - # Create the initial component configuration test data - tool_result = await mcp_client.call_tool( - name='create_config', - arguments={ - 'name': 'Initial Test Configuration', - 'description': 'Initial test configuration created by automated test', - 'component_id': configs[0].component_id, - 'parameters': {'initial_param': 'initial_value'}, - 'storage': {'input': {'tables': [{'source': 'in.c-bucket.table', 'destination': 'input.csv'}]}}, - }, - ) - try: - yield ConfigToolOutput.model_validate(tool_result.structured_content) - finally: - # Clean up: Delete the configuration - await keboola_client.storage_client.configuration_delete( - component_id=configs[0].component_id, - configuration_id=tool_result.structured_content['configuration_id'], - skip_trash=True, - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - 'updates', - [ - { - 'name': 'Updated Test Configuration', - 'description': 'Updated test configuration by automated test', - 'parameter_updates': [{'op': 'set', 'path': 'updated_param', 'value': 'updated_value'}], - 'storage': {'output': {'tables': [{'source': 'output.csv', 'destination': 'out.c-bucket.table'}]}}, - }, - {'name': 'Updated just name'}, - {'description': 'Updated just description'}, - {'parameter_updates': [{'op': 'set', 'path': 'updated_param', 'value': 'Updated just parameters'}]}, - {'storage': {'output': {'tables': [{'source': 'output.csv', 'destination': 'out.c-bucket.table'}]}}}, - ], -) -async def test_update_config( - updates: dict[str, Any], - initial_cmpconf: ConfigToolOutput, - mcp_client: Client, - keboola_project: ProjectDef, - keboola_client: KeboolaClient, - storage_api_url: str, -): - """Tests that `update_config` updates a configuration with correct metadata.""" - project_id = keboola_project.project_id - component_id = initial_cmpconf.component_id - configuration_id = initial_cmpconf.configuration_id - param_update_dicts = updates.get('parameter_updates') - - if param_update_dicts is not None: - # Get the original configuration so we can compare the parameters - orig_config = await keboola_client.storage_client.configuration_detail( - component_id=component_id, configuration_id=configuration_id - ) - orig_parameters = cast(dict, orig_config.get('configuration', {}).get('parameters', {})) - - # Convert the parameter update dicts to ConfigParamUpdate objects - param_updates = [] - for update_dict in param_update_dicts: - update = TypeAdapter(ConfigParamUpdate).validate_python(update_dict) - param_updates.append(update) - - tool_result = await mcp_client.call_tool( - name='update_config', - arguments={ - 'change_description': 'Integration test update', - 'component_id': component_id, - 'configuration_id': configuration_id, - **updates, - }, - ) - - # Check the tool's output - update_result = ConfigToolOutput.model_validate(tool_result.structured_content) - assert update_result.component_id == component_id - assert update_result.configuration_id == configuration_id - assert update_result.success is True - assert update_result.timestamp is not None - assert update_result.version is not None - - expected_name = updates.get('name') or 'Initial Test Configuration' - expected_description = updates.get('description') or initial_cmpconf.description - assert update_result.description == expected_description - assert frozenset(update_result.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Configuration: {expected_name}', - url=f'{storage_api_url}/admin' f'/projects/{project_id}/components/{component_id}/{configuration_id}', - ), - Link( - type='ui-dashboard', - title=f'Component "{component_id}" Configurations Dashboard', - url=f'{storage_api_url}/admin/projects/{project_id}/components/{component_id}', - ), - ] - ) - - # Verify the configuration was updated - updated_config = await keboola_client.storage_client.configuration_detail( - component_id=update_result.component_id, configuration_id=update_result.configuration_id - ) - - assert updated_config['name'] == expected_name - assert updated_config['description'] == expected_description - - updated_config_data = updated_config.get('configuration') - assert isinstance(updated_config_data, dict), f'Expecting dict, got: {type(updated_config_data)}' - - if param_update_dicts is not None: - expected_parameters = update_params(orig_parameters, param_updates) - assert updated_config_data['parameters'] == expected_parameters - - if (expected_storage := updates.get('storage')) is not None: - # Storage API might return more keys than what we set, so we check subset - for k, v in expected_storage.items(): - assert k in updated_config_data['storage'] - assert updated_config_data['storage'][k] == v - - current_version = updated_config['version'] - assert isinstance(current_version, int), f'Expecting int, got: {type(current_version)}' - assert current_version == 2 - - # Check that KBC.MCP.updatedBy.version.{version} is set to 'true' - metadata = await keboola_client.storage_client.configuration_metadata_get( - component_id=update_result.component_id, configuration_id=update_result.configuration_id - ) - assert isinstance(metadata, list), f'Expecting list, got: {type(metadata)}' - - meta_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{current_version}' - meta_value = get_metadata_property(metadata, meta_key) - assert meta_value == 'true' - # Check that the original creation metadata is still there - assert get_metadata_property(metadata, MetadataField.CREATED_BY_MCP) == 'true' - - -@pytest.mark.asyncio -async def test_add_config_row( - mcp_context: Context, configs: list[ConfigDef], keboola_project: ProjectDef, storage_api_url: str -): - """Tests that `add_config_row` creates a row configuration with correct metadata.""" - - # Use the first component from configs for testing - test_config = configs[0] - component_id = test_config.component_id - - # Define root configuration test data - root_config_name = 'Root Configuration for Row Test' - root_config_description = 'Root configuration created for row configuration test' - root_config_parameters = {} - root_config_storage = {} - - # Define row configuration test data - row_name = 'Test Row Configuration' - row_description = 'Test row configuration created by automated test' - row_parameters = {'row_param': 'row_value'} - row_storage = {} - - client = KeboolaClient.from_state(mcp_context.session.state) - - project_id = keboola_project.project_id - - # First create a root configuration to add row to - root_config = await create_config( - ctx=mcp_context, - name=root_config_name, - description=root_config_description, - component_id=component_id, - parameters=root_config_parameters, - storage=root_config_storage, - ) - - try: - - # Create the row configuration - created_row_config = await add_config_row( - ctx=mcp_context, - name=row_name, - description=row_description, - component_id=component_id, - configuration_id=root_config.configuration_id, - parameters=row_parameters, - storage=row_storage, - ) - - assert isinstance(created_row_config, ConfigToolOutput) - assert created_row_config.success is True - assert created_row_config.timestamp is not None - assert created_row_config.description == row_description - assert created_row_config.component_id == component_id - assert created_row_config.configuration_id == root_config.configuration_id - assert created_row_config.version is not None - assert frozenset(created_row_config.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Configuration: {row_name}', - url=( - f'{storage_api_url}/admin/projects/{project_id}/components/{component_id}/' - + f'{root_config.configuration_id}' - ), - ), - Link( - type='ui-dashboard', - title=f'Component "{component_id}" Configurations Dashboard', - url=f'{storage_api_url}/admin/projects/{project_id}/components/{component_id}', - ), - ] - ) - - # Verify the row configuration exists by fetching the root configuration and checking its rows - config_detail = await client.storage_client.configuration_detail( - component_id=component_id, configuration_id=root_config.configuration_id - ) - - assert 'rows' in config_detail - rows = cast(list, config_detail['rows']) - assert len(rows) == 1 - - # Find the row we just created - created_row = None - for row in rows: - if isinstance(row, dict) and row.get('name') == row_name: - created_row = row - break - - assert created_row is not None - assert created_row['description'] == row_description - assert 'configuration' in created_row - - # Verify the parameters and storage were set correctly - row_configuration_data = cast(dict, created_row['configuration']) - assert row_configuration_data['parameters'] == row_parameters - assert row_configuration_data['storage'] == row_storage - - # Verify metadata was set for the parent configuration - metadata = await client.storage_client.configuration_metadata_get( - component_id=component_id, configuration_id=root_config.configuration_id - ) - - assert isinstance(metadata, list) - metadata_dict = {item['key']: item['value'] for item in metadata if isinstance(item, dict)} - # The updated metadata should be present since we added a row to the configuration - updated_by_md_keys = [ - key - for key in metadata_dict.keys() - if isinstance(key, str) and key.startswith(MetadataField.UPDATED_BY_MCP_PREFIX) - ] - assert len(updated_by_md_keys) > 0 - - finally: - # Delete the configuration (this will also delete the rows) - await client.storage_client.configuration_delete( - component_id=component_id, - configuration_id=root_config.configuration_id, - skip_trash=True, - ) - - -@pytest_asyncio.fixture -async def initial_cmpconf_row( - initial_cmpconf: ConfigToolOutput, mcp_client: Client, keboola_client: KeboolaClient -) -> ConfigToolOutput: - # Create initial row configuration test data - tool_result = await mcp_client.call_tool( - name='add_config_row', - arguments={ - 'name': 'Initial Test Row Configuration', - 'description': 'Initial row configuration for update test', - 'component_id': initial_cmpconf.component_id, - 'configuration_id': initial_cmpconf.configuration_id, - 'parameters': {'initial_row_param': 'initial_row_value'}, - 'storage': {}, - }, - ) - return ConfigToolOutput.model_validate(tool_result.structured_content) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - 'updates', - [ - { - 'name': 'Updated Row Configuration', - 'description': 'Updated row configuration by automated test', - 'parameter_updates': [{'op': 'set', 'path': '$', 'value': {'updated_row_param': 'updated_row_value'}}], - 'storage': {}, - }, - {'name': 'Updated just name'}, - {'description': 'Updated just description'}, - {'parameter_updates': [{'op': 'set', 'path': '$', 'value': {'updated_row_param': 'Updated just parameters'}}]}, - {'storage': {'output': {'tables': [{'source': 'output.csv', 'destination': 'out.c-bucket.table'}]}}}, - {'is_disabled': True}, - ], -) -async def test_update_config_row( - updates: dict[str, Any], - initial_cmpconf_row: ConfigToolOutput, - mcp_client: Client, - keboola_project: ProjectDef, - keboola_client: KeboolaClient, - storage_api_url: str, -): - """Tests that `update_config_row` updates a row configuration with correct metadata.""" - project_id = keboola_project.project_id - component_id = initial_cmpconf_row.component_id - configuration_id = initial_cmpconf_row.configuration_id - - # Get the row ID from the configuration detail - config_detail = await keboola_client.storage_client.configuration_detail( - component_id=component_id, configuration_id=configuration_id - ) - rows = config_detail['rows'] - assert isinstance(rows, list) - assert len(rows) == 1 - row_id = rows[0]['id'] - - tool_result = await mcp_client.call_tool( - name='update_config_row', - arguments={ - 'change_description': 'Integration test update', - 'component_id': component_id, - 'configuration_id': configuration_id, - 'configuration_row_id': row_id, - **updates, - }, - ) - - # Check the tool's output - updated_row_config = ConfigToolOutput.model_validate(tool_result.structured_content) - assert updated_row_config.component_id == component_id - assert updated_row_config.configuration_id == configuration_id - assert updated_row_config.success is True - assert updated_row_config.timestamp is not None - assert updated_row_config.version is not None - - expected_row_name = updates.get('name') or 'Initial Test Row Configuration' - expected_row_description = updates.get('description') or initial_cmpconf_row.description - expected_row_is_disabled = updates.get('is_disabled') or False - assert updated_row_config.description == expected_row_description - assert frozenset(updated_row_config.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Configuration: {expected_row_name}', - url=f'{storage_api_url}/admin' f'/projects/{project_id}/components/{component_id}/{configuration_id}', - ), - Link( - type='ui-dashboard', - title=f'Component "{component_id}" Configurations Dashboard', - url=f'{storage_api_url}/admin/projects/{project_id}/components/{component_id}', - ), - ] - ) - - # Verify the row configuration was updated - row_config_detail = await keboola_client.storage_client.configuration_detail( - component_id=updated_row_config.component_id, configuration_id=updated_row_config.configuration_id - ) - updated_rows = row_config_detail['rows'] - assert isinstance(updated_rows, list), f'Expecting list, got: {type(updated_rows)}' - # Find the updated row - updated_row = next(filter(lambda x: x.get('id') == row_id, updated_rows), None) - assert updated_row, f'No row for row_id: {row_id}' - - assert isinstance(updated_row, dict), f'Expecting dict, got: {type(updated_row)}' - assert updated_row['name'] == expected_row_name - assert updated_row['description'] == expected_row_description - assert updated_row['isDisabled'] == expected_row_is_disabled - - row_config_data = updated_row['configuration'] - assert isinstance(row_config_data, dict), f'Expecting dict, got: {type(row_config_data)}' - - if (parameter_updates := updates.get('parameter_updates')) is not None: - # Using the assumption that parameter_updates is a list with one element with 'set' operation on root path - assert row_config_data['parameters'] == parameter_updates[0]['value'] - - if (expected_storage := updates.get('storage')) is not None: - # Storage API might return more keys than what we set, so we check subset - for k, v in expected_storage.items(): - assert k in row_config_data['storage'] - assert row_config_data['storage'][k] == v - - current_version = config_detail['version'] - assert isinstance(current_version, int), f'Expecting int, got: {type(current_version)}' - assert current_version == 2 - - # Check that KBC.MCP.updatedBy.version.{version} is set to 'true' - metadata = await keboola_client.storage_client.configuration_metadata_get( - component_id=updated_row_config.component_id, configuration_id=updated_row_config.configuration_id - ) - assert isinstance(metadata, list), f'Expecting list, got: {type(metadata)}' - - meta_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{current_version}' - meta_value = get_metadata_property(metadata, meta_key) - assert meta_value == 'true' - # Check that the original creation metadata is still there - assert get_metadata_property(metadata, MetadataField.CREATED_BY_MCP) == 'true' - - -@pytest.mark.asyncio -async def test_create_sql_transformation(mcp_context: Context, keboola_project: ProjectDef, storage_api_url: str): - """Tests that `create_sql_transformation` creates a SQL transformation with correct configuration.""" - - test_name = 'Test SQL Transformation' - test_description = 'Test SQL transformation created by automated test' - - # Define test SQL code blocks - test_sql_code_blocks = [ - SimplifiedTfBlocks.Block.Code( - name='Main transformation', script='SELECT 1 as test_column; SELECT 2 as another_column;' - ) - ] - - test_created_table_names = ['test_output_table'] - - client = KeboolaClient.from_state(mcp_context.session.state) - - # Create the SQL transformation - created_transformation = await create_sql_transformation( - ctx=mcp_context, - name=test_name, - description=test_description, - sql_code_blocks=test_sql_code_blocks, - created_table_names=test_created_table_names, - ) - sql_dialect = await WorkspaceManager.from_state(mcp_context.session.state).get_sql_dialect() - expected_component_id = get_sql_transformation_id_from_sql_dialect(sql_dialect) - project_id = keboola_project.project_id - - try: - # Verify the response structure - assert isinstance(created_transformation, ConfigToolOutput) - assert created_transformation.success is True - assert created_transformation.timestamp is not None - assert created_transformation.description == test_description - assert created_transformation.component_id == expected_component_id - assert created_transformation.configuration_id is not None - assert created_transformation.version is not None - expected_links = frozenset( - [ - Link( - type='ui-detail', - title=f'Transformation: {test_name}', - url=( - f'{storage_api_url}/admin/projects/{project_id}/transformations-v2/' - f'{expected_component_id}/{created_transformation.configuration_id}' - ), - ), - Link( - type='ui-dashboard', - title='Transformations dashboard', - url=(f'{storage_api_url}/admin/projects/{project_id}/transformations-v2'), - ), - ] - ) - - assert frozenset(created_transformation.links) == expected_links - - # Verify the configuration exists in the backend by fetching it - config_detail = await client.storage_client.configuration_detail( - component_id=created_transformation.component_id, configuration_id=created_transformation.configuration_id - ) - - assert config_detail['name'] == test_name - assert config_detail['description'] == test_description - assert 'configuration' in config_detail - - # Verify the configuration structure - configuration_data = cast(dict, config_detail['configuration']) - assert 'parameters' in configuration_data - assert 'storage' in configuration_data - - # Verify the parameters structure matches expected - bucket_name = clean_bucket_name(test_name) - expected_script = test_sql_code_blocks[0].script - expected_script = await split_sql_statements(expected_script) - expected_parameters = { - 'blocks': [ - { - 'name': 'Blocks', - 'codes': [ - { - 'name': test_sql_code_blocks[0].name, - 'script': expected_script, - } - ], - } - ] - } - assert configuration_data['parameters'] == expected_parameters - - # Verify the storage structure matches expected - expected_storage = { - 'input': {'tables': []}, - 'output': { - 'tables': [ - { - 'source': test_created_table_names[0], - 'destination': f'out.c-{bucket_name}.{test_created_table_names[0]}', - } - ] - }, - } - assert configuration_data['storage'] == expected_storage - - # Verify the metadata - check that KBC.MCP.createdBy is set to 'true' - metadata = await client.storage_client.configuration_metadata_get( - component_id=created_transformation.component_id, configuration_id=created_transformation.configuration_id - ) - - # Convert metadata list to dictionary for easier checking - assert isinstance(metadata, list) - metadata_dict = {item['key']: item['value'] for item in metadata if isinstance(item, dict)} - assert MetadataField.CREATED_BY_MCP in metadata_dict - assert metadata_dict[MetadataField.CREATED_BY_MCP] == 'true' - - finally: - # Clean up: Delete the configuration - await client.storage_client.configuration_delete( - component_id=created_transformation.component_id, - configuration_id=created_transformation.configuration_id, - skip_trash=True, - ) - - -@pytest_asyncio.fixture -async def initial_sqltrfm( - mcp_client: Client, configs: list[ConfigDef], keboola_client: KeboolaClient -) -> AsyncGenerator[ConfigToolOutput, None]: - # Create the initial component configuration test data - tool_result = await mcp_client.call_tool( - name='create_sql_transformation', - arguments={ - 'name': 'Initial Test SQL Transformation', - 'description': 'Initial SQL transformation for update test', - 'sql_code_blocks': [{'name': 'Initial transformation', 'script': 'SELECT 1 as initial_column;'}], - 'created_table_names': ['initial_output_table'], - }, - ) - try: - yield ConfigToolOutput.model_validate(tool_result.structured_content) - finally: - # Clean up: Delete the configuration - await keboola_client.storage_client.configuration_delete( - component_id=tool_result.structured_content['component_id'], - configuration_id=tool_result.structured_content['configuration_id'], - skip_trash=True, - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - 'updates', - [ - { - 'name': 'Updated SQL transformation name', - 'description': 'Updated SQL transformation description', - 'parameter_updates': [ - TfRenameBlock(op='rename_block', block_id='b0', block_name='Updated block'), - TfRenameCode(op='rename_code', block_id='b0', code_id='b0.c0', code_name='Updated code'), - TfSetCode( - op='set_code', - block_id='b0', - code_id='b0.c0', - script=( - 'SELECT 1 as updated_column;\n\nSELECT 2 as additional_column;\n\n' - 'SELECT 3 as third_column;\n\n' - ), - ), - ], - 'storage': { - 'input': {'tables': [{'source': 'in.c-bucket.input_table', 'destination': 'input.csv'}]}, - 'output': { - 'tables': [ - {'source': 'updated_output_table', 'destination': 'out.c-bucket.updated_output_table'}, - {'source': 'second_output_table', 'destination': 'out.c-bucket.second_output_table'}, - ] - }, - }, - }, - {'name': 'Updated SQL transformation name'}, - {'description': 'Updated SQL transformation description'}, - { - 'parameter_updates': [ - TfStrReplace( - op='str_replace', - block_id='b0', - code_id='b0.c0', - search_for='SELECT 1', - replace_with='SELECT 12', - ), - TfAddScript( - op='add_script', - block_id='b0', - code_id='b0.c0', - script='SELECT 2 as additional_column', - position='end', - ), - ] - }, - { - 'storage': { - 'input': {'tables': [{'source': 'in.c-bucket.input_table', 'destination': 'input.csv'}]}, - 'output': { - 'tables': [ - {'source': 'updated_output_table', 'destination': 'out.c-bucket.updated_output_table'}, - {'source': 'second_output_table', 'destination': 'out.c-bucket.second_output_table'}, - ] - }, - } - }, - ], -) -async def test_update_sql_transformation( - updates: dict[str, Any], - initial_sqltrfm: ConfigToolOutput, - mcp_client: Client, - keboola_project: ProjectDef, - keboola_client: KeboolaClient, - storage_api_url: str, -): - """Tests that `update_sql_transformation` updates an existing SQL transformation correctly.""" - project_id = keboola_project.project_id - component_id = initial_sqltrfm.component_id - configuration_id = initial_sqltrfm.configuration_id - param_update_objects = updates.get('parameter_updates') - - if param_update_objects is not None: - # Get the original configuration so we can compare the parameters - orig_config = await keboola_client.storage_client.configuration_detail( - component_id=component_id, configuration_id=configuration_id - ) - orig_parameters_dict = cast(dict, orig_config.get('configuration', {}).get('parameters', {})) - - # Convert the parameter update objects to TfParamUpdate if needed - param_updates: list[TfParamUpdate] = [] - for update_obj in param_update_objects: - if isinstance(update_obj, dict): - update = TypeAdapter(TfParamUpdate).validate_python(update_obj) - else: - update = update_obj - param_updates.append(update) - - # Convert parameter update objects to dict format for tool call if needed - updates_dict = updates.copy() - if param_update_objects is not None: - param_updates_list = [] - for update_obj in param_update_objects: - if isinstance(update_obj, dict): - param_updates_list.append(update_obj) - else: - # Convert Pydantic model to dict - param_updates_list.append(update_obj.model_dump()) - updates_dict['parameter_updates'] = param_updates_list - - tool_result = await mcp_client.call_tool( - name='update_sql_transformation', - arguments={ - 'change_description': 'Integration test update', - 'configuration_id': configuration_id, - **updates_dict, - }, - ) - - # Check the tool's output - updated_trfm = ConfigToolOutput.model_validate(tool_result.structured_content) - - assert updated_trfm.component_id == component_id - assert updated_trfm.configuration_id == configuration_id - assert updated_trfm.success is True - assert updated_trfm.timestamp is not None - assert updated_trfm.version is not None - - expected_name = updates.get('name') or 'Initial Test SQL Transformation' - expected_description = updates.get('description') or initial_sqltrfm.description - assert updated_trfm.description == expected_description - assert frozenset(updated_trfm.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Transformation: {expected_name}', - url=f'{storage_api_url}/admin' - f'/projects/{project_id}/transformations-v2/{component_id}/{configuration_id}', - ), - Link( - type='ui-dashboard', - title='Transformations dashboard', - url=f'{storage_api_url}/admin/projects/{project_id}/transformations-v2', - ), - ] - ) - - # Verify the transformation was updated - trfm_detail = await keboola_client.storage_client.configuration_detail( - component_id=updated_trfm.component_id, configuration_id=updated_trfm.configuration_id - ) - - assert trfm_detail['name'] == expected_name - assert trfm_detail['description'] == expected_description - - trfm_data = trfm_detail.get('configuration') - assert isinstance(trfm_data, dict), f'Expecting dict, got: {type(trfm_data)}' - - actual_parameters = trfm_data.get('parameters') - assert isinstance(actual_parameters, dict), f'Expecting dict, got: {type(actual_parameters)}' - - if param_update_objects is not None: - # Convert original parameters to SimplifiedTfBlocks, apply updates, then convert back - orig_raw_parameters = TransformationConfiguration.Parameters.model_validate(orig_parameters_dict) - orig_simplified_parameters = await orig_raw_parameters.to_simplified_parameters() - - updated_params, _ = update_transformation_parameters( - orig_simplified_parameters, param_updates, sql_dialect='snowflake' - ) - updated_raw_parameters = await updated_params.to_raw_parameters() - - expected_parameters = updated_raw_parameters.model_dump(exclude_none=True) - assert actual_parameters == expected_parameters - - actual_storage = trfm_data.get('storage') - assert isinstance(actual_storage, dict), f'Expecting dict, got: {type(actual_storage)}' - if (expected_storage := updates.get('storage')) is not None: - # Storage API might return more keys than what we set, so we check subset - for k, v in expected_storage.items(): - assert k in trfm_data['storage'] - assert trfm_data['storage'][k] == v - - current_version = trfm_detail['version'] - assert isinstance(current_version, int), f'Expecting int, got: {type(current_version)}' - assert current_version == 2 - - # Check that KBC.MCP.updatedBy.version.{version} is set to 'true' - metadata = await keboola_client.storage_client.configuration_metadata_get( - component_id=updated_trfm.component_id, configuration_id=updated_trfm.configuration_id - ) - assert isinstance(metadata, list), f'Expecting list, got: {type(metadata)}' - - meta_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{current_version}' - meta_value = get_metadata_property(metadata, meta_key) - assert meta_value == 'true' - # Check that the original creation metadata is still there - assert get_metadata_property(metadata, MetadataField.CREATED_BY_MCP) == 'true' - - -@pytest.mark.asyncio -async def test_get_components(mcp_context: Context, configs: list[ConfigDef]): - """Tests that `get_components` returns component details for multiple components.""" - # Get unique component IDs from test configs - component_ids = list({config.component_id for config in configs}) - assert len(component_ids) > 0 - - result = await get_components(component_ids=component_ids, ctx=mcp_context) - - # Verify result structure - assert isinstance(result, GetComponentsOutput) - assert len(result.components) == len(component_ids) - - # Verify each component - returned_ids = {comp.component_id for comp in result.components} - assert returned_ids == set(component_ids) - - for component in result.components: - assert component.component_id in component_ids - assert component.component_name is not None - assert component.component_type is not None - # Verify links are present - assert component.links, 'Component links should not be empty.' - for link in component.links: - assert isinstance(link, Link) - - # Verify output-level links - assert result.links, 'Output links should not be empty.' - - -@pytest.mark.asyncio -async def test_get_config_examples(mcp_context: Context, configs: list[ConfigDef]): - """Tests that `get_config_examples` returns configuration examples in markdown format.""" - test_config = configs[0] - component_id = test_config.component_id - - result = await get_config_examples(component_id=component_id, ctx=mcp_context) - - # Verify the result is a markdown formatted string - assert isinstance(result, str) - assert f'# Configuration Examples for `{component_id}`' in result - assert f'{component_id}`' in result - assert 'parameters' in result - - -@pytest.mark.asyncio -async def test_get_config_examples_with_invalid_component(mcp_context: Context): - """Tests that `get_config_examples` handles non-existent components properly.""" - - result = await get_config_examples(ctx=mcp_context, component_id='completely-non-existent-component-12345') - - assert result == '' diff --git a/integtests/tools/data_apps.test.ts b/integtests/tools/data_apps.test.ts new file mode 100644 index 000000000..d68f2c7a6 --- /dev/null +++ b/integtests/tools/data_apps.test.ts @@ -0,0 +1,416 @@ +import { execFileSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { callToolText, connectMcp } from '../helpers/mcp'; +import { getTestProjectForTest, type TestProject } from '../testproject/fixture'; + +import { createKeboolaClients } from '@/clients/keboola'; +import { Config } from '@/config'; +import { createDataScience, type DataScience } from '@/tools/data_apps/client'; + +// Ported from integtests/tools/test_data_apps.py. +// +// Data App tools are gated to the MAIN/production branch (filtering.ts +// DATA_APP_BRANCH_GATED_TOOLS). The pool leases its default (main) branch, so the tools +// are always available here — confirmed live: get_data_apps + modify_* + deploy_data_app +// are all listed and callable. +// +// These tests create REAL data-science apps. The MCP surface has no delete tool for +// Streamlit apps, so teardown talks to the data-science client directly (the same path +// the Python suite's `initial_data_app` fixture used: +// keboola_client.data_science_client.delete_data_app). Each created app is registered for +// best-effort suspend+delete in an afterEach so a leaked app never lingers in the project. +// +// The MCP client returns TOON text (snake_case field names preserved by the tools), so we +// assert on substrings / regexes rather than re-parsing structured content. + +const SAMPLE_STREAMLIT_IMPORTS = 'import streamlit as st\n\n'; +const SAMPLE_STREAMLIT_ENTRYPOINT = + 'def main():\n' + + " st.title('Integration Test Data App')\n" + + " st.write('Hello from integration test')\n\n" + + 'if __name__ == "__main__":\n' + + ' main()\n'; +// Mirrors the Python `sample_streamlit_app` fixture: imports + {QUERY_DATA_FUNCTION} +// placeholder (where the tool injects the query_data helper) + entrypoint. +const SAMPLE_STREAMLIT_APP = `${SAMPLE_STREAMLIT_IMPORTS}{QUERY_DATA_FUNCTION}\n\n${SAMPLE_STREAMLIT_ENTRYPOINT}`; + +const PYTHON_JS_APP_PY = + 'from http.server import BaseHTTPRequestHandler, HTTPServer\n' + + 'import os\n\n' + + 'class H(BaseHTTPRequestHandler):\n' + + ' def do_GET(self):\n' + + ' self.send_response(200)\n' + + ' self.end_headers()\n' + + " self.wfile.write(b'integration-test-ok')\n\n" + + "if __name__ == '__main__':\n" + + " port = int(os.environ.get('PORT', '8000'))\n" + + " HTTPServer(('0.0.0.0', port), H).serve_forever()\n"; + +const uniqueSuffix = (): string => randomUUID().replace(/-/g, '').slice(0, 8); + +// TOON renders string scalars that look numeric (or otherwise need quoting) wrapped in +// double quotes, e.g. `data_app_id: "74015536"`, while opaque ids stay bare, e.g. +// `configuration_id: 01kw...`. Strip an optional pair of surrounding quotes when extracting. +const unquote = (value: string): string => value.replace(/^"(.*)"$/, '$1'); + +/** Pulls the data_app_id out of a TOON tool response (`data_app_id: `). */ +const extractDataAppId = (toon: string): string | null => { + const m = toon.match(/data_app_id:\s*("[^"]*"|\S+)/); + return m ? unquote(m[1]!) : null; +}; + +/** Pulls the (first) configuration_id out of a TOON tool response. */ +const extractConfigurationId = (toon: string): string | null => { + const m = toon.match(/configuration_id:\s*("[^"]*"|\S+)/); + return m ? unquote(m[1]!) : null; +}; + +/** + * Provisions a read-only workspace (mirrors the Python `workspace_schema` fixture) and + * returns its schema plus a deleter. Streamlit data app tools require a configured + * workspace schema (resolveWorkspace throws otherwise), so the Streamlit tests build a + * Config carrying this schema. + */ +const provisionWorkspace = async ( + project: TestProject, +): Promise<{ schema: string; remove: () => Promise }> => { + const base = `${project.storageApiUrl}/v2/storage`; + const headers = { + 'X-StorageApi-Token': project.storageApiToken, + 'Content-Type': 'application/json', + }; + const verify = (await ( + await fetch(`${base}/tokens/verify`, { headers }) + ).json()) as { owner?: { defaultBackend?: string } }; + const backend = verify.owner?.defaultBackend; + const loginType = + backend === 'snowflake' ? 'snowflake-person-sso' : backend === 'bigquery' ? 'default' : null; + if (!loginType) throw new Error(`Unexpected project default backend: ${backend}`); + + const res = await fetch(`${base}/branch/default/workspaces`, { + method: 'POST', + headers, + body: JSON.stringify({ backend, loginType, readOnlyStorageAccess: true }), + }); + const ws = (await res.json()) as { + id?: number | string; + connection?: { schema?: string }; + }; + if (!res.ok || !ws.id || !ws.connection?.schema) { + throw new Error(`Failed to create workspace: ${res.status} ${JSON.stringify(ws)}`); + } + const schema = ws.connection.schema; + const workspaceId = ws.id; + return { + schema, + remove: async () => { + try { + await fetch(`${base}/branch/default/workspaces/${workspaceId}`, { + method: 'DELETE', + headers, + }); + } catch { + // best-effort + } + }, + }; +}; + +/** Builds a Config identical to the leased one but carrying a workspace schema. */ +const withWorkspaceSchema = (config: Config, schema: string): Config => + new Config({ ...config.toFields(), workspaceSchema: schema }); + +describe('data app tools (integration)', () => { + // Best-effort teardown registry: { ds, dataAppId } pairs to suspend+delete after each test. + let cleanup: { ds: DataScience; dataAppId: string }[] = []; + // Workspaces provisioned for Streamlit tests, deleted after each test. + let workspaceCleanup: (() => Promise)[] = []; + + afterEach(async () => { + for (const { ds, dataAppId } of cleanup.reverse()) { + try { + await ds.suspendDataApp(dataAppId); + } catch { + // ignore: app may already be stopped or gone. + } + try { + await ds.deleteDataApp(dataAppId); + } catch { + // ignore: best-effort. + } + } + cleanup = []; + for (const remove of workspaceCleanup.reverse()) { + await remove(); + } + workspaceCleanup = []; + }); + + const dataScienceFor = (config: Config): DataScience => + createDataScience(createKeboolaClients(config), config); + + // Port of test_get_data_apps_listing: create an app, then get_data_apps must list it. + it('get_data_apps lists a freshly created Streamlit app', async () => { + const project = await getTestProjectForTest({ clean: false }); + const ws = await provisionWorkspace(project); + workspaceCleanup.push(ws.remove); + const config = withWorkspaceSchema(project.config, ws.schema); + const ds = dataScienceFor(config); + const session = await connectMcp(config); + try { + const appName = `Integration Test Data App ${uniqueSuffix()}`; + const created = await callToolText(session.client, 'modify_streamlit_data_app', { + name: appName, + description: 'Data app created by integration test', + source_code: SAMPLE_STREAMLIT_APP, + packages: ['numpy', 'streamlit'], + authentication_type: 'no-auth', + }); + expect(created).toContain('response: created'); + const dataAppId = extractDataAppId(created); + const configurationId = extractConfigurationId(created); + expect(dataAppId).toBeTruthy(); + expect(configurationId).toBeTruthy(); + cleanup.push({ ds, dataAppId: dataAppId! }); + + const listed = await callToolText(session.client, 'get_data_apps', { limit: 500 }); + // Listing returns DataAppSummary entries; our app's configuration_id must appear. + expect(listed).toContain(configurationId!); + } finally { + await session.close(); + } + }); + + // Port of test_data_app_lifecycle (streamlit): create -> detail -> update -> detail. + it('Streamlit data app create/detail/update lifecycle', async () => { + const project = await getTestProjectForTest({ clean: false }); + const ws = await provisionWorkspace(project); + workspaceCleanup.push(ws.remove); + const config = withWorkspaceSchema(project.config, ws.schema); + const ds = dataScienceFor(config); + const session = await connectMcp(config); + try { + const appName = `Integration Test Data App ${uniqueSuffix()}`; + const appDescription = 'Data app created by integration test'; + + // Create. + const created = await callToolText(session.client, 'modify_streamlit_data_app', { + name: appName, + description: appDescription, + source_code: SAMPLE_STREAMLIT_APP, + packages: ['numpy', 'streamlit'], + authentication_type: 'no-auth', + }); + expect(created).toContain('response: created'); + const dataAppId = extractDataAppId(created)!; + const configurationId = extractConfigurationId(created)!; + expect(dataAppId).toBeTruthy(); + expect(configurationId).toBeTruthy(); + cleanup.push({ ds, dataAppId }); + + // Detail by configuration_id reflects the created app, the injected query_data + // function, and the imports/entrypoint we sent. + const detail = await callToolText(session.client, 'get_data_apps', { + configuration_ids: [configurationId], + }); + expect(detail).toContain(configurationId); + expect(detail).toContain(dataAppId); + expect(detail).toContain(appName); + expect(detail).toContain(appDescription); + expect(detail).toContain('import streamlit as st'); + expect(detail).toContain("st.title('Integration Test Data App')"); + + // Update: new name/description, new source + packages. + const updatedName = `${appName} - Updated`; + const updatedDescription = 'Data app updated by integration test'; + const updated = await callToolText(session.client, 'modify_streamlit_data_app', { + name: updatedName, + description: updatedDescription, + source_code: 'import numpy as np\n\n', + packages: ['streamlit'], + authentication_type: 'no-auth', + configuration_id: configurationId, + change_description: 'Update Code', + }); + // Same app/config, response is an update (not "created"). + expect(updated).toContain(dataAppId); + expect(updated).toContain(configurationId); + expect(updated).not.toContain('response: created'); + + // Detail reflects the updated name/description + the new source code; the old + // streamlit imports/entrypoint are gone. + const detail2 = await callToolText(session.client, 'get_data_apps', { + configuration_ids: [configurationId], + }); + expect(detail2).toContain(updatedName); + expect(detail2).toContain(updatedDescription); + expect(detail2).toContain('import numpy as np'); + expect(detail2).not.toContain("st.title('Integration Test Data App')"); + } finally { + await session.close(); + } + }); + + // Port of test_python_js_data_app_prod_and_draft_lifecycle. + // + // create prod (managed repo) -> create draft (external-git pointing at prod's repo) -> + // clone via embedded credential, push branch -> deploy draft mode='dev' -> assert the + // prod detail surfaces the draft under `drafts:` -> merge into main, redeploy prod -> + // delete the draft via delete_python_js_data_app_draft, assert it's gone from prod drafts. + it('python-js prod + external-git draft lifecycle', async () => { + const project = await getTestProjectForTest({ clean: false }); + // modify_python_js_data_app injects the query_data code, which needs the workspace + // dialect — provision a read-only workspace and pass its schema (as the Streamlit tests do). + const ws = await provisionWorkspace(project); + const cfg = withWorkspaceSchema(project.config, ws.schema); + const ds = dataScienceFor(cfg); + const session = await connectMcp(cfg); + const repoDir = mkdtempSync(join(tmpdir(), 'kbc-pyjs-')); + const gitEnv = { ...process.env, GIT_TERMINAL_PROMPT: '0' }; + const git = (...args: string[]): void => { + execFileSync('git', args, { cwd: repoDir, env: gitEnv, stdio: 'pipe' }); + }; + + const unique = uniqueSuffix(); + let prodDataAppId: string | null = null; + let draftDataAppId: string | null = null; + let draftDeletedViaTool = false; + + try { + // Step 1: create prod (managed repo). Response carries repo_url, no git_clone_url/branch. + const prodResp = await callToolText(session.client, 'modify_python_js_data_app', { + name: `Integration prod ${unique}`, + description: 'AI-3286 prod app integration test', + slug: `int-prod-${unique}`, + authentication_type: 'no-auth', + }); + expect(prodResp).toContain('response: created'); + const prodConfigId = extractConfigurationId(prodResp)!; + prodDataAppId = extractDataAppId(prodResp)!; + expect(prodConfigId).toBeTruthy(); + expect(prodDataAppId).toBeTruthy(); + cleanup.push({ ds, dataAppId: prodDataAppId }); + expect(prodResp).toMatch(/repo_url:\s*"?https:\/\//); + + // Step 2: create draft pointing at prod's repo. Branch defaults to 'init'. + const draftResp = await callToolText(session.client, 'modify_python_js_data_app', { + name: `Integration draft ${unique}`, + description: 'AI-3286 draft integration test', + slug: `int-draft-${unique}`, + parent_configuration_id: prodConfigId, + authentication_type: 'no-auth', + }); + expect(draftResp).toContain('response: created'); + const draftConfigId = extractConfigurationId(draftResp)!; + draftDataAppId = extractDataAppId(draftResp)!; + expect(draftConfigId).toBeTruthy(); + expect(draftDataAppId).toBeTruthy(); + cleanup.push({ ds, dataAppId: draftDataAppId }); + // git_clone_url is an authenticated https://kai:@... clone URL; branch is 'init'. + const cloneMatch = draftResp.match(/git_clone_url:\s*("[^"]*"|\S+)/); + expect(cloneMatch).toBeTruthy(); + const gitCloneUrl = unquote(cloneMatch![1]!); + expect(gitCloneUrl).toMatch(/^https:\/\/kai:/); + expect(draftResp).toMatch(/branch:\s*"?init\b/); + + // Step 3: clone via the embedded credential; init main if empty, branch off, push. + execFileSync('git', ['clone', gitCloneUrl, repoDir], { env: gitEnv, stdio: 'pipe' }); + git('config', 'user.email', 'mcp-integration@keboola.com'); + git('config', 'user.name', 'MCP Integration Test'); + let hasMain = true; + try { + execFileSync('git', ['rev-parse', '--verify', 'refs/heads/main'], { + cwd: repoDir, + env: gitEnv, + stdio: 'pipe', + }); + } catch { + hasMain = false; + } + if (!hasMain) { + git('checkout', '-b', 'main'); + writeFileSync(join(repoDir, 'README.md'), `# integration test ${unique}\n`); + git('add', 'README.md'); + git('commit', '-m', 'init main'); + git('push', '-u', 'origin', 'main'); + } + git('checkout', '-b', 'init', 'main'); + writeFileSync(join(repoDir, 'app.py'), PYTHON_JS_APP_PY); + git('add', 'app.py'); + git('commit', '-m', `AI-3286 integration test commit ${unique}`); + git('push', '-u', 'origin', 'init'); + + // Step 4: deploy draft in mode='dev'. Fire-and-return: we only assert it was accepted. + const draftDeploy = await callToolText(session.client, 'deploy_data_app', { + action: 'deploy', + configuration_id: draftConfigId, + mode: 'dev', + }); + expect(draftDeploy).toMatch(/state:/); + + // Prod detail must now list the draft under `drafts`. + const prodDetailBefore = await callToolText(session.client, 'get_data_apps', { + configuration_ids: [prodConfigId], + }); + expect(prodDetailBefore).toContain(draftConfigId); + + // Draft's stored config carries the external-git block + parent linkage. + const draftDetail = await callToolText(session.client, 'get_data_apps', { + configuration_ids: [draftConfigId], + }); + expect(draftDetail).toContain(prodConfigId); // parentConfigurationId + expect(draftDetail).toMatch(/isDraft/); + // Encrypted git password stored as KBC::-prefixed secret. + expect(draftDetail).toContain('KBC::'); + + // Step 5: merge into main and push. + git('checkout', 'main'); + git('merge', '--no-ff', '-m', 'Merge init', 'init'); + git('push', 'origin', 'main'); + + // Step 6: redeploy prod (no mode, no branch). Fire-and-return. + const prodDeploy = await callToolText(session.client, 'deploy_data_app', { + action: 'deploy', + configuration_id: prodConfigId, + }); + expect(prodDeploy).toMatch(/state:/); + + // Step 7: stop the draft (DSAPI delete requires desiredState == currentState), then + // delete it via the MCP tool and verify it's gone from prod's drafts. + try { + await ds.suspendDataApp(draftDataAppId); + } catch { + // best-effort + } + const deleted = await callToolText(session.client, 'delete_python_js_data_app_draft', { + configuration_id: draftConfigId, + }); + expect(deleted).toContain('response: deleted'); + expect(deleted).toContain(draftConfigId); + expect(deleted).toContain(prodConfigId); // parent_configuration_id + draftDeletedViaTool = true; + + const prodDetailAfter = await callToolText(session.client, 'get_data_apps', { + configuration_ids: [prodConfigId], + }); + // The draft config id should no longer appear in the prod's drafts listing. (It may + // still appear as part of the prod's own fields? No — drafts is the only place a draft + // cfg id is surfaced on the prod detail, so absence is meaningful.) + expect(prodDetailAfter).not.toContain(draftConfigId); + } finally { + // If the tool already deleted the draft, drop it from the cleanup registry so we don't + // try to delete it twice (which would log a spurious DSAPI error). + if (draftDeletedViaTool && draftDataAppId) { + cleanup = cleanup.filter((c) => c.dataAppId !== draftDataAppId); + } + rmSync(repoDir, { recursive: true, force: true }); + await session.close(); + await ws.remove(); + } + }); +}); diff --git a/integtests/tools/doc.test.ts b/integtests/tools/doc.test.ts new file mode 100644 index 000000000..6b2afb346 --- /dev/null +++ b/integtests/tools/doc.test.ts @@ -0,0 +1,77 @@ +import { Pool } from 'pg'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +import { embedInputFor, FIXTURE_SOURCES, migrateDocsIndex, seedDocsIndex } from '../../scripts/docsIndex'; +import { callToolText, connectMcp } from '../helpers/mcp'; +import { getTestProjectForTest } from '../testproject/fixture'; + +import { StubEmbedder } from '@/clients/docsSearch'; + +// Ported from integtests/tools/test_doc.py, rebuilt on the pgvector docs-search index +// (RFC: feature_spec/docs-search-pgvector/). Drives docs_query + find_component_id through +// the MCP against a real Postgres provided by docker-compose (see the `integration_tests` +// CI job). The suite seeds the fixture corpus with the deterministic StubEmbedder — the +// same embedder the server uses at query time (DOCS_EMBEDDER_MODEL=stub) — so retrieval is +// reproducible offline. +// +// Requires DATABASE_URL + DOCS_EMBEDDER_MODEL=stub; skips otherwise (parity with +// storage_branches). Locally: +// docker compose up -d pgvector +// DATABASE_URL=postgres://mcp:mcp@localhost:5432/docs DOCS_EMBEDDER_MODEL=stub npm run test:integ +// +// StubEmbedder is deterministic but not semantic, so queries use a fixture's exact embed +// input (title\ncontent) to guarantee a top hit — mirroring the SDK's own integ tier. + +const DATABASE_URL = (process.env.DATABASE_URL ?? '').trim(); +const STUB = process.env.DOCS_EMBEDDER_MODEL === 'stub'; +const describeDocs = DATABASE_URL && STUB ? describe : describe.skip; +if (!(DATABASE_URL && STUB)) { + console.warn('SKIP: docs tools integration — set DATABASE_URL + DOCS_EMBEDDER_MODEL=stub.'); +} + +const OVERVIEW = FIXTURE_SOURCES.find((d) => d.sourceKey === 'connection-docs:overview')!; +const MYSQL = FIXTURE_SOURCES.find((d) => d.sourceKey === 'component:keboola.ex-db-mysql')!; + +describeDocs('docs tools (integration, pgvector)', () => { + let pool: Pool | undefined; + + beforeAll(async () => { + pool = new Pool({ connectionString: DATABASE_URL }); + const embedder = new StubEmbedder(3072); + await migrateDocsIndex(pool, embedder.dim); + await seedDocsIndex(pool, embedder, FIXTURE_SOURCES); + }, 60_000); + + afterAll(async () => { + await pool?.end(); + }); + + it('docs_query returns an answer with text and source URLs', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const text = await callToolText(session.client, 'docs_query', { + query: embedInputFor(OVERVIEW), + }); + expect(text.length).toBeGreaterThan(0); + expect(text).toContain('Keboola Connection'); + expect(text).toContain(OVERVIEW.sourceUrl); + } finally { + await session.close(); + } + }); + + it('find_component_id recommends a component id from the index', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const text = await callToolText(session.client, 'find_component_id', { + query: embedInputFor(MYSQL), + }); + // The component id is recovered from the doc's `component:` source key. + expect(text).toContain('keboola.ex-db-mysql'); + } finally { + await session.close(); + } + }); +}); diff --git a/integtests/tools/flow.test.ts b/integtests/tools/flow.test.ts new file mode 100644 index 000000000..08be31a4a --- /dev/null +++ b/integtests/tools/flow.test.ts @@ -0,0 +1,859 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolRaw, callToolText, connectMcp, type McpSession } from '../helpers/mcp'; +import { seedProject } from '../helpers/seed'; +import { getTestProjectForTest, type TestProject } from '../testproject/fixture'; + +import { CONDITIONAL_FLOW_COMPONENT_ID, ORCHESTRATOR_COMPONENT_ID } from '@/constants'; + +// Ported from integtests/tools/flow/test_tools.py (10 tests) and +// integtests/tools/flow/test_scheduler.py (2 tests). +// +// The Python suite drove the tool *functions* directly (and mocked out the tool-filtering +// middleware so every tool was always callable). Here we go through the real MCP server over an +// in-memory transport, so two server-side gates apply that the Python tests bypassed: +// +// 1. Conditional-vs-legacy mutual exclusivity (feature `hide-conditional-flows`): +// - conditional-enabled project -> ONLY `create_conditional_flow` is callable; +// `create_flow` is blocked ("...use create_conditional_flow tool instead"). +// - legacy-only project -> ONLY `create_flow` is callable; +// `create_conditional_flow` is blocked. +// So a given leased project supports exactly one create variant. Each create/update test +// detects the project's variant via get_project_info (`conditional_flows`) and returns +// early with a logged SKIP when the leased project is the wrong variant. (Vitest has no +// late `it.skip`, so the early-return-with-warning is the equivalent.) +// +// 2. update_flow vs modify_flow depends on the token role: admin/OAuth tokens may call +// ONLY `modify_flow`; other tokens may call ONLY `update_flow`. We pick the right tool +// from get_project_info (`user_role`). +// +// Positive assertions are on the TOON text the client returns (substrings/regex); negative +// paths assert on the raw CallToolResult (isError + message). + +// ============================================================================= +// HELPERS +// ============================================================================= + +/** Pulls the first `configuration_id: ` out of a tool's TOON text. */ +const extractConfigId = (text: string): string => { + const m = text.match(/configuration_id:\s*(\S+)/); + if (!m) throw new Error(`No configuration_id found in tool output:\n${text}`); + return m[1]!; +}; + +const projectInfo = (session: McpSession): Promise => + callToolText(session.client, 'get_project_info'); + +/** conditional_flows == true (no `hide-conditional-flows` feature). */ +const isConditionalProject = (info: string): boolean => /conditional_flows:\s*true/i.test(info); + +/** The update tool callable for this token: admin/OAuth -> modify_flow, else update_flow. */ +const updateToolName = (info: string): 'modify_flow' | 'update_flow' => + /user_role:\s*(admin|share)/i.test(info) ? 'modify_flow' : 'update_flow'; + +const isAdmin = (info: string): boolean => /user_role:\s*admin/i.test(info); + +const skip = (reason: string): void => { + // eslint-disable-next-line no-console + console.warn(`SKIP: ${reason}`); +}; + +/** Seeds the standard fixtures and returns the first real config (component + config id). */ +const seedFirstConfig = async ( + project: TestProject, +): Promise<{ componentId: string; configurationId: string }> => { + const seeded = await seedProject(project); + const c = seeded.configs[0]!; + return { componentId: c.componentId, configurationId: c.configurationId }; +}; + +/** Creates the standard "Initial Test Flow" legacy flow (port of conftest.initial_lf). */ +const createInitialLegacyFlow = async ( + session: McpSession, + componentId: string, + configurationId: string, +): Promise => { + const text = await callToolText(session.client, 'create_flow', { + name: 'Initial Test Flow', + description: 'Initial test flow created by automated test', + phases: [{ name: 'Phase1', dependsOn: [], description: 'First phase' }], + tasks: [ + { + id: 20001, + name: 'Task1', + phase: 1, + continueOnFailure: false, + enabled: false, + task: { componentId, configId: configurationId, mode: 'run' }, + }, + ], + }); + return extractConfigId(text); +}; + +/** Creates the standard "Initial Test Flow" conditional flow (port of conftest.initial_cf). */ +const createInitialConditionalFlow = async ( + session: McpSession, + componentId: string, + configurationId: string, +): Promise => { + const text = await callToolText(session.client, 'create_conditional_flow', { + name: 'Initial Test Flow', + description: 'Initial test flow created by automated test', + phases: [ + { + id: 'phase1', + name: 'Phase1', + description: 'First phase', + next: [{ id: 'phase1_end', name: 'End Flow', goto: null }], + }, + ], + tasks: [ + { + id: 'task1', + name: 'Task1', + phase: 'phase1', + task: { type: 'job', componentId, configId: configurationId, mode: 'run' }, + }, + ], + }); + return extractConfigId(text); +}; + +// Flow tests are backend-agnostic; pin to snowflake (the bigquery pool entries currently fail +// the harness's synchronous-bucket-drop reset, unrelated to flows). +const leaseProject = (clean = true) => getTestProjectForTest({ backend: 'snowflake', clean }); + +describe('flow tools (integration)', () => { + // =========================================================================== + // test_create_and_retrieve_flow (legacy) — runs only on a legacy-only project + // =========================================================================== + it('create_flow creates a legacy flow and get_flows retrieves it', async () => { + const project = await leaseProject(); + const { componentId, configurationId } = await seedFirstConfig(project); + const session = await connectMcp(project.config); + try { + if (isConditionalProject(await projectInfo(session))) { + return skip('project is conditional-enabled; create_flow (legacy) is not available.'); + } + const flowName = 'Integration Test Flow'; + const created = await callToolText(session.client, 'create_flow', { + name: flowName, + description: 'Flow created by integration test.', + phases: [ + { name: 'Extract', dependsOn: [], description: 'Extract data' }, + { name: 'Transform', dependsOn: [1], description: 'Transform data' }, + ], + tasks: [ + { name: 'Extract Task', phase: 1, task: { componentId, configId: configurationId } }, + { name: 'Transform Task', phase: 2, task: { componentId, configId: configurationId } }, + ], + }); + expect(created).toContain(ORCHESTRATOR_COMPONENT_ID); + expect(created).toContain('Flow created by integration test.'); + expect(created).toMatch(/success:\s*true/i); + expect(created).toMatch(/version:/); + expect(created).toContain('https://help.keboola.com/flows/'); + const flowId = extractConfigId(created); + + const list = await callToolText(session.client, 'get_flows'); + expect(list).toContain(flowName); + expect(list).toContain(flowId); + + const detail = await callToolText(session.client, 'get_flows', { flow_ids: [flowId] }); + expect(detail).toContain(ORCHESTRATOR_COMPONENT_ID); + expect(detail).toContain('Extract'); + expect(detail).toContain('Transform'); + expect(detail).toContain(componentId); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_create_and_retrieve_conditional_flow — runs only on a conditional project + // =========================================================================== + it('create_conditional_flow creates a conditional flow and get_flows retrieves it', async () => { + const project = await leaseProject(); + const { componentId, configurationId } = await seedFirstConfig(project); + const session = await connectMcp(project.config); + try { + if (!isConditionalProject(await projectInfo(session))) { + return skip('project is legacy-only; create_conditional_flow is not available.'); + } + const flowName = 'Integration Test Conditional Flow'; + const created = await callToolText(session.client, 'create_conditional_flow', { + name: flowName, + description: 'Conditional flow created by integration test.', + phases: [ + { + id: 'extract_phase', + name: 'Extract', + description: 'Extract data', + next: [{ id: 'extract_to_transform', name: 'Extract to Transform', goto: 'transform_phase' }], + }, + { + id: 'transform_phase', + name: 'Transform', + description: 'Transform data', + next: [{ id: 'transform_end', name: 'End Flow', goto: null }], + }, + ], + tasks: [ + { + id: 'extract_task', + name: 'Extract Task', + phase: 'extract_phase', + task: { type: 'job', componentId, configId: configurationId, mode: 'run' }, + }, + { + id: 'transform_task', + name: 'Transform Task', + phase: 'transform_phase', + task: { type: 'job', componentId, configId: configurationId, mode: 'run' }, + }, + ], + }); + expect(created).toContain(CONDITIONAL_FLOW_COMPONENT_ID); + expect(created).toContain('Conditional flow created by integration test.'); + expect(created).toMatch(/success:\s*true/i); + expect(created).toMatch(/version:/); + const flowId = extractConfigId(created); + + const list = await callToolText(session.client, 'get_flows'); + expect(list).toContain(flowName); + expect(list).toContain(flowId); + + const detail = await callToolText(session.client, 'get_flows', { flow_ids: [flowId] }); + expect(detail).toContain(CONDITIONAL_FLOW_COMPONENT_ID); + expect(detail).toContain('Extract'); + expect(detail).toContain('Transform'); + expect(detail).toContain(componentId); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_update_flow — legacy parametrized cases (legacy-only projects) + // =========================================================================== + const legacyUpdateCases: { label: string; updates: Record }[] = [ + { + label: 'phases + tasks + name + description', + updates: { + phases: [ + { id: 1, name: 'Phase1', dependsOn: [], description: 'First phase updated' }, + { id: 2, name: 'Phase2', dependsOn: [], description: 'Second phase added' }, + ], + tasks: [ + { + id: 20001, + name: 'Task1 - Updated', + phase: 1, + continueOnFailure: false, + enabled: false, + task: { componentId: 'ex-generic-v2', configId: 'test_config_001', mode: 'run' }, + }, + { + id: 20002, + name: 'Task2 - Added', + phase: 2, + continueOnFailure: false, + enabled: false, + task: { componentId: 'ex-generic-v2', configId: 'test_config_002', mode: 'run' }, + }, + ], + name: 'Updated Test Flow', + description: 'The test flow updated by an automated test.', + }, + }, + { + label: 'phases only', + updates: { + phases: [ + { id: 1, name: 'Phase1', dependsOn: [], description: 'First phase updated' }, + { id: 2, name: 'Phase2', dependsOn: [], description: 'Second phase added' }, + ], + }, + }, + { + label: 'tasks only', + updates: { + tasks: [ + { + id: 20001, + name: 'Task1 - Updated', + phase: 1, + continueOnFailure: false, + enabled: false, + task: { componentId: 'ex-generic-v2', configId: 'test_config_001', mode: 'run' }, + }, + { + id: 20002, + name: 'Task2 - Added', + phase: 1, + continueOnFailure: false, + enabled: false, + task: { componentId: 'ex-generic-v2', configId: 'test_config_002', mode: 'run' }, + }, + ], + }, + }, + { label: 'name only', updates: { name: 'Updated just name' } }, + { label: 'description only', updates: { description: 'Updated just description' } }, + { label: 'is_disabled true', updates: { is_disabled: true } }, + ]; + + it.each(legacyUpdateCases)('update legacy flow ($label)', async ({ updates }) => { + const project = await leaseProject(); + const { componentId, configurationId } = await seedFirstConfig(project); + const session = await connectMcp(project.config); + try { + const info = await projectInfo(session); + if (isConditionalProject(info)) { + return skip('project is conditional-enabled; legacy create_flow is not available.'); + } + const tool = updateToolName(info); + const flowId = await createInitialLegacyFlow(session, componentId, configurationId); + + const result = await callToolText(session.client, tool, { + configuration_id: flowId, + flow_type: ORCHESTRATOR_COMPONENT_ID, + change_description: 'Integration test update', + ...updates, + }); + expect(result).toContain(flowId); + expect(result).toContain(ORCHESTRATOR_COMPONENT_ID); + expect(result).toMatch(/success:\s*true/i); + expect(result).toMatch(/timestamp:/); + expect(result).toMatch(/version:/); + + const expectedName = (updates.name as string) ?? 'Initial Test Flow'; + const expectedDescription = + (updates.description as string) ?? 'Initial test flow created by automated test'; + + const detail = await callToolText(session.client, 'get_flows', { flow_ids: [flowId] }); + expect(detail).toContain(expectedName); + expect(detail).toContain(expectedDescription); + // The update bumps the configuration to version 2. (The Python test also asserted the + // KBC.MCP.updatedBy/createdBy metadata, but that is read via the Storage client, not + // surfaced by the get_flows MCP tool, so it is not assertable through this surface.) + expect(detail).toMatch(/version:\s*2/); + if (updates.is_disabled === true) expect(detail).toMatch(/is_disabled:\s*true/i); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_update_flow — conditional parametrized cases (conditional projects) + // =========================================================================== + const conditionalUpdateCases: { label: string; updates: Record }[] = [ + { + label: 'phases + tasks', + updates: { + phases: [ + { + id: 'phase1', + name: 'Phase1', + description: 'First phase updated', + next: [{ id: 'phase1_phase2', name: 'End Flow', goto: 'phase2' }], + }, + { + id: 'phase2', + name: 'Phase2', + description: 'Second phase added', + next: [{ id: 'phase2_end', name: 'End Flow', goto: null }], + }, + ], + tasks: [ + { + id: 'task1', + name: 'Task1 - Updated', + phase: 'phase1', + task: { type: 'job', componentId: 'ex-generic-v2', configId: 'test_config_001', mode: 'run' }, + }, + { + id: 'task2', + name: 'Task2 - Added', + phase: 'phase2', + task: { type: 'job', componentId: 'ex-generic-v2', configId: 'test_config_002', mode: 'run' }, + }, + ], + }, + }, + { + label: 'phases only', + updates: { + phases: [ + { + id: 'phase1', + name: 'Phase1', + description: 'First phase updated', + next: [{ id: 'phase1_phase2', name: 'End Flow', goto: 'phase2' }], + }, + { + id: 'phase2', + name: 'Phase2', + description: 'Second phase added', + next: [{ id: 'phase2_end', name: 'End Flow', goto: null }], + }, + ], + }, + }, + { + label: 'tasks only', + updates: { + tasks: [ + { + id: 'task1', + name: 'Task1 - Updated', + phase: 'phase1', + task: { type: 'job', componentId: 'ex-generic-v2', configId: 'test_config_001', mode: 'run' }, + }, + { + id: 'task2', + name: 'Task2 - Added', + phase: 'phase1', + task: { type: 'job', componentId: 'ex-generic-v2', configId: 'test_config_002', mode: 'run' }, + }, + ], + }, + }, + { label: 'name only', updates: { name: 'Updated just name' } }, + { label: 'description only', updates: { description: 'Updated just description' } }, + { label: 'is_disabled true', updates: { is_disabled: true } }, + ]; + + it.each(conditionalUpdateCases)('update conditional flow ($label)', async ({ updates }) => { + const project = await leaseProject(); + const { componentId, configurationId } = await seedFirstConfig(project); + const session = await connectMcp(project.config); + try { + const info = await projectInfo(session); + if (!isConditionalProject(info)) { + return skip('project is legacy-only; create_conditional_flow is not available.'); + } + const tool = updateToolName(info); + const flowId = await createInitialConditionalFlow(session, componentId, configurationId); + + const result = await callToolText(session.client, tool, { + configuration_id: flowId, + flow_type: CONDITIONAL_FLOW_COMPONENT_ID, + change_description: 'Integration test update', + ...updates, + }); + expect(result).toContain(flowId); + expect(result).toContain(CONDITIONAL_FLOW_COMPONENT_ID); + expect(result).toMatch(/success:\s*true/i); + + const expectedName = (updates.name as string) ?? 'Initial Test Flow'; + const expectedDescription = + (updates.description as string) ?? 'Initial test flow created by automated test'; + + const detail = await callToolText(session.client, 'get_flows', { flow_ids: [flowId] }); + expect(detail).toContain(expectedName); + expect(detail).toContain(expectedDescription); + // version bumps to 2 on update; MCP-tracking metadata is not surfaced by get_flows. + expect(detail).toMatch(/version:\s*2/); + if (updates.is_disabled === true) expect(detail).toMatch(/is_disabled:\s*true/i); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_get_flows_empty + // =========================================================================== + it('get_flows returns an empty list when no flows exist', async () => { + const project = await leaseProject(); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_flows'); + expect(text).not.toMatch(/configuration_id:/); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_get_flows_list — creates whichever flow variant the project supports + // =========================================================================== + it('get_flows lists created flows with dashboard links', async () => { + const project = await leaseProject(); + const { componentId, configurationId } = await seedFirstConfig(project); + const session = await connectMcp(project.config); + try { + const info = await projectInfo(session); + const id = isConditionalProject(info) + ? await createInitialConditionalFlow(session, componentId, configurationId) + : await createInitialLegacyFlow(session, componentId, configurationId); + + const list = await callToolText(session.client, 'get_flows'); + expect(list).toContain(id); + // Dashboard links for both flow surfaces are always present in the list output. + expect(list).toContain('/flows'); + expect(list).toContain('/flows-v2'); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_get_flow_schema (read-only; no project reset needed) + // =========================================================================== + it('get_flow_schema returns the legacy (and conditional) JSON schema', async () => { + const project = await leaseProject(false); + const session = await connectMcp(project.config); + try { + const legacy = await callToolText(session.client, 'get_flow_schema', { + flow_type: ORCHESTRATOR_COMPONENT_ID, + }); + expect(legacy.startsWith('```json\n')).toBe(true); + expect(legacy.endsWith('\n```')).toBe(true); + expect(legacy).toContain('dependsOn'); + const legacyParsed = JSON.parse(legacy.slice(8, -4)); + expect(legacyParsed).toHaveProperty('$schema'); + expect(legacyParsed.properties).toHaveProperty('phases'); + expect(legacyParsed.properties).toHaveProperty('tasks'); + + if (isConditionalProject(await projectInfo(session))) { + const conditional = await callToolText(session.client, 'get_flow_schema', { + flow_type: CONDITIONAL_FLOW_COMPONENT_ID, + }); + expect(conditional.startsWith('```json\n')).toBe(true); + expect(conditional).not.toBe(legacy); + const conditionalParsed = JSON.parse(conditional.slice(8, -4)); + expect(conditionalParsed.properties.phases.items.properties).toHaveProperty('next'); + expect(conditionalParsed.properties.tasks.items.properties.task).toHaveProperty('oneOf'); + } else { + const result = await callToolRaw(session.client, 'get_flow_schema', { + flow_type: CONDITIONAL_FLOW_COMPONENT_ID, + }); + expect(result.isError).toBeTruthy(); + expect((result.content as { text: string }[])[0]!.text).toMatch( + /conditional flows are not supported/i, + ); + } + } finally { + await session.close(); + } + }); + + // =========================================================================== + // get_flow_examples (read-only) + // =========================================================================== + it('get_flow_examples returns example configurations for the legacy flow type', async () => { + const project = await leaseProject(false); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_flow_examples', { + flow_type: ORCHESTRATOR_COMPONENT_ID, + }); + expect(text).toContain(`Flow Configuration Examples for \`${ORCHESTRATOR_COMPONENT_ID}\``); + expect(text).toContain('```json'); + expect(text).toContain('phases'); + expect(text).toContain('tasks'); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_create_legacy_flow_invalid_structure + // (legacy validation runs only on legacy-only projects, where create_flow is callable) + // =========================================================================== + it('create_flow rejects a legacy flow that depends on a non-existent phase', async () => { + const project = await leaseProject(false); + const session = await connectMcp(project.config); + try { + if (isConditionalProject(await projectInfo(session))) { + return skip('project is conditional-enabled; create_flow (legacy) is not available.'); + } + const result = await callToolRaw(session.client, 'create_flow', { + name: 'Invalid Legacy Flow', + description: 'Should fail', + phases: [{ name: 'Phase1', dependsOn: [99], description: 'Depends on non-existent phase' }], + tasks: [{ name: 'Task1', phase: 1, task: { componentId: 'ex-generic-v2', configId: 'x' } }], + }); + expect(result.isError).toBeTruthy(); + expect((result.content as { text: string }[])[0]!.text).toMatch(/non-existent phase/i); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_create_conditional_flow_invalid_structure (type/schema validation) + // =========================================================================== + it('create_conditional_flow rejects structurally invalid phases/tasks', async () => { + const project = await leaseProject(false); + const session = await connectMcp(project.config); + try { + if (!isConditionalProject(await projectInfo(session))) { + return skip('project is legacy-only; create_conditional_flow is not available.'); + } + const result = await callToolRaw(session.client, 'create_conditional_flow', { + name: 'Invalid Conditional Flow', + description: 'Should fail', + phases: [ + { + id: 123, // invalid: should be a string + name: '', // invalid: empty + next: [{ id: 'transition-1', goto: 'phase-2' }], + }, + ], + tasks: [ + { + id: 'task-1', + name: 'Task1', + phase: 'phase-1', + enabled: true, + task: { type: 'invalid_type', componentId: 'ex-generic-v2', configId: 'x', mode: 'invalid_mode' }, + }, + ], + }); + expect(result.isError).toBeTruthy(); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_create_conditional_flow_semantically_invalid_structure + // =========================================================================== + const semanticInvalidCases: { + label: string; + phases: Record[]; + tasks: Record[]; + expected: RegExp; + }[] = [ + { + label: 'multiple entry phases', + phases: [ + { id: 'phase-1', name: 'Phase1', next: [{ id: 'transition-1', goto: null }] }, + { id: 'phase-2', name: 'Phase2', next: [{ id: 'transition-2', goto: null }] }, + ], + tasks: [ + { + id: 'task-1', + name: 'Task1', + phase: 'phase-1', + task: { type: 'job', componentId: 'ex-generic-v2', configId: 'test_config_002', mode: 'run' }, + }, + ], + expected: /multiple entry phases/i, + }, + { + label: 'no ending phases', + phases: [ + { id: 'phase-1', name: 'Phase1', next: [{ id: 'transition-1', goto: 'phase-2' }] }, + { id: 'phase-2', name: 'Phase2', next: [{ id: 'transition-2', goto: 'phase-1' }] }, + ], + tasks: [ + { + id: 'task-1', + name: 'Task1', + phase: 'phase-1', + task: { type: 'job', componentId: 'ex-generic-v2', configId: 'test_config_002', mode: 'run' }, + }, + { + id: 'task-2', + name: 'Task2', + phase: 'phase-2', + task: { type: 'job', componentId: 'ex-generic-v2', configId: 'test_config_002', mode: 'run' }, + }, + ], + expected: /no ending phases/i, + }, + ]; + + it.each(semanticInvalidCases)( + 'create_conditional_flow rejects a semantically invalid flow ($label)', + async ({ phases, tasks, expected }) => { + const project = await leaseProject(false); + const session = await connectMcp(project.config); + try { + if (!isConditionalProject(await projectInfo(session))) { + return skip('project is legacy-only; create_conditional_flow is not available.'); + } + const result = await callToolRaw(session.client, 'create_conditional_flow', { + name: 'Invalid Conditional Flow', + description: 'Should fail', + phases, + tasks, + }); + expect(result.isError).toBeTruthy(); + expect((result.content as { text: string }[])[0]!.text).toMatch(expected); + } finally { + await session.close(); + } + }, + ); + + // =========================================================================== + // test_flow_lifecycle_integration — create the supported variant, retrieve, list + // =========================================================================== + it('full flow lifecycle: create, retrieve individually, list', async () => { + const project = await leaseProject(); + const { componentId, configurationId } = await seedFirstConfig(project); + const session = await connectMcp(project.config); + try { + const info = await projectInfo(session); + const created: { type: string; id: string }[] = []; + + if (isConditionalProject(info)) { + const text = await callToolText(session.client, 'create_conditional_flow', { + name: 'Integration Test Conditional Flow', + description: 'Conditional flow created by integration test', + phases: [ + { + id: 'phase-1', + name: 'Extract', + description: 'Extract data from source', + next: [{ id: 'transition-1', goto: 'phase-2' }], + }, + { id: 'phase-2', name: 'Load', description: 'Load data to destination', next: [] }, + ], + tasks: [ + { + id: 'task-1', + name: 'Extract from API', + phase: 'phase-1', + enabled: true, + task: { type: 'job', componentId, configId: configurationId, mode: 'run' }, + }, + { + id: 'task-2', + name: 'Load to Warehouse', + phase: 'phase-2', + enabled: true, + task: { type: 'job', componentId, configId: configurationId, mode: 'run' }, + }, + ], + }); + expect(text).toMatch(/success:\s*true/i); + created.push({ type: CONDITIONAL_FLOW_COMPONENT_ID, id: extractConfigId(text) }); + } else { + const text = await callToolText(session.client, 'create_flow', { + name: 'Integration Test Orchestrator Flow', + description: 'Orchestrator flow created by integration test', + phases: [ + { id: 1, name: 'Extract', description: 'Extract data from source', dependsOn: [] }, + { id: 2, name: 'Load', description: 'Load data to destination', dependsOn: [1] }, + ], + tasks: [ + { + id: 20001, + name: 'Extract from API', + phase: 1, + enabled: true, + continueOnFailure: false, + task: { componentId, configId: configurationId, mode: 'run' }, + }, + { + id: 20002, + name: 'Load to Warehouse', + phase: 2, + enabled: true, + continueOnFailure: false, + task: { componentId, configId: configurationId, mode: 'run' }, + }, + ], + }); + expect(text).toMatch(/success:\s*true/i); + created.push({ type: ORCHESTRATOR_COMPONENT_ID, id: extractConfigId(text) }); + } + + // Retrieve each individually. + for (const { type, id } of created) { + const detail = await callToolText(session.client, 'get_flows', { flow_ids: [id] }); + expect(detail).toContain(id); + expect(detail).toContain(type); + expect(detail).toContain('Extract'); + expect(detail).toContain('Load'); + } + + // List all and verify presence. + const list = await callToolText(session.client, 'get_flows'); + for (const { id } of created) expect(list).toContain(id); + } finally { + await session.close(); + } + }); + + // =========================================================================== + // test_scheduler_lifecycle_tooling — modify_flow add/update/remove schedules + // (admin token only; admins must use modify_flow) + // =========================================================================== + it('modify_flow manages a flow schedule via tooling (add, update, remove)', async () => { + const project = await leaseProject(); + const { componentId, configurationId } = await seedFirstConfig(project); + const session = await connectMcp(project.config); + try { + const info = await projectInfo(session); + if (!isAdmin(info)) return skip('scheduler tooling requires an admin token (modify_flow).'); + + // Create whatever flow variant the project supports, schedule that one. + const conditional = isConditionalProject(info); + const flowType = conditional ? CONDITIONAL_FLOW_COMPONENT_ID : ORCHESTRATOR_COMPONENT_ID; + const flowId = conditional + ? await createInitialConditionalFlow(session, componentId, configurationId) + : await createInitialLegacyFlow(session, componentId, configurationId); + + // Add a schedule. + const added = await callToolText(session.client, 'modify_flow', { + configuration_id: flowId, + flow_type: flowType, + change_description: 'Add scheduler via tooling', + schedules: [{ action: 'add', cron_tab: '0 8 * * *', timezone: 'UTC', state: 'enabled' }], + }); + expect(added).toMatch(/success:\s*true/i); + + let detail = await callToolText(session.client, 'get_flows', { flow_ids: [flowId] }); + expect(detail).toMatch(/n_schedules:\s*1/); + expect(detail).toContain('0 8 * * *'); + expect(detail).toContain('UTC'); + expect(detail).toMatch(/state:\s*enabled/); + const scheduleId = detail.match(/scheduleId:\s*(\S+)/)?.[1]; + expect(scheduleId).toBeTruthy(); + + // Update the schedule. + const updated = await callToolText(session.client, 'modify_flow', { + configuration_id: flowId, + flow_type: flowType, + change_description: 'Update scheduler via tooling', + schedules: [ + { + action: 'update', + schedule_id: scheduleId, + cron_tab: '0 12 * * *', + timezone: 'America/New_York', + state: 'disabled', + }, + ], + }); + expect(updated).toMatch(/success:\s*true/i); + + detail = await callToolText(session.client, 'get_flows', { flow_ids: [flowId] }); + expect(detail).toMatch(/n_schedules:\s*1/); + expect(detail).toContain('0 12 * * *'); + expect(detail).toContain('America/New_York'); + expect(detail).toMatch(/state:\s*disabled/); + + // Remove the schedule. + const removed = await callToolText(session.client, 'modify_flow', { + configuration_id: flowId, + flow_type: flowType, + change_description: 'Remove scheduler via tooling', + schedules: [{ action: 'remove', schedule_id: scheduleId }], + }); + expect(removed).toMatch(/success:\s*true/i); + + detail = await callToolText(session.client, 'get_flows', { flow_ids: [flowId] }); + expect(detail).toMatch(/n_schedules:\s*0/); + } finally { + await session.close(); + } + }); +}); diff --git a/integtests/tools/flow/__init__.py b/integtests/tools/flow/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/integtests/tools/flow/conftest.py b/integtests/tools/flow/conftest.py deleted file mode 100644 index af9c2aabc..000000000 --- a/integtests/tools/flow/conftest.py +++ /dev/null @@ -1,126 +0,0 @@ -import logging -from typing import AsyncGenerator -from unittest.mock import AsyncMock - -import pytest -import pytest_asyncio -from fastmcp import Client, FastMCP -from fastmcp.server.middleware import CallNext, MiddlewareContext -from mcp import types as mt - -from integtests.conftest import ConfigDef -from keboola_mcp_server.clients.client import ( - CONDITIONAL_FLOW_COMPONENT_ID, - ORCHESTRATOR_COMPONENT_ID, - KeboolaClient, -) -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.server import create_server -from keboola_mcp_server.tools.flow.tools import FlowToolOutput - -LOG = logging.getLogger(__name__) - - -@pytest.fixture -def mcp_server(storage_api_url: str, storage_api_token: str, workspace_schema: str, mocker) -> FastMCP: - # allow all tool calls regardless the testing project features - async def on_call_tool( - context: MiddlewareContext[mt.CallToolRequestParams], - call_next: CallNext[mt.CallToolRequestParams, mt.CallToolResult], - ) -> mt.CallToolResult: - return await call_next(context) - - mocker.patch( - 'keboola_mcp_server.server.ToolsFilteringMiddleware.on_call_tool', new=AsyncMock(side_effect=on_call_tool) - ) - - config = Config(storage_api_url=storage_api_url, storage_token=storage_api_token, workspace_schema=workspace_schema) - return create_server(config, runtime_info=ServerRuntimeInfo(transport='stdio')) - - -@pytest_asyncio.fixture -async def mcp_client(mcp_server: FastMCP) -> AsyncGenerator[Client, None]: - async with Client(mcp_server) as client: - yield client - - -@pytest_asyncio.fixture -async def initial_lf( - mcp_client: Client, configs: list[ConfigDef], keboola_client: KeboolaClient -) -> AsyncGenerator[FlowToolOutput, None]: - # Create the initial component configuration test data - tool_result = await mcp_client.call_tool( - name='create_flow', - arguments={ - 'name': 'Initial Test Flow', - 'description': 'Initial test flow created by automated test', - 'phases': [{'name': 'Phase1', 'dependsOn': [], 'description': 'First phase'}], - 'tasks': [ - { - 'id': 20001, - 'name': 'Task1', - 'phase': 1, - 'continueOnFailure': False, - 'enabled': False, - 'task': { - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - 'mode': 'run', - }, - } - ], - }, - ) - try: - yield FlowToolOutput.model_validate(tool_result.structured_content) - finally: - # Clean up: Delete the configuration - await keboola_client.storage_client.configuration_delete( - component_id=ORCHESTRATOR_COMPONENT_ID, - configuration_id=tool_result.structured_content['configuration_id'], - skip_trash=True, - ) - - -@pytest_asyncio.fixture -async def initial_cf( - mcp_client: Client, configs: list[ConfigDef], keboola_client: KeboolaClient -) -> AsyncGenerator[FlowToolOutput, None]: - # Create the initial component configuration test data - tool_result = await mcp_client.call_tool( - name='create_conditional_flow', - arguments={ - 'name': 'Initial Test Flow', - 'description': 'Initial test flow created by automated test', - 'phases': [ - { - 'id': 'phase1', - 'name': 'Phase1', - 'description': 'First phase', - 'next': [{'id': 'phase1_end', 'name': 'End Flow', 'goto': None}], - }, - ], - 'tasks': [ - { - 'id': 'task1', - 'name': 'Task1', - 'phase': 'phase1', - 'task': { - 'type': 'job', - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - 'mode': 'run', - }, - }, - ], - }, - ) - try: - yield FlowToolOutput.model_validate(tool_result.structured_content) - finally: - # Clean up: Delete the configuration - await keboola_client.storage_client.configuration_delete( - component_id=CONDITIONAL_FLOW_COMPONENT_ID, - configuration_id=tool_result.structured_content['configuration_id'], - skip_trash=True, - ) diff --git a/integtests/tools/flow/test_scheduler.py b/integtests/tools/flow/test_scheduler.py deleted file mode 100644 index 6c22f33c7..000000000 --- a/integtests/tools/flow/test_scheduler.py +++ /dev/null @@ -1,277 +0,0 @@ -import logging -from typing import Any, cast - -import pytest -from fastmcp import Client - -from integtests.conftest import ConfigDef -from keboola_mcp_server.clients.client import ORCHESTRATOR_COMPONENT_ID, KeboolaClient -from keboola_mcp_server.tools.flow.model import GetFlowsDetailOutput -from keboola_mcp_server.tools.flow.scheduler import ( - SCHEDULER_COMPONENT_ID, - create_schedule, - list_schedules_for_config, - remove_schedule, - update_schedule, -) -from keboola_mcp_server.tools.flow.scheduler_model import ScheduleDetail -from keboola_mcp_server.tools.flow.tools import FlowToolOutput - -LOG = logging.getLogger(__name__) - - -@pytest.mark.asyncio -async def test_scheduler_lifecycle(mcp_context, configs, keboola_client) -> None: - """ - Test complete scheduler lifecycle: create, retrieve, update, and delete. - - :param mcp_context: The test context fixture. - :param configs: List of real configuration definitions. - :param keboola_client: KeboolaClient instance. - """ - token_info = await keboola_client.storage_client.verify_token() - admin_data = token_info.get('admin', {}) - token_role = admin_data.get('role') if isinstance(admin_data, dict) else None - if token_role != 'admin': - pytest.skip('Scheduler tooling requires an admin token, skipping test.') - - assert configs - assert configs[0].configuration_id is not None - - # Use the first config as our target for scheduling - target_component_id = configs[0].component_id - target_configuration_id = configs[0].configuration_id - - # Initial schedule parameters - initial_cron_tab = '0 8 * * *' # Daily at 8 AM - initial_timezone = 'UTC' - initial_state = 'enabled' - schedule_name = 'Integration Test Schedule' - schedule_description = 'Schedule created by integration test' - - created_schedule: ScheduleDetail | None = None - scheduler_id: str | None = None - try: - # Step 1: Create a schedule - LOG.info(f'Creating schedule for {target_component_id}/{target_configuration_id}') - created_schedule = await create_schedule( - client=keboola_client, - target_component_id=target_component_id, - target_configuration_id=target_configuration_id, - cron_tab=initial_cron_tab, - timezone=initial_timezone, - state=initial_state, - schedule_name=schedule_name, - schedule_description=schedule_description, - ) - scheduler_id = created_schedule.schedule_id - assert isinstance(created_schedule, ScheduleDetail) - assert created_schedule.schedule_id is not None - assert created_schedule.cron_tab == initial_cron_tab - assert created_schedule.timezone == initial_timezone - assert created_schedule.state == initial_state - LOG.info(f'Created schedule with ID: {created_schedule.schedule_id}') - - # Step 2: Retrieve the schedule using list_schedules_for_config - LOG.info('Retrieving schedules for configuration') - schedules = await list_schedules_for_config( - client=keboola_client, - component_id=target_component_id, - configuration_id=target_configuration_id, - ) - - assert len(schedules) >= 1, 'At least one schedule should exist' - found_schedule = next((s for s in schedules if s.schedule_id == created_schedule.schedule_id), None) - assert found_schedule is not None, 'Created schedule should be in the list' - assert found_schedule.cron_tab == initial_cron_tab - assert found_schedule.timezone == initial_timezone - assert found_schedule.state == initial_state - - # Step 3: Update the schedule - updated_cron_tab = '0 12 * * *' # Daily at 12 PM - updated_timezone = 'America/New_York' - updated_state = 'disabled' - - LOG.info(f'Updating schedule {created_schedule.schedule_id}') - updated_schedule = await update_schedule( - client=keboola_client, - schedule_config_id=created_schedule.schedule_id, - cron_tab=updated_cron_tab, - timezone=updated_timezone, - state=updated_state, - change_description='Integration test update', - ) - - assert isinstance(updated_schedule, ScheduleDetail) - assert updated_schedule.schedule_id == created_schedule.schedule_id - assert updated_schedule.cron_tab == updated_cron_tab - assert updated_schedule.timezone == updated_timezone - assert updated_schedule.state == updated_state - - # Step 4: Retrieve the schedule again to verify the update - LOG.info('Retrieving schedules after update') - schedules_after_update = await list_schedules_for_config( - client=keboola_client, - component_id=target_component_id, - configuration_id=target_configuration_id, - ) - - found_updated_schedule = next( - (s for s in schedules_after_update if s.schedule_id == created_schedule.schedule_id), None - ) - assert found_updated_schedule is not None - assert found_updated_schedule.cron_tab == updated_cron_tab - assert found_updated_schedule.timezone == updated_timezone - assert found_updated_schedule.state == updated_state - - # Step 5: Delete the schedule - LOG.info(f'Deleting schedule {created_schedule.schedule_id}') - await remove_schedule( - client=keboola_client, - schedule_config_id=created_schedule.schedule_id, - ) - - # Step 6: Verify the schedule is deleted - LOG.info('Verifying schedule deletion') - schedules_after_delete = await list_schedules_for_config( - client=keboola_client, - component_id=target_component_id, - configuration_id=target_configuration_id, - ) - - deleted_schedule_exists = any(s.schedule_id == created_schedule.schedule_id for s in schedules_after_delete) - assert not deleted_schedule_exists, 'Schedule should be deleted' - - LOG.info('Scheduler lifecycle test completed successfully') - finally: - if scheduler_id: - try: - await remove_schedule(client=keboola_client, schedule_config_id=scheduler_id) - await remove_schedule(client=keboola_client, schedule_config_id=scheduler_id) - await keboola_client.storage_client.configuration_delete( - component_id=SCHEDULER_COMPONENT_ID, configuration_id=scheduler_id, skip_trash=True - ) - except Exception: - LOG.info('Schedule cleanup error; schedule already removed.') - - -@pytest.mark.asyncio -async def test_scheduler_lifecycle_tooling( - initial_lf: FlowToolOutput, mcp_client: Client, configs: list[ConfigDef], keboola_client: KeboolaClient -) -> None: - """ - Test scheduler lifecycle using MCP tools: create schedule for a flow, update, and remove it. - """ - token_info = await keboola_client.storage_client.verify_token() - token_role = (token_info.get('admin', {}) or {}).get('role') - if token_role != 'admin': - pytest.skip('Scheduler tooling requires an admin token, skipping test.') - - assert configs - assert configs[0].configuration_id is not None - - flow_id = initial_lf.configuration_id - - schedule_id: str | None = None - try: - initial_cron_tab = '0 8 * * *' - initial_timezone = 'UTC' - - add_result = await mcp_client.call_tool( - name='modify_flow', - arguments={ - 'configuration_id': flow_id, - 'flow_type': ORCHESTRATOR_COMPONENT_ID, - 'change_description': 'Add scheduler via tooling', - 'schedules': [ - { - 'action': 'add', - 'cron_tab': initial_cron_tab, - 'timezone': initial_timezone, - 'state': 'enabled', - } - ], - }, - ) - add_output = FlowToolOutput.model_validate(add_result.structured_content) - assert add_output.success is True - - tool_call_result = await mcp_client.call_tool(name='get_flows', arguments={'flow_ids': [flow_id]}) - struct_call_result = cast(dict[str, Any], tool_call_result.structured_content) - flow_detail_result = GetFlowsDetailOutput.model_validate(struct_call_result['result']) - flow_detail = flow_detail_result.flows[0] - schedule = flow_detail.schedules.schedules[0] - schedule_id = schedule.schedule_id - - assert flow_detail.schedules is not None - assert flow_detail.schedules.n_schedules == 1 - assert schedule.cron_tab == initial_cron_tab - assert schedule.timezone == initial_timezone - assert schedule.state == 'enabled' - - updated_cron_tab = '0 12 * * *' - updated_timezone = 'America/New_York' - - update_result = await mcp_client.call_tool( - name='modify_flow', - arguments={ - 'configuration_id': flow_id, - 'flow_type': ORCHESTRATOR_COMPONENT_ID, - 'change_description': 'Update scheduler via tooling', - 'schedules': [ - { - 'action': 'update', - 'schedule_id': schedule_id, - 'cron_tab': updated_cron_tab, - 'timezone': updated_timezone, - 'state': 'disabled', - } - ], - }, - ) - update_output = FlowToolOutput.model_validate(update_result.structured_content) - assert update_output.success is True - - tool_call_result = await mcp_client.call_tool(name='get_flows', arguments={'flow_ids': [flow_id]}) - struct_call_result = cast(dict[str, Any], tool_call_result.structured_content) - flow_detail_result = GetFlowsDetailOutput.model_validate(struct_call_result['result']) - flow_detail = flow_detail_result.flows[0] - - assert flow_detail.schedules is not None - assert flow_detail.schedules.n_schedules == 1 - schedule = flow_detail.schedules.schedules[0] - assert schedule.schedule_id == schedule_id - assert schedule.cron_tab == updated_cron_tab - assert schedule.timezone == updated_timezone - assert schedule.state == 'disabled' - - remove_result = await mcp_client.call_tool( - name='modify_flow', - arguments={ - 'configuration_id': flow_id, - 'flow_type': ORCHESTRATOR_COMPONENT_ID, - 'change_description': 'Remove scheduler via tooling', - 'schedules': [{'action': 'remove', 'schedule_id': schedule_id}], - }, - ) - remove_output = FlowToolOutput.model_validate(remove_result.structured_content) - assert remove_output.success is True - - tool_call_result = await mcp_client.call_tool(name='get_flows', arguments={'flow_ids': [flow_id]}) - struct_call_result = cast(dict[str, Any], tool_call_result.structured_content) - flow_detail_result = GetFlowsDetailOutput.model_validate(struct_call_result['result']) - flow_detail = flow_detail_result.flows[0] - - assert flow_detail.schedules is not None - assert flow_detail.schedules.n_schedules == 0 - assert flow_detail.schedules.schedules == [] - finally: - if schedule_id: - try: - await remove_schedule(client=keboola_client, schedule_config_id=schedule_id) - await remove_schedule(client=keboola_client, schedule_config_id=schedule_id) - await keboola_client.storage_client.configuration_delete( - component_id=SCHEDULER_COMPONENT_ID, configuration_id=schedule_id, skip_trash=True - ) - except Exception: - LOG.info('Schedule cleanup error; schedule already removed.') diff --git a/integtests/tools/flow/test_tools.py b/integtests/tools/flow/test_tools.py deleted file mode 100644 index d1aadb157..000000000 --- a/integtests/tools/flow/test_tools.py +++ /dev/null @@ -1,1032 +0,0 @@ -import json -import logging -from typing import Any, cast - -import pydantic -import pytest -import toon_format -import yaml -from fastmcp import Client, Context -from pydantic import ValidationError - -from integtests.conftest import ConfigDef, ProjectDef -from keboola_mcp_server.clients.client import ( - CONDITIONAL_FLOW_COMPONENT_ID, - ORCHESTRATOR_COMPONENT_ID, - FlowType, - KeboolaClient, - get_metadata_property, -) -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.errors import ToolError -from keboola_mcp_server.links import Link, ProjectLinksManager -from keboola_mcp_server.tools.constants import MODIFY_FLOW_TOOL_NAME, UPDATE_FLOW_TOOL_NAME -from keboola_mcp_server.tools.flow.model import ConditionalFlowPhase, Flow, GetFlowsDetailOutput, GetFlowsListOutput -from keboola_mcp_server.tools.flow.tools import ( - FlowToolOutput, - create_conditional_flow, - create_flow, - get_flow_schema, - get_flows, -) -from keboola_mcp_server.tools.project import get_project_info - -LOG = logging.getLogger(__name__) - - -PYDANTIC_DOCS_VERSION = '.'.join(pydantic.__version__.split('.')[:2]) - - -@pytest.mark.asyncio -async def test_create_and_retrieve_flow(mcp_context: Context, configs: list[ConfigDef]) -> None: - """ - Create a flow and retrieve it using get_flows. - :param mcp_context: The test context fixture. - :param configs: List of real configuration definitions. - """ - assert configs - assert configs[0].configuration_id is not None - flow_type = ORCHESTRATOR_COMPONENT_ID - phases = [ - {'name': 'Extract', 'dependsOn': [], 'description': 'Extract data'}, - {'name': 'Transform', 'dependsOn': [1], 'description': 'Transform data'}, - ] - tasks = [ - { - 'name': 'Extract Task', - 'phase': 1, - 'task': { - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - }, - }, - { - 'name': 'Transform Task', - 'phase': 2, - 'task': { - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - }, - }, - ] - flow_name = 'Integration Test Flow' - flow_description = 'Flow created by integration test.' - - created = await create_flow( - ctx=mcp_context, - name=flow_name, - description=flow_description, - phases=phases, - tasks=tasks, - ) - flow_id = created.configuration_id - client = KeboolaClient.from_state(mcp_context.session.state) - links_manager = await ProjectLinksManager.from_client(client) - expected_links = [ - links_manager.get_flow_detail_link(flow_id=flow_id, flow_name=flow_name, flow_type=flow_type), - links_manager.get_flows_dashboard_link(flow_type=flow_type), - links_manager.get_flows_docs_link(), - ] - try: - assert isinstance(created, FlowToolOutput) - assert created.component_id == ORCHESTRATOR_COMPONENT_ID - assert created.description == flow_description - # Verify the links of created flow - assert created.success is True - assert set(created.links) == set(expected_links) - assert created.version is not None - - # Verify the flow is listed in the get_flows tool - result = await get_flows(mcp_context) - assert isinstance(result, GetFlowsListOutput) - assert any(f.name == flow_name for f in result.flows) - found = [f for f in result.flows if f.configuration_id == flow_id][0] - flow_detail_result = await get_flows(mcp_context, flow_ids=[found.configuration_id]) - assert isinstance(flow_detail_result, GetFlowsDetailOutput) - flow = flow_detail_result.flows[0] - - assert isinstance(flow, Flow) - assert flow.component_id == ORCHESTRATOR_COMPONENT_ID - assert flow.configuration_id == found.configuration_id - assert flow.configuration.phases[0].name == 'Extract' - assert flow.configuration.phases[1].name == 'Transform' - assert flow.configuration.tasks[0].task['componentId'] == configs[0].component_id - assert set(flow.links) == set(expected_links) - - # Verify the metadata - check that KBC.MCP.createdBy is set to 'true' - metadata = await client.storage_client.configuration_metadata_get( - component_id=ORCHESTRATOR_COMPONENT_ID, configuration_id=flow_id - ) - - # Convert metadata list to dictionary for easier checking - # metadata is a list of dicts with 'key' and 'value' keys - assert isinstance(metadata, list) - metadata_dict = {item['key']: item['value'] for item in metadata if isinstance(item, dict)} - assert MetadataField.CREATED_BY_MCP in metadata_dict - assert metadata_dict[MetadataField.CREATED_BY_MCP] == 'true' - finally: - await client.storage_client.configuration_delete( - component_id=ORCHESTRATOR_COMPONENT_ID, - configuration_id=flow_id, - skip_trash=True, - ) - - -@pytest.mark.asyncio -async def test_create_and_retrieve_conditional_flow(mcp_context: Context, configs: list[ConfigDef]) -> None: - """ - Create a conditional flow and retrieve it using get_flows. - :param mcp_context: The test context fixture. - :param configs: List of real configuration definitions. - """ - assert configs - assert configs[0].configuration_id is not None - flow_type = CONDITIONAL_FLOW_COMPONENT_ID - - phases = [ - { - 'id': 'extract_phase', - 'name': 'Extract', - 'description': 'Extract data', - 'next': [{'id': 'extract_to_transform', 'name': 'Extract to Transform', 'goto': 'transform_phase'}], - }, - { - 'id': 'transform_phase', - 'name': 'Transform', - 'description': 'Transform data', - 'next': [{'id': 'transform_end', 'name': 'End Flow', 'goto': None}], - }, - ] - tasks = [ - { - 'id': 'extract_task', - 'name': 'Extract Task', - 'phase': 'extract_phase', - 'task': { - 'type': 'job', - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - 'mode': 'run', - }, - }, - { - 'id': 'transform_task', - 'name': 'Transform Task', - 'phase': 'transform_phase', - 'task': { - 'type': 'job', - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - 'mode': 'run', - }, - }, - ] - flow_name = 'Integration Test Conditional Flow' - flow_description = 'Conditional flow created by integration test.' - - created = await create_conditional_flow( - ctx=mcp_context, - name=flow_name, - description=flow_description, - phases=phases, - tasks=tasks, - ) - flow_id = created.configuration_id - client = KeboolaClient.from_state(mcp_context.session.state) - links_manager = await ProjectLinksManager.from_client(client) - expected_links = [ - links_manager.get_flow_detail_link(flow_id=flow_id, flow_name=flow_name, flow_type=flow_type), - links_manager.get_flows_dashboard_link(flow_type=flow_type), - links_manager.get_flows_docs_link(), - ] - try: - assert isinstance(created, FlowToolOutput) - assert created.component_id == CONDITIONAL_FLOW_COMPONENT_ID - assert created.description == flow_description - assert created.success is True - assert set(created.links) == set(expected_links) - assert created.version is not None - - # Verify the flow is listed in the get_flows tool - result = await get_flows(mcp_context) - assert isinstance(result, GetFlowsListOutput) - assert any(f.name == flow_name for f in result.flows) - found = [f for f in result.flows if f.configuration_id == flow_id][0] - flow_detail_result = await get_flows(mcp_context, flow_ids=[found.configuration_id]) - assert isinstance(flow_detail_result, GetFlowsDetailOutput) - flow = flow_detail_result.flows[0] - - assert isinstance(flow, Flow) - assert flow.component_id == CONDITIONAL_FLOW_COMPONENT_ID - assert flow.configuration_id == found.configuration_id - assert flow.configuration.phases[0].name == 'Extract' - assert flow.configuration.phases[1].name == 'Transform' - assert flow.configuration.tasks[0].task.component_id == configs[0].component_id - assert set(flow.links) == set(expected_links) - - # Verify the metadata - check that KBC.MCP.createdBy is set to 'true' - metadata = await client.storage_client.configuration_metadata_get( - component_id=CONDITIONAL_FLOW_COMPONENT_ID, configuration_id=flow_id - ) - - # Convert metadata list to dictionary for easier checking - # metadata is a list of dicts with 'key' and 'value' keys - assert isinstance(metadata, list) - metadata_dict = {item['key']: item['value'] for item in metadata if isinstance(item, dict)} - assert MetadataField.CREATED_BY_MCP in metadata_dict - assert metadata_dict[MetadataField.CREATED_BY_MCP] == 'true' - finally: - await client.storage_client.configuration_delete( - component_id=CONDITIONAL_FLOW_COMPONENT_ID, - configuration_id=flow_id, - skip_trash=True, - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('flow_type', 'updates'), - [ - ( - ORCHESTRATOR_COMPONENT_ID, - { - 'phases': [ - {'id': 1, 'name': 'Phase1', 'dependsOn': [], 'description': 'First phase updated'}, - {'id': 2, 'name': 'Phase2', 'dependsOn': [], 'description': 'Second phase added'}, - ], - 'tasks': [ - { - 'id': 20001, - 'name': 'Task1 - Updated', - 'phase': 1, - 'continueOnFailure': False, - 'enabled': False, - 'task': {'componentId': 'ex-generic-v2', 'configId': 'test_config_001', 'mode': 'run'}, - }, - { - 'id': 20002, - 'name': 'Task2 - Added', - 'phase': 2, - 'continueOnFailure': False, - 'enabled': False, - 'task': {'componentId': 'ex-generic-v2', 'configId': 'test_config_002', 'mode': 'run'}, - }, - ], - 'name': 'Updated Test Flow', - 'description': 'The test flow updated by an automated test.', - }, - ), - ( - ORCHESTRATOR_COMPONENT_ID, - { - 'phases': [ - {'id': 1, 'name': 'Phase1', 'dependsOn': [], 'description': 'First phase updated'}, - {'id': 2, 'name': 'Phase2', 'dependsOn': [], 'description': 'Second phase added'}, - ] - }, - ), - ( - ORCHESTRATOR_COMPONENT_ID, - { - 'tasks': [ - { - 'id': 20001, - 'name': 'Task1 - Updated', - 'phase': 1, - 'continueOnFailure': False, - 'enabled': False, - 'task': {'componentId': 'ex-generic-v2', 'configId': 'test_config_001', 'mode': 'run'}, - }, - { - 'id': 20002, - 'name': 'Task2 - Added', - 'phase': 1, - 'continueOnFailure': False, - 'enabled': False, - 'task': {'componentId': 'ex-generic-v2', 'configId': 'test_config_002', 'mode': 'run'}, - }, - ] - }, - ), - (ORCHESTRATOR_COMPONENT_ID, {'name': 'Updated just name'}), - (ORCHESTRATOR_COMPONENT_ID, {'description': 'Updated just description'}), - ( - CONDITIONAL_FLOW_COMPONENT_ID, - { - 'phases': [ - { - 'id': 'phase1', - 'name': 'Phase1', - 'description': 'First phase updated', - 'next': [{'id': 'phase1_phase2', 'name': 'End Flow', 'goto': 'phase2'}], - }, - { - 'id': 'phase2', - 'name': 'Phase2', - 'description': 'Second phase added', - 'next': [{'id': 'phase2_end', 'name': 'End Flow', 'goto': None}], - }, - ], - 'tasks': [ - { - 'id': 'task1', - 'name': 'Task1 - Updated', - 'phase': 'phase1', - 'task': { - 'type': 'job', - 'componentId': 'ex-generic-v2', - 'configId': 'test_config_001', - 'mode': 'run', - }, - }, - { - 'id': 'task2', - 'name': 'Task2 - Added', - 'phase': 'phase2', - 'task': { - 'type': 'job', - 'componentId': 'ex-generic-v2', - 'configId': 'test_config_002', - 'mode': 'run', - }, - }, - ], - }, - ), - ( - CONDITIONAL_FLOW_COMPONENT_ID, - { - 'phases': [ - { - 'id': 'phase1', - 'name': 'Phase1', - 'description': 'First phase updated', - 'next': [{'id': 'phase1_phase2', 'name': 'End Flow', 'goto': 'phase2'}], - }, - { - 'id': 'phase2', - 'name': 'Phase2', - 'description': 'Second phase added', - 'next': [{'id': 'phase2_end', 'name': 'End Flow', 'goto': None}], - }, - ] - }, - ), - ( - CONDITIONAL_FLOW_COMPONENT_ID, - { - 'tasks': [ - { - 'id': 'task1', - 'name': 'Task1 - Updated', - 'phase': 'phase1', - 'task': { - 'type': 'job', - 'componentId': 'ex-generic-v2', - 'configId': 'test_config_001', - 'mode': 'run', - }, - }, - { - 'id': 'task2', - 'name': 'Task2 - Added', - 'phase': 'phase1', - 'task': { - 'type': 'job', - 'componentId': 'ex-generic-v2', - 'configId': 'test_config_002', - 'mode': 'run', - }, - }, - ] - }, - ), - (CONDITIONAL_FLOW_COMPONENT_ID, {'name': 'Updated just name'}), - (CONDITIONAL_FLOW_COMPONENT_ID, {'description': 'Updated just description'}), - (ORCHESTRATOR_COMPONENT_ID, {'is_disabled': True}), - (CONDITIONAL_FLOW_COMPONENT_ID, {'is_disabled': True}), - ], -) -async def test_update_flow( - flow_type: FlowType, - updates: dict[str, Any], - initial_lf: FlowToolOutput, - initial_cf: FlowToolOutput, - mcp_client: Client, - keboola_project: ProjectDef, - keboola_client: KeboolaClient, - storage_api_url: str, -) -> None: - """Tests that 'update_flow' tool works as expected.""" - flow_id = initial_lf.configuration_id if flow_type == ORCHESTRATOR_COMPONENT_ID else initial_cf.configuration_id - tool_call_result = await mcp_client.call_tool(name='get_flows', arguments={'flow_ids': [flow_id]}) - struct_call_result = cast(dict[str, Any], tool_call_result.structured_content) - initial_flow_result = GetFlowsDetailOutput.model_validate(struct_call_result['result']) - initial_flow = initial_flow_result.flows[0] - - # Determine the tool name to use based on the token role, should not break if not using schedulers - token_info = await keboola_client.storage_client.verify_token() - token_role = (token_info.get('admin', {}) or {}).get('role') - if token_role == 'admin': - tool_name = MODIFY_FLOW_TOOL_NAME - else: - tool_name = UPDATE_FLOW_TOOL_NAME - - project_id = keboola_project.project_id - tool_result = await mcp_client.call_tool( - name=tool_name, - arguments={ - 'configuration_id': flow_id, - 'flow_type': flow_type, - 'change_description': 'Integration test update', - **updates, - }, - ) - - # Check the tool's output - updated_flow = FlowToolOutput.model_validate(tool_result.structured_content) - assert updated_flow.configuration_id == flow_id - assert updated_flow.component_id == flow_type - assert updated_flow.success is True - assert updated_flow.timestamp is not None - assert updated_flow.version is not None - - expected_name = updates.get('name') or 'Initial Test Flow' - expected_description = updates.get('description') or initial_flow.description - assert updated_flow.description == expected_description - if flow_type == ORCHESTRATOR_COMPONENT_ID: - flow_path = 'flows' - flow_label = 'Flows' - else: - flow_path = 'flows-v2' - flow_label = 'Conditional Flows' - assert frozenset(updated_flow.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Flow: {expected_name}', - url=f'{storage_api_url}/admin/projects/{project_id}/{flow_path}/{flow_id}', - ), - Link( - type='ui-dashboard', - title=f'{flow_label} in the project', - url=f'{storage_api_url}/admin/projects/{project_id}/{flow_path}', - ), - Link(type='docs', title='Documentation for Keboola Flows', url='https://help.keboola.com/flows/'), - ] - ) - - # Verify the configuration was updated - tool_call_result = await mcp_client.call_tool(name='get_flows', arguments={'flow_ids': [flow_id]}) - struct_call_result = cast(dict[str, Any], tool_call_result.structured_content) - flow_detail_result = GetFlowsDetailOutput.model_validate(struct_call_result['result']) - flow_detail = flow_detail_result.flows[0] - - assert flow_detail.name == expected_name - assert flow_detail.description == expected_description - - # Verify is_disabled if it was updated - expected_is_disabled = updates.get('is_disabled') - if expected_is_disabled is not None: - assert flow_detail.is_disabled == expected_is_disabled - - flow_data = flow_detail.configuration.model_dump(exclude_unset=True, by_alias=True) - - # Check that ids, names, and transitions match for phases using assert all - if updates.get('phases'): - # Convert the phases to get the expected format. - if flow_type == ORCHESTRATOR_COMPONENT_ID: - expected_phases = updates['phases'] - else: - expected_phases = [ - ConditionalFlowPhase.model_validate(phase).model_dump(exclude_unset=True, by_alias=True) - for phase in updates['phases'] - ] - else: - expected_phases = [ - phase.model_dump(exclude_unset=True, by_alias=True) for phase in initial_flow.configuration.phases - ] - assert len(flow_data['phases']) == len( - expected_phases - ), f"Phases count mismatch: {len(flow_data['phases'])} vs {len(expected_phases)}" - assert all( - actual['id'] == expected['id'] - and actual['name'] == expected['name'] - and len(actual.get('next', [])) == len(expected.get('next', [])) - and all( - act_tr['id'] == exp_tr['id'] and act_tr['goto'] == exp_tr['goto'] - for act_tr, exp_tr in zip(actual.get('next', []), expected.get('next', [])) - ) - for actual, expected in zip(flow_data['phases'], expected_phases) - ), f"Phase id, name, or transitions do not match!\nExpected: {expected_phases}\nGot: {flow_data['phases']}" - - # Check that all task ids and names match between actual and expected using all() - if updates.get('tasks'): - expected_tasks = updates['tasks'] - else: - expected_tasks = [ - task.model_dump(exclude_unset=True, by_alias=True) for task in initial_flow.configuration.tasks - ] - assert all( - actual_task['id'] == expected_task['id'] and actual_task['name'] == expected_task['name'] - for actual_task, expected_task in zip(flow_data['tasks'], expected_tasks) - ), f"Task id or name mismatch!\nExpected: {expected_tasks}\nGot: {flow_data['tasks']}" - - current_version = flow_detail.version - assert current_version == 2 - - # Verify is_disabled in the raw API configuration if it was updated - if expected_is_disabled is not None: - raw_config = await keboola_client.storage_client.configuration_detail( - component_id=flow_type, configuration_id=flow_id - ) - assert raw_config.get('isDisabled') == expected_is_disabled - - # Check that KBC.MCP.updatedBy.version.{version} is set to 'true' - metadata = await keboola_client.storage_client.configuration_metadata_get( - component_id=flow_type, configuration_id=updated_flow.configuration_id - ) - assert isinstance(metadata, list), f'Expecting list, got: {type(metadata)}' - - meta_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{current_version}' - meta_value = get_metadata_property(metadata, meta_key) - assert meta_value == 'true' - # Check that the original creation metadata is still there - assert get_metadata_property(metadata, MetadataField.CREATED_BY_MCP) == 'true' - - -@pytest.mark.asyncio -async def test_get_flows_empty(mcp_context: Context) -> None: - """ - Retrieve flows when none exist (should not error, may return empty list). - :param mcp_context: The test context fixture. - """ - flows = await get_flows(mcp_context) - assert isinstance(flows, GetFlowsListOutput) - assert len(flows.flows) == 0 - - -@pytest.mark.asyncio -async def test_get_flows_list( - keboola_project: ProjectDef, - mcp_client: Client, - initial_lf: FlowToolOutput, - initial_cf: FlowToolOutput, - storage_api_url: str, -) -> None: - """Tests that `get_flows` tool works as expected when listing all flows.""" - tool_call_result = await mcp_client.call_tool(name='get_flows', arguments={}) - struct_call_result = cast(dict[str, Any], tool_call_result.structured_content) - flows = GetFlowsListOutput.model_validate(struct_call_result['result']) - assert len(flows.flows) == 2 - assert frozenset(flows.links) == frozenset( - [ - Link( - type='ui-dashboard', - title='Flows in the project', - url=f'{storage_api_url}/admin/projects/{keboola_project.project_id}/flows', - ), - Link( - type='ui-dashboard', - title='Conditional Flows in the project', - url=f'{storage_api_url}/admin/projects/{keboola_project.project_id}/flows-v2', - ), - ] - ) - assert flows.flows[0].configuration_id == initial_cf.configuration_id - assert flows.flows[1].configuration_id == initial_lf.configuration_id - assert tool_call_result.content is not None - assert len(tool_call_result.content) == 1 - assert tool_call_result.content[0].type == 'text' - toon_decoded = toon_format.decode(tool_call_result.content[0].text) - assert GetFlowsListOutput.model_validate(toon_decoded) == flows - - -@pytest.mark.asyncio -async def test_get_flow_schema(mcp_context: Context) -> None: - """ - Test that get_flow_schema returns the flow configuration JSON schema. - Tests the conditional behavior where the tool might return a different schema - than requested based on project settings. - """ - project_info = await get_project_info(mcp_context) - - # Test 1: Request orchestrator schema (should always work) - legacy_flow_schema = await get_flow_schema(mcp_context, ORCHESTRATOR_COMPONENT_ID) - - assert isinstance(legacy_flow_schema, str) - assert legacy_flow_schema.startswith('```json\n') - assert legacy_flow_schema.endswith('\n```') - assert 'dependsOn' in legacy_flow_schema - - # Extract and parse the JSON content to verify it's valid - json_content = legacy_flow_schema[8:-4] # Remove ```json\n and \n``` - parsed_legacy_schema = json.loads(json_content) - - # Verify basic schema structure for legacy flow - assert isinstance(parsed_legacy_schema, dict) - assert '$schema' in parsed_legacy_schema - assert 'properties' in parsed_legacy_schema - assert 'phases' in parsed_legacy_schema['properties'] - assert 'tasks' in parsed_legacy_schema['properties'] - - # Test 2: Request conditional flow schema (behavior depends on project settings) - conditional_schema = await get_flow_schema(mcp_context, CONDITIONAL_FLOW_COMPONENT_ID) - - assert isinstance(conditional_schema, str) - assert conditional_schema.startswith('```json\n') - assert conditional_schema.endswith('\n```') - - # Extract and parse the JSON content - json_content = conditional_schema[8:-4] # Remove ```json\n and \n``` - parsed_conditional_schema = json.loads(json_content) - - # Test 3: Verify the conditional behavior - if not project_info.conditional_flows: - # If the project does not support conditional flows, both requests should return the same schema - assert legacy_flow_schema == conditional_schema - LOG.info('Project has conditional flows disabled - both schemas are identical') - else: - # If conditional flows are enabled, the schemas should be different - assert legacy_flow_schema != conditional_schema - LOG.info('Project has conditional flows enabled - schemas are different') - - # Verify that the conditional schema has conditional-specific properties - conditional_phases = parsed_conditional_schema['properties']['phases']['items']['properties'] - assert 'next' in conditional_phases # Conditional flows use 'next' instead of 'dependsOn' - - conditional_tasks = parsed_conditional_schema['properties']['tasks']['items']['properties']['task'] - assert 'oneOf' in conditional_tasks # Conditional flows have structured task types - - # The conditional schema is sourced live from the Developer Portal — it must be non-empty - # and structurally a flow schema (not a stale/empty bundled placeholder). - assert parsed_conditional_schema # non-empty dict - assert parsed_conditional_schema['properties']['phases']['items']['properties'] - assert parsed_conditional_schema['properties']['tasks']['items']['properties'] - - -@pytest.mark.asyncio -async def test_create_legacy_flow_invalid_structure(mcp_context: Context, configs: list[ConfigDef]) -> None: - """ - Create a legacy flow with invalid structure (should raise ValueError). - :param mcp_context: The test context fixture. - :param configs: List of real configuration definitions. - """ - assert configs - assert configs[0].configuration_id is not None - phases = [ - {'name': 'Phase1', 'dependsOn': [99], 'description': 'Depends on non-existent phase'}, - ] - tasks = [ - { - 'name': 'Task1', - 'phase': 1, - 'task': { - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - }, - }, - ] - with pytest.raises(ValueError, match='depends on non-existent phase'): - await create_flow( - ctx=mcp_context, - name='Invalid Legacy Flow', - description='Should fail', - phases=phases, - tasks=tasks, - ) - - -@pytest.mark.asyncio -async def test_create_conditional_flow_invalid_structure(mcp_context: Context, configs: list[ConfigDef]) -> None: - """ - Create a conditional flow with invalid structure (should raise ValueError). - :param mcp_context: The test context fixture. - :param configs: List of real configuration definitions. - """ - assert configs - assert configs[0].configuration_id is not None - - # Test invalid conditional flow structure - missing required fields and invalid types - phases = [ - { - 'id': 123, # Invalid: should be string, not integer - 'name': '', # Invalid: empty string not allowed - 'next': [{'id': 'transition-1', 'goto': 'phase-2'}], - } - ] - - tasks = [ - { - 'id': 'task-1', - 'name': 'Task1', - 'phase': 'phase-1', - 'enabled': True, - 'task': { - 'type': 'invalid_type', # Invalid: not one of job, notification, variable - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - 'mode': 'invalid_mode', # Invalid: should be 'run' - }, - } - ] - - with pytest.raises(ToolError) as exc_info: - await create_conditional_flow( - ctx=mcp_context, - name='Invalid Conditional Flow', - description='Should fail', - phases=phases, - tasks=tasks, - ) - - err = exc_info.value - assert isinstance(err.__cause__, ValidationError) - - lines = str(err).splitlines() - assert len(lines) > 0, 'Empty error message' - assert lines[0] == 'Found 2 validation error(s) for ConditionalFlowPhase' - assert yaml.safe_load('\n'.join(lines[1:])) == { - 'errors': [ - { - 'field': 'id', - 'message': 'Input should be a valid string', - 'extra': { - 'type': 'string_type', - 'input': '123', - 'url': f'https://errors.pydantic.dev/{PYDANTIC_DOCS_VERSION}/v/string_type', - }, - }, - { - 'field': 'name', - 'message': 'String should have at least 1 character', - 'extra': { - 'type': 'string_too_short', - 'input': '', - 'ctx': "{'min_length': 1}", - 'url': f'https://errors.pydantic.dev/{PYDANTIC_DOCS_VERSION}/v/string_too_short', - }, - }, - ] - } - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('new_config', 'expected_error_message'), - [ - ( - { - 'phases': [ - { - 'id': 'phase-1', - 'name': 'Phase1', - 'next': [{'id': 'transition-1', 'goto': None}], - }, - { - 'id': 'phase-2', - 'name': 'Phase2', - 'next': [{'id': 'transition-2', 'goto': None}], - }, - ], - 'tasks': [ - { - 'id': 'task-1', - 'name': 'Task1', - 'phase': 'phase-1', - 'task': { - 'type': 'job', - 'componentId': 'ex-generic-v2', - 'configId': 'test_config_002', - 'mode': 'run', - }, - } - ], - }, - 'Flow has multiple entry phases', - ), - ( - { - 'phases': [ - { - 'id': 'phase-1', - 'name': 'Phase1', - 'next': [{'id': 'transition-1', 'goto': 'phase-2'}], - }, - { - 'id': 'phase-2', - 'name': 'Phase2', - 'next': [{'id': 'transition-2', 'goto': 'phase-1'}], - }, - ], - 'tasks': [ - { - 'id': 'task-1', - 'name': 'Task1', - 'phase': 'phase-1', - 'task': { - 'type': 'job', - 'componentId': 'ex-generic-v2', - 'configId': 'test_config_002', - 'mode': 'run', - }, - }, - { - 'id': 'task-2', - 'name': 'Task2', - 'phase': 'phase-2', - 'task': { - 'type': 'job', - 'componentId': 'ex-generic-v2', - 'configId': 'test_config_002', - 'mode': 'run', - }, - }, - ], - }, - 'Flow has no ending phases', - ), - ], -) -async def test_create_conditional_flow_semantically_invalid_structure( - mcp_context: Context, new_config: dict[str, list[dict]], expected_error_message: str -) -> None: - # Test invalid conditional flow structure - missing required fields and invalid types - phases = new_config['phases'] - tasks = new_config['tasks'] - - with pytest.raises(ValueError, match=expected_error_message): - await create_conditional_flow( - ctx=mcp_context, - name='Invalid Conditional Flow', - description='Should fail', - phases=phases, - tasks=tasks, - ) - - -@pytest.mark.asyncio -async def test_flow_lifecycle_integration(mcp_context: Context, configs: list[ConfigDef]) -> None: - """ - Test complete flow lifecycle for both legacy and conditional flows. - Creates flows, retrieves them individually, and lists all flows. - Tests project-aware behavior based on conditional flows setting. - """ - assert configs - assert configs[0].configuration_id is not None - - project_info = await get_project_info(mcp_context) - - # Test data for legacy flow - legacy_phases = [ - {'id': 1, 'name': 'Extract', 'description': 'Extract data from source', 'dependsOn': []}, - {'id': 2, 'name': 'Load', 'description': 'Load data to destination', 'dependsOn': [1]}, - ] - - legacy_tasks = [ - { - 'id': 20001, - 'name': 'Extract from API', - 'phase': 1, - 'enabled': True, - 'continueOnFailure': False, - 'task': {'componentId': configs[0].component_id, 'configId': configs[0].configuration_id, 'mode': 'run'}, - }, - { - 'id': 20002, - 'name': 'Load to Warehouse', - 'phase': 2, - 'enabled': True, - 'continueOnFailure': False, - 'task': {'componentId': configs[0].component_id, 'configId': configs[0].configuration_id, 'mode': 'run'}, - }, - ] - - # Test data for conditional flow - conditional_phases = [ - { - 'id': 'phase-1', - 'name': 'Extract', - 'description': 'Extract data from source', - 'next': [{'id': 'transition-1', 'goto': 'phase-2'}], - }, - {'id': 'phase-2', 'name': 'Load', 'description': 'Load data to destination', 'next': []}, - ] - - conditional_tasks = [ - { - 'id': 'task-1', - 'name': 'Extract from API', - 'phase': 'phase-1', - 'enabled': True, - 'task': { - 'type': 'job', - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - 'mode': 'run', - }, - }, - { - 'id': 'task-2', - 'name': 'Load to Warehouse', - 'phase': 'phase-2', - 'enabled': True, - 'task': { - 'type': 'job', - 'componentId': configs[0].component_id, - 'configId': configs[0].configuration_id, - 'mode': 'run', - }, - }, - ] - - created_flows = [] - - # Step 1: Create orchestrator flow (should always work) - orchestrator_flow_name = 'Integration Test Orchestrator Flow' - orchestrator_flow_description = 'Orchestrator flow created by integration test' - - orchestrator_result = await create_flow( - ctx=mcp_context, - name=orchestrator_flow_name, - description=orchestrator_flow_description, - phases=legacy_phases, - tasks=legacy_tasks, - ) - - assert isinstance(orchestrator_result, FlowToolOutput) - assert orchestrator_result.success is True - assert orchestrator_result.component_id == ORCHESTRATOR_COMPONENT_ID - assert orchestrator_result.description == orchestrator_flow_description - assert orchestrator_result.version is not None - created_flows.append((ORCHESTRATOR_COMPONENT_ID, orchestrator_result.configuration_id)) - - # Step 2: Try to create conditional flow (only if project allows it) - conditional_flow_name = 'Integration Test Conditional Flow' - conditional_flow_description = 'Conditional flow created by integration test' - - if project_info.conditional_flows: - conditional_result = await create_conditional_flow( - ctx=mcp_context, - name=conditional_flow_name, - description=conditional_flow_description, - phases=conditional_phases, - tasks=conditional_tasks, - ) - - assert isinstance(conditional_result, FlowToolOutput) - assert conditional_result.success is True - assert conditional_result.component_id == CONDITIONAL_FLOW_COMPONENT_ID - assert conditional_result.description == conditional_flow_description - assert conditional_result.version is not None - created_flows.append((CONDITIONAL_FLOW_COMPONENT_ID, conditional_result.configuration_id)) - else: - LOG.info('Conditional flows are disabled in this project, skipping conditional flow creation') - - # Step 3: Get individual flows - for flow_type, flow_id in created_flows: - flow_result = await get_flows(mcp_context, flow_ids=[flow_id]) - assert isinstance(flow_result, GetFlowsDetailOutput) - flow = flow_result.flows[0] - - assert isinstance(flow, Flow) - assert flow.configuration_id == flow_id - - if flow_type == ORCHESTRATOR_COMPONENT_ID: - assert flow.name == orchestrator_flow_name - assert flow.component_id == ORCHESTRATOR_COMPONENT_ID - assert len(flow.configuration.phases) == 2 - assert len(flow.configuration.tasks) == 2 - assert flow.configuration.phases[0].name == 'Extract' - assert flow.configuration.phases[1].name == 'Load' - else: - assert flow.name == conditional_flow_name - assert flow.component_id == CONDITIONAL_FLOW_COMPONENT_ID - assert len(flow.configuration.phases) == 2 - assert len(flow.configuration.tasks) == 2 - assert flow.configuration.phases[0].name == 'Extract' - assert flow.configuration.phases[1].name == 'Load' - - # Step 4: List all flows and verify our created flows are there - flows_list = await get_flows(mcp_context) - - assert isinstance(flows_list, GetFlowsListOutput) - assert len(flows_list.flows) >= len(created_flows) - - # Verify our created flows are in the list - flow_ids = [flow.configuration_id for flow in flows_list.flows] - for flow_type, flow_id in created_flows: - assert flow_id in flow_ids, f'Created {flow_type} flow {flow_id} not found in flows list' - - # Step 5: Clean up - delete all created flows - client = KeboolaClient.from_state(mcp_context.session.state) - for flow_type, flow_id in created_flows: - try: - await client.storage_client.configuration_delete( - component_id=flow_type, - configuration_id=flow_id, - skip_trash=True, - ) - LOG.info(f'Successfully deleted {flow_type} flow {flow_id}') - except Exception as e: - LOG.warning(f'Failed to delete {flow_type} flow {flow_id}: {e}') diff --git a/integtests/tools/jobs.test.ts b/integtests/tools/jobs.test.ts new file mode 100644 index 000000000..dce4acd46 --- /dev/null +++ b/integtests/tools/jobs.test.ts @@ -0,0 +1,180 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolText, connectMcp } from '../helpers/mcp'; +import { seedProject } from '../helpers/seed'; +import { getTestProjectForTest } from '../testproject/fixture'; + +// Ported from integtests/tools/test_jobs.py. Each case leases a fresh project, seeds the +// standard fixtures (which include an ex-generic-v2 config), then runs a job against that +// config and polls get_jobs. Real jobs against the live stack take ~10-40s, so the suite is +// intentionally slow. + +const RUN_TIMEOUT = 120_000; + +/** + * Polls get_jobs (listing mode, filtered by component + config) until the just-started job + * id shows up, mirroring the Python `_wait_for_job_in_list` retry helper (the queue is + * eventually-consistent right after a job is created). + */ +const waitForJobInList = async ( + client: Parameters[0], + jobId: string, + componentId: string, + configId: string, + maxRetries = 15, + delayMs = 1000, +): Promise => { + for (let attempt = 0; attempt < maxRetries; attempt++) { + const text = await callToolText(client, 'get_jobs', { + component_id: componentId, + config_id: configId, + limit: 10, + sort_by: 'startTime', + sort_order: 'desc', + }); + if (text.includes(jobId)) return text; + if (attempt < maxRetries - 1) await new Promise((r) => setTimeout(r, delayMs)); + } + throw new Error(`Job ${jobId} not found in job list after ${maxRetries} attempts`); +}; + +/** Extracts the started job id from the run_job TOON output (top-level `id: `). */ +const extractJobId = (runJobText: string): string => { + // TOON quotes numeric-looking string scalars, so the id may be `id: "123"` or `id: 123`. + const match = runJobText.match(/\bid:\s*"?(\d+)"?/); + expect(match, `run_job output should contain a job id. Got: ${runJobText}`).not.toBeNull(); + return match![1]!; +}; + +describe('jobs tools (integration)', () => { + it( + 'run_job starts a job and get_jobs lists it under the component/config filter', + async () => { + const project = await getTestProjectForTest(); + const seeded = await seedProject(project); + const config = seeded.configs.find((c) => c.componentId === 'ex-generic-v2')!; + const session = await connectMcp(project.config); + try { + const runText = await callToolText(session.client, 'run_job', { + component_id: config.componentId, + configuration_id: config.configurationId, + }); + expect(runText).toContain(config.componentId); + expect(runText).toContain(config.configurationId); + const jobId = extractJobId(runText); + + const listText = await waitForJobInList( + session.client, + jobId, + config.componentId, + config.configurationId, + ); + // Every listed job under this filter must belong to the same component + config. + expect(listText).toContain(config.componentId); + expect(listText).toContain(config.configurationId); + } finally { + await session.close(); + } + }, + RUN_TIMEOUT, + ); + + it( + 'run_job then get_jobs(job_ids) returns the job detail with status, url and links', + async () => { + const project = await getTestProjectForTest(); + const seeded = await seedProject(project); + const config = seeded.configs.find((c) => c.componentId === 'ex-generic-v2')!; + const session = await connectMcp(project.config); + try { + const runText = await callToolText(session.client, 'run_job', { + component_id: config.componentId, + configuration_id: config.configurationId, + }); + const jobId = extractJobId(runText); + // The started-job response carries the component/config + UI links. + expect(runText).toContain(config.componentId); + expect(runText).toContain(config.configurationId); + expect(runText).toMatch(/queue\/\d+/); + expect(runText).toContain(`/queue/${jobId}`); + + const detailText = await callToolText(session.client, 'get_jobs', { job_ids: [jobId] }); + expect(detailText).toContain(jobId); + expect(detailText).toContain(config.componentId); + expect(detailText).toContain(config.configurationId); + // Detail includes a status, a url and the ui-detail / ui-dashboard links. + expect(detailText).toMatch(/status/i); + expect(detailText).toContain(`/queue/${jobId}`); + expect(detailText).toContain('ui-detail'); + expect(detailText).toContain('ui-dashboard'); + } finally { + await session.close(); + } + }, + RUN_TIMEOUT, + ); + + it( + 'get_jobs(job_ids, include_logs) returns the job detail and a logs section', + async () => { + const project = await getTestProjectForTest(); + const seeded = await seedProject(project); + const config = seeded.configs.find((c) => c.componentId === 'ex-generic-v2')!; + const session = await connectMcp(project.config); + try { + const runText = await callToolText(session.client, 'run_job', { + component_id: config.componentId, + configuration_id: config.configurationId, + }); + const jobId = extractJobId(runText); + + const detailText = await callToolText(session.client, 'get_jobs', { + job_ids: [jobId], + include_logs: true, + }); + expect(detailText).toContain(jobId); + expect(detailText).toMatch(/logs/i); + } finally { + await session.close(); + } + }, + RUN_TIMEOUT, + ); + + it( + 'run_job works against a freshly created config', + async () => { + const project = await getTestProjectForTest(); + const session = await connectMcp(project.config); + try { + const componentId = 'ex-generic-v2'; + const createText = await callToolText(session.client, 'create_config', { + name: 'Test Config for Job Run', + description: 'Test configuration created for job run test', + component_id: componentId, + parameters: { api: { baseUrl: 'https://wttr.in' } }, + storage: {}, + }); + const cfgMatch = createText.match(/configuration_id:\s*"?([^\s"]+)"?/); + expect(cfgMatch, `create_config should return a configuration id. Got: ${createText}`).not.toBeNull(); + const configurationId = cfgMatch![1]!; + + const runText = await callToolText(session.client, 'run_job', { + component_id: componentId, + configuration_id: configurationId, + }); + const jobId = extractJobId(runText); + expect(runText).toContain(componentId); + expect(runText).toContain(configurationId); + + const detailText = await callToolText(session.client, 'get_jobs', { job_ids: [jobId] }); + expect(detailText).toContain(jobId); + expect(detailText).toContain(componentId); + expect(detailText).toContain(configurationId); + } finally { + await session.close(); + } + }, + RUN_TIMEOUT, + ); +}); diff --git a/integtests/tools/project.test.ts b/integtests/tools/project.test.ts new file mode 100644 index 000000000..b79031029 --- /dev/null +++ b/integtests/tools/project.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolText, connectMcp } from '../helpers/mcp'; +import { getTestProjectForTest } from '../testproject/fixture'; + +// Ported from integtests/tools/test_project.py. +describe('get_project_info (integration)', () => { + it('returns project id, sql dialect, links, and branch context', async () => { + const { config, projectId } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const text = await callToolText(session.client, 'get_project_info'); + + // project_id of the leased project. + expect(text).toContain(String(projectId)); + // sql_dialect is one of the two supported backends. + expect(text).toMatch(/Snowflake|BigQuery/); + // links list is present (ui-detail / ui-dashboard / docs). + expect(text).toMatch(/ui-detail|ui-dashboard|docs/); + // The pool runs on the default (production) branch. + expect(text).toMatch(/is_development_branch[^\n]*false/i); + } finally { + await session.close(); + } + }); +}); diff --git a/integtests/tools/search.test.ts b/integtests/tools/search.test.ts new file mode 100644 index 000000000..be128586a --- /dev/null +++ b/integtests/tools/search.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolText, connectMcp } from '../helpers/mcp'; +import { seedProject } from '../helpers/seed'; +import { getTestProjectForTest } from '../testproject/fixture'; + +// Ported from integtests/tools/test_search.py. (find_component_id moved to +// integtests/tools/doc.test.ts — it is now served by the pgvector docs-search index.) +describe('search (integration)', () => { + it('finds seeded buckets, tables and configs end-to-end', async () => { + const project = await getTestProjectForTest(); + const seeded = await seedProject(project); + const session = await connectMcp(project.config); + try { + // Unscoped textual search across all item types for the 'test' name prefix. + const text = await callToolText(session.client, 'search', { + patterns: ['test'], + limit: 50, + offset: 0, + }); + + // The seeded buckets and table appear by id. + for (const bucket of seeded.buckets) expect(text).toContain(bucket.id); + for (const table of seeded.tables) expect(text).toContain(table.id); + // Both seeded configurations appear by id (ex-generic-v2 + snowflake-transformation). + for (const config of seeded.configs) expect(text).toContain(config.configurationId); + } finally { + await session.close(); + } + }); + + it('config-based scoped search matches the ex-generic-v2 config by api.baseUrl', async () => { + const project = await getTestProjectForTest(); + const seeded = await seedProject(project); + const config = seeded.configs.find((c) => c.componentId === 'ex-generic-v2')!; + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'search', { + patterns: ['wttr.in'], + item_types: ['configuration'], + search_type: 'config-based', + scopes: ['parameters.api.baseUrl'], + limit: 20, + offset: 0, + }); + expect(text).toContain('ex-generic-v2'); + expect(text).toContain(config.configurationId); + } finally { + await session.close(); + } + }); +}); diff --git a/integtests/tools/semantic.test.ts b/integtests/tools/semantic.test.ts new file mode 100644 index 000000000..3fbe46b5a --- /dev/null +++ b/integtests/tools/semantic.test.ts @@ -0,0 +1,346 @@ +import { randomUUID } from 'node:crypto'; +import { describe, expect, it } from 'vitest'; + +import { callToolRaw, callToolText, connectMcp, type McpSession } from '../helpers/mcp'; +import { getTestProjectForTest, type TestProject } from '../testproject/fixture'; + +import { createRawClient } from '@/clients/raw'; +import { deriveServiceUrls } from '@/clients/urls'; +import type { Config } from '@/config'; + +// Ported from integtests/tools/semantic/test_tools.py. +// +// The four semantic tools (get_semantic_context, get_semantic_schema, +// search_semantic_context, validate_semantic_query) are gated behind the +// `mcp-semantic-tooling` project feature (src/mcp/filtering.ts). When the feature is +// absent the tools are FILTERED OUT of tools/list and any call is DENIED with an +// McpError -> the SDK surfaces that as a thrown/rejected promise carrying the message +// 'is not available in this project ... "Semantic Layer Tooling" feature'. +// +// The shared pool projects do NOT have this feature (verified live: tokens/verify returns +// no mcp-semantic-tooling for any pool project). So in this environment every semantic +// test is expected to take the "feature absent" branch: we assert the documented +// unavailable behavior (tool hidden + call denied) once, and skip the metastore-seeded +// happy-path tests with a clear reason. The happy-path assertions are kept inline (guarded +// behind the feature probe) so they run as soon as a project with the feature is leased. + +const SEMANTIC_TOOL_NAMES = [ + 'search_semantic_context', + 'get_semantic_context', + 'get_semantic_schema', + 'validate_semantic_query', +] as const; + +/** True if the leased project has the semantic-tooling feature surfaced (tools listed). */ +const semanticToolsAvailable = async (session: McpSession): Promise => { + const { tools } = await session.client.listTools(); + const names = new Set(tools.map((t) => t.name)); + return SEMANTIC_TOOL_NAMES.every((n) => names.has(n)); +}; + +// --------------------------------------------------------------------------- +// Metastore seeding (port of the Python `semantic_test_setup` fixture). Only used when the +// feature is present; the raw metastore client mirrors keboola_client.metastore_client. +// --------------------------------------------------------------------------- + +type MetastoreObject = { id: string; [k: string]: unknown }; + +const createMetastore = (config: Config) => { + const urls = deriveServiceUrls(config.storageApiUrl!); + const token = config.bearerToken ? `Bearer ${config.bearerToken}` : config.storageToken!; + const raw = createRawClient({ baseUrl: urls.metastore, token }); + return { + createObject: async ( + objectType: string, + name: string, + data: Record, + ): Promise => + raw.post(`objects/${objectType}`, { body: { name, data } }), + deleteObject: async (objectType: string, id: string): Promise => { + await raw.delete(`objects/${objectType}/${id}`); + }, + }; +}; + +type SemanticSetup = { + slug: string; + modelId: string; + modelName: string; + primaryDatasetId: string; + secondaryDatasetId: string; + primaryTableId: string; + primaryFqn: string; + metricId: string; + metricName: string; + relationshipId: string; + constraintId: string; + cleanup: () => Promise; +}; + +/** + * Seeds a full semantic model (model + 2 datasets + metric + relationship + constraint), + * returning the created IDs plus a teardown that deletes them (twice, to clear the + * soft-delete), mirroring the Python fixture. Only invoked when the feature is enabled. + */ +const seedSemanticModel = async (project: TestProject): Promise => { + const metastore = createMetastore(project.config); + const uniqueId = randomUUID().slice(0, 8); + const slug = `it-semantic-${uniqueId}`; + const primaryTableId = `in.c-it-semantic.${slug}_orders`; + const secondaryTableId = `in.c-it-semantic.${slug}_orders_aux`; + const primaryFqn = `${slug}_orders`; + const secondaryFqn = `${slug}_orders_aux`; + const sqlDialect = project.backend === 'bigquery' ? 'bigquery' : 'snowflake'; + + const created: [string, string][] = []; + const cleanup = async (): Promise => { + for (const [objectType, id] of [...created].reverse()) { + try { + await metastore.deleteObject(objectType, id); + await metastore.deleteObject(objectType, id); + } catch { + // best-effort (401/403/404 expected on the second pass). + } + } + }; + + try { + const modelName = `${slug} model`; + const model = await metastore.createObject('semantic-model', modelName, { + name: modelName, + description: `Semantic walkthrough model ${slug}`, + sql_dialect: sqlDialect, + }); + created.push(['semantic-model', model.id]); + + const primaryDataset = await metastore.createObject('semantic-dataset', `${slug} orders`, { + name: `${slug} orders`, + description: `Primary walkthrough dataset ${slug}`, + tableId: primaryTableId, + fqn: primaryFqn, + modelUUID: model.id, + }); + created.push(['semantic-dataset', primaryDataset.id]); + + const secondaryDataset = await metastore.createObject( + 'semantic-dataset', + `${slug} orders aux`, + { + name: `${slug} orders aux`, + description: `Secondary walkthrough dataset ${slug}`, + tableId: secondaryTableId, + fqn: secondaryFqn, + modelUUID: model.id, + }, + ); + created.push(['semantic-dataset', secondaryDataset.id]); + + const metricName = `${slug} total items`; + const metric = await metastore.createObject('semantic-metric', metricName, { + name: metricName, + description: `Walkthrough metric ${slug}`, + sql: 'SUM(item_count)', + dataset: primaryTableId, + modelUUID: model.id, + }); + created.push(['semantic-metric', metric.id]); + + const relationship = await metastore.createObject( + 'semantic-relationship', + `${slug} relationship`, + { + name: `${slug} relationship`, + modelUUID: model.id, + from: primaryTableId, + to: secondaryTableId, + type: 'left', + on: 'orders.id = orders_aux.id', + }, + ); + created.push(['semantic-relationship', relationship.id]); + + const constraintName = `it_semantic_${uniqueId}_constraint`; + const constraint = await metastore.createObject('semantic-constraint', constraintName, { + name: constraintName, + description: `Walkthrough exclusion rule ${slug}`, + modelUUID: model.id, + constraintType: 'exclusion', + severity: 'warning', + rule: 'Do not combine both walkthrough datasets in one query.', + metrics: [metricName], + datasets: [primaryTableId, secondaryTableId], + }); + created.push(['semantic-constraint', constraint.id]); + + return { + slug, + modelId: model.id, + modelName, + primaryDatasetId: primaryDataset.id, + secondaryDatasetId: secondaryDataset.id, + primaryTableId, + primaryFqn, + metricId: metric.id, + metricName, + relationshipId: relationship.id, + constraintId: constraint.id, + cleanup, + }; + } catch (err) { + await cleanup(); + throw err; + } +}; + +describe('semantic tools (integration)', () => { + // Documented unavailable-behavior: with the feature absent the tools are hidden from + // tools/list and any call is denied. This keeps the suite meaningful (not pure skips) + // against the feature-less pool. When the feature IS enabled this asserts availability. + it('semantic tools follow the project-feature gate', async () => { + const project = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(project.config); + try { + const available = await semanticToolsAvailable(session); + if (available) { + // Feature present: a call must NOT be denied by the gate (it may still need data, + // but get_semantic_schema works with no project data). + const schema = await callToolText(session.client, 'get_semantic_schema', { + semantic_types: ['semantic-dataset'], + }); + expect(schema).toContain('semantic-dataset'); + return; + } + // Feature absent: tool hidden from list AND call denied with the gating message. + const result = await callToolRaw(session.client, 'get_semantic_schema', { + semantic_types: ['semantic-dataset'], + }).then( + (r) => ({ thrown: false as const, r }), + (e: unknown) => ({ thrown: true as const, message: (e as Error).message }), + ); + expect(result.thrown).toBe(true); + if (result.thrown) { + expect(result.message).toMatch(/not available in this project/i); + expect(result.message).toMatch(/Semantic Layer Tooling/i); + } + } finally { + await session.close(); + } + }); + + // Port of test_search_semantic_context. + it('search_semantic_context groups matches by semantic model', async (ctx) => { + const project = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(project.config); + try { + if (!(await semanticToolsAvailable(session))) { + // Skipped: project lacks the `mcp-semantic-tooling` feature (none of the shared pool + // projects have it), so the tool is filtered out / denied. See the gate test above. + return ctx.skip(); + } + const setup = await seedSemanticModel(project); + try { + const text = await callToolText(session.client, 'search_semantic_context', { + patterns: [setup.slug], + max_results: 20, + }); + // One model group, matching our seeded model, surfacing all object types. + expect(text).toContain(setup.modelId); + expect(text).toContain('semantic-model'); + expect(text).toContain('semantic-dataset'); + expect(text).toContain('semantic-metric'); + expect(text).toContain('semantic-relationship'); + expect(text).toContain('semantic-constraint'); + } finally { + await setup.cleanup(); + } + } finally { + await session.close(); + } + }); + + // Port of test_get_semantic_context. + it('get_semantic_context returns objects grouped by type', async (ctx) => { + const project = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(project.config); + try { + if (!(await semanticToolsAvailable(session))) { + return ctx.skip(); + } + const setup = await seedSemanticModel(project); + try { + const text = await callToolText(session.client, 'get_semantic_context', { + semantic_objects: [ + { object_type: 'semantic-model', ids: [setup.modelId] }, + { + object_type: 'semantic-dataset', + ids: [setup.primaryDatasetId, setup.secondaryDatasetId], + }, + { object_type: 'semantic-metric', ids: [setup.metricId] }, + { object_type: 'semantic-relationship', ids: [setup.relationshipId] }, + { object_type: 'semantic-constraint', ids: [setup.constraintId] }, + ], + semantic_model_ids: [setup.modelId], + }); + expect(text).toContain(setup.modelId); + expect(text).toContain(setup.primaryDatasetId); + expect(text).toContain(setup.secondaryDatasetId); + // ids-supplied selections return full objects with an `attributes` block. + expect(text).toMatch(/attributes/); + } finally { + await setup.cleanup(); + } + } finally { + await session.close(); + } + }); + + // Port of test_get_semantic_schema. Needs no seeded data. + it('get_semantic_schema returns JSON schemas for requested types', async (ctx) => { + const project = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(project.config); + try { + if (!(await semanticToolsAvailable(session))) { + return ctx.skip(); + } + const text = await callToolText(session.client, 'get_semantic_schema', { + semantic_types: ['semantic-dataset', 'semantic-metric'], + }); + expect(text).toContain('semantic-dataset'); + expect(text).toContain('semantic-metric'); + expect(text).toMatch(/schema/); + } finally { + await session.close(); + } + }); + + // Port of test_validate_semantic_query. + it('validate_semantic_query validates a query against the seeded model', async (ctx) => { + const project = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(project.config); + try { + if (!(await semanticToolsAvailable(session))) { + return ctx.skip(); + } + const setup = await seedSemanticModel(project); + try { + const text = await callToolText(session.client, 'validate_semantic_query', { + sql_query: `SELECT SUM(item_count) AS total_items FROM ${setup.primaryFqn}`, + semantic_model_ids: [setup.modelId], + expected_semantic_objects: [ + { object_type: 'semantic-dataset', ids: [setup.primaryDatasetId] }, + { object_type: 'semantic-metric', ids: [setup.metricId] }, + ], + }); + // Auto-detected validation is valid and resolves to our model. + expect(text).toMatch(/valid:\s*true/i); + expect(text).toContain(setup.modelId); + expect(text).toContain(setup.metricId); + expect(text).toContain(setup.primaryDatasetId); + } finally { + await setup.cleanup(); + } + } finally { + await session.close(); + } + }); +}); diff --git a/integtests/tools/semantic/__init__.py b/integtests/tools/semantic/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/integtests/tools/semantic/test_tools.py b/integtests/tools/semantic/test_tools.py deleted file mode 100644 index 1f694832a..000000000 --- a/integtests/tools/semantic/test_tools.py +++ /dev/null @@ -1,344 +0,0 @@ -from __future__ import annotations - -from collections import Counter -from dataclasses import dataclass -from typing import Any, cast -from urllib.parse import urljoin - -import httpx -import pytest -import pytest_asyncio -from fastmcp import Client - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.tools.semantic.model import SemanticObjectType, SemanticSchemaDefinition -from keboola_mcp_server.tools.semantic.tools import ( - SemanticObjectTypeContext, - SemanticSearchModelGroup, - ValidateSemanticQueryOutput, -) -from keboola_mcp_server.workspace import WorkspaceManager - -SEMANTIC_TOOLING_FEATURE = 'mcp-semantic-tooling' - - -@dataclass(frozen=True) -class SemanticTestSetup: - slug: str - model_id: str - model_name: str - primary_dataset_id: str - secondary_dataset_id: str - primary_table_id: str - secondary_table_id: str - primary_fqn: str - metric_id: str - metric_name: str - relationship_id: str - constraint_id: str - - -async def _delete_metastore_object(client: KeboolaClient, object_type: str, object_id: str) -> None: - try: - await client.metastore_client.delete_object(object_type, object_id) - # Delete the soft deleted object - await client.metastore_client.delete_object(object_type, object_id) - except httpx.HTTPStatusError as exc: - if exc.response.status_code not in (401, 403, 404): - raise - - -@pytest.fixture(scope='module') -def metastore_url(storage_api_url: str) -> str: - return storage_api_url.replace('connection.', 'metastore.', 1) - - -@pytest.fixture(scope='module', autouse=True) -def _require_metastore_available( - storage_api_token: str, - metastore_url: str, -) -> None: - try: - probe_url = urljoin(metastore_url, '/health-check') - with httpx.Client( - headers={'X-StorageApi-Token': storage_api_token}, - timeout=httpx.Timeout(3.0, connect=1.0), - ) as client: - response = client.get(probe_url) - response.raise_for_status() - except httpx.ConnectError as exc: - pytest.skip(f'Metastore endpoint is not reachable in this environment: {exc}') - except httpx.TimeoutException as exc: - pytest.skip(f'Metastore endpoint timed out in this environment: {exc}') - except httpx.HTTPStatusError as exc: - pytest.skip(f'Metastore endpoint returned HTTP {exc.response.status_code}: {exc}') - - -@pytest_asyncio.fixture(autouse=True) -async def semantic_tools_enabled(keboola_client: KeboolaClient) -> None: - token_info = await keboola_client.storage_client.verify_token() - owner = token_info.get('owner', {}) - features = owner.get('features', []) if isinstance(owner, dict) else [] - if SEMANTIC_TOOLING_FEATURE not in features: - pytest.skip(f'Semantic tooling feature "{SEMANTIC_TOOLING_FEATURE}" is not enabled in this environment.') - - -@pytest_asyncio.fixture -async def semantic_test_setup( - keboola_client: KeboolaClient, - unique_id: str, - workspace_manager: WorkspaceManager, -) -> SemanticTestSetup: - sql_dialect = await workspace_manager.get_sql_dialect() - slug = f'it-semantic-{unique_id}' - primary_table_id = f'in.c-it-semantic.{slug}_orders' - secondary_table_id = f'in.c-it-semantic.{slug}_orders_aux' - primary_fqn = f'{slug}_orders' - secondary_fqn = f'{slug}_orders_aux' - - created_objects: list[tuple[str, str]] = [] - - try: - model_name = f'{slug} model' - constraint_name = f'it_semantic_{unique_id}_constraint' - model = await keboola_client.metastore_client.create_object( - SemanticObjectType.SEMANTIC_MODEL.value, - name=model_name, - data={ - 'name': model_name, - 'description': f'Semantic walkthrough model {slug}', - 'sql_dialect': sql_dialect, - }, - ) - created_objects.append((SemanticObjectType.SEMANTIC_MODEL.value, model.id)) - - primary_dataset = await keboola_client.metastore_client.create_object( - SemanticObjectType.SEMANTIC_DATASET.value, - name=f'{slug} orders', - data={ - 'name': f'{slug} orders', - 'description': f'Primary walkthrough dataset {slug}', - 'tableId': primary_table_id, - 'fqn': primary_fqn, - 'modelUUID': model.id, - }, - ) - created_objects.append((SemanticObjectType.SEMANTIC_DATASET.value, primary_dataset.id)) - - secondary_dataset = await keboola_client.metastore_client.create_object( - SemanticObjectType.SEMANTIC_DATASET.value, - name=f'{slug} orders aux', - data={ - 'name': f'{slug} orders aux', - 'description': f'Secondary walkthrough dataset {slug}', - 'tableId': secondary_table_id, - 'fqn': secondary_fqn, - 'modelUUID': model.id, - }, - ) - created_objects.append((SemanticObjectType.SEMANTIC_DATASET.value, secondary_dataset.id)) - - metric_name = f'{slug} total items' - metric = await keboola_client.metastore_client.create_object( - SemanticObjectType.SEMANTIC_METRIC.value, - name=metric_name, - data={ - 'name': metric_name, - 'description': f'Walkthrough metric {slug}', - 'sql': 'SUM(item_count)', - 'dataset': primary_table_id, - 'modelUUID': model.id, - }, - ) - created_objects.append((SemanticObjectType.SEMANTIC_METRIC.value, metric.id)) - - relationship = await keboola_client.metastore_client.create_object( - SemanticObjectType.SEMANTIC_RELATIONSHIP.value, - name=f'{slug} relationship', - data={ - 'name': f'{slug} relationship', - 'modelUUID': model.id, - 'from': primary_table_id, - 'to': secondary_table_id, - 'type': 'left', - 'on': 'orders.id = orders_aux.id', - }, - ) - created_objects.append((SemanticObjectType.SEMANTIC_RELATIONSHIP.value, relationship.id)) - - constraint = await keboola_client.metastore_client.create_object( - SemanticObjectType.SEMANTIC_CONSTRAINT.value, - name=constraint_name, - data={ - 'name': constraint_name, - 'description': f'Walkthrough exclusion rule {slug}', - 'modelUUID': model.id, - 'constraintType': 'exclusion', - 'severity': 'warning', - 'rule': 'Do not combine both walkthrough datasets in one query.', - 'metrics': [metric_name], - 'datasets': [primary_table_id, secondary_table_id], - }, - ) - created_objects.append((SemanticObjectType.SEMANTIC_CONSTRAINT.value, constraint.id)) - - yield SemanticTestSetup( - slug=slug, - model_id=model.id, - model_name=model_name, - primary_dataset_id=primary_dataset.id, - secondary_dataset_id=secondary_dataset.id, - primary_table_id=primary_table_id, - secondary_table_id=secondary_table_id, - primary_fqn=primary_fqn, - metric_id=metric.id, - metric_name=metric_name, - relationship_id=relationship.id, - constraint_id=constraint.id, - ) - finally: - for object_type, object_id in reversed(created_objects): - await _delete_metastore_object(keboola_client, object_type, object_id) - - -@pytest.mark.asyncio -async def test_search_semantic_context( - mcp_client: Client, - semantic_test_setup: SemanticTestSetup, -) -> None: - search_result = await mcp_client.call_tool( - 'search_semantic_context', - { - 'patterns': [semantic_test_setup.slug], - 'max_results': 20, - }, - ) - search_payload = cast(dict[str, Any], search_result.structured_content)['result'] - search_groups = [ - SemanticSearchModelGroup.model_validate(item) for item in cast(list[dict[str, Any]], search_payload) - ] - - assert len(search_groups) == 1 - assert search_groups[0].semantic_model_id == semantic_test_setup.model_id - match_counts = Counter(match.object_type for match in search_groups[0].matches) - assert match_counts == Counter( - { - SemanticObjectType.SEMANTIC_MODEL: 1, - SemanticObjectType.SEMANTIC_DATASET: 2, - SemanticObjectType.SEMANTIC_METRIC: 1, - SemanticObjectType.SEMANTIC_RELATIONSHIP: 1, - SemanticObjectType.SEMANTIC_CONSTRAINT: 1, - } - ) - - -@pytest.mark.asyncio -async def test_get_semantic_context( - mcp_client: Client, - semantic_test_setup: SemanticTestSetup, -) -> None: - context_result = await mcp_client.call_tool( - 'get_semantic_context', - { - 'semantic_objects': [ - {'object_type': SemanticObjectType.SEMANTIC_MODEL.value, 'ids': [semantic_test_setup.model_id]}, - { - 'object_type': SemanticObjectType.SEMANTIC_DATASET.value, - 'ids': [semantic_test_setup.primary_dataset_id, semantic_test_setup.secondary_dataset_id], - }, - {'object_type': SemanticObjectType.SEMANTIC_METRIC.value, 'ids': [semantic_test_setup.metric_id]}, - { - 'object_type': SemanticObjectType.SEMANTIC_RELATIONSHIP.value, - 'ids': [semantic_test_setup.relationship_id], - }, - { - 'object_type': SemanticObjectType.SEMANTIC_CONSTRAINT.value, - 'ids': [semantic_test_setup.constraint_id], - }, - ], - 'semantic_model_ids': [semantic_test_setup.model_id], - }, - ) - context_payload = cast(dict[str, Any], context_result.structured_content)['result'] - contexts = [SemanticObjectTypeContext.model_validate(item) for item in cast(list[dict[str, Any]], context_payload)] - contexts_by_type = {context.object_type: context for context in contexts} - - assert contexts_by_type[SemanticObjectType.SEMANTIC_MODEL].objects[0].id == semantic_test_setup.model_id - dataset_context = contexts_by_type[SemanticObjectType.SEMANTIC_DATASET] - assert {item.id for item in dataset_context.objects} == { - semantic_test_setup.primary_dataset_id, - semantic_test_setup.secondary_dataset_id, - } - assert all(hasattr(item, 'attributes') for item in dataset_context.objects) - - -@pytest.mark.asyncio -async def test_get_semantic_schema( - mcp_client: Client, -) -> None: - schema_result = await mcp_client.call_tool( - 'get_semantic_schema', - { - 'semantic_types': [ - SemanticObjectType.SEMANTIC_DATASET.value, - SemanticObjectType.SEMANTIC_METRIC.value, - ] - }, - ) - schema_payload = cast(dict[str, Any], schema_result.structured_content)['result'] - schemas = [SemanticSchemaDefinition.model_validate(item) for item in cast(list[dict[str, Any]], schema_payload)] - schemas_by_type = {item.semantic_type: item for item in schemas} - - assert set(schemas_by_type) == { - SemanticObjectType.SEMANTIC_DATASET, - SemanticObjectType.SEMANTIC_METRIC, - } - assert isinstance(schemas_by_type[SemanticObjectType.SEMANTIC_DATASET].schema_definition, dict) - assert isinstance(schemas_by_type[SemanticObjectType.SEMANTIC_METRIC].schema_definition, dict) - assert schemas_by_type[SemanticObjectType.SEMANTIC_DATASET].schema_definition - assert schemas_by_type[SemanticObjectType.SEMANTIC_METRIC].schema_definition - - -@pytest.mark.asyncio -async def test_validate_semantic_query( - mcp_client: Client, - semantic_test_setup: SemanticTestSetup, -) -> None: - validate_result = await mcp_client.call_tool( - 'validate_semantic_query', - { - 'sql_query': f'SELECT SUM(item_count) AS total_items FROM {semantic_test_setup.primary_fqn}', - 'semantic_model_ids': [semantic_test_setup.model_id], - 'expected_semantic_objects': [ - { - 'object_type': SemanticObjectType.SEMANTIC_DATASET.value, - 'ids': [semantic_test_setup.primary_dataset_id], - }, - { - 'object_type': SemanticObjectType.SEMANTIC_METRIC.value, - 'ids': [semantic_test_setup.metric_id], - }, - ], - }, - ) - validation = ValidateSemanticQueryOutput.model_validate(validate_result.structured_content) - - assert validation.validation_auto_detected.valid is True - assert len(validation.validation_auto_detected.semantic_models) == 1 - assert validation.validation_auto_detected.semantic_models[0].id == semantic_test_setup.model_id - assert validation.validation_auto_detected.semantic_models[0].name == semantic_test_setup.model_name - assert validation.validation_detected_from_expected is not None - assert validation.validation_detected_from_expected.valid is True - assert {(item.object_type, item.id) for item in validation.matched_expected_objects} == { - (SemanticObjectType.SEMANTIC_DATASET, semantic_test_setup.primary_dataset_id), - (SemanticObjectType.SEMANTIC_METRIC, semantic_test_setup.metric_id), - } - assert validation.missing_expected_objects == [] - assert validation.unexpected_detected_objects == [] - assert [dataset.id for dataset in validation.validation_auto_detected.used_datasets] == [ - semantic_test_setup.primary_dataset_id - ] - assert [metric.id for metric in validation.validation_auto_detected.used_metrics] == [semantic_test_setup.metric_id] - assert validation.validation_auto_detected.matched_relationships == [] - assert validation.validation_auto_detected.violations == [] - assert validation.validation_auto_detected.post_execution_checks == [] diff --git a/integtests/tools/sql.test.ts b/integtests/tools/sql.test.ts new file mode 100644 index 000000000..059eded33 --- /dev/null +++ b/integtests/tools/sql.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolRaw, callToolText, connectMcp } from '../helpers/mcp'; +import { seedProject } from '../helpers/seed'; +import { getTestProjectForTest } from '../testproject/fixture'; +import type { Backend } from '../testproject/types'; + +// Ported from integtests/tools/test_sql.py: a literal SELECT, the invalid-query error path, +// and the faithful seeded COUNT(*) case (get_buckets -> get_tables -> query_data against the +// table's fully-qualified name). +describe('query_data (integration)', () => { + it.each(['snowflake', 'bigquery'])( + 'counts rows of a seeded table via its fully-qualified name (%s)', + async (backend) => { + // get_tables now resolves the FQN dialect-aware, so the seeded COUNT works on both + // Snowflake (double-quoted db.schema.table) and BigQuery (backtick dataset.table). + const project = await getTestProjectForTest({ backend }); + await seedProject(project); + const session = await connectMcp(project.config); + try { + // Resolve the seeded table's FQN: list a bucket, then fetch table detail. + const tablesListing = await callToolText(session.client, 'get_tables', { + bucket_ids: ['in.c-test_bucket_01'], + }); + expect(tablesListing).toContain('in.c-test_bucket_01.test_table_01'); + + const detail = await callToolText(session.client, 'get_tables', { + table_ids: ['in.c-test_bucket_01.test_table_01'], + }); + // Pull the fully-qualified name out of the detail. The Snowflake FQN ("DB"."SCHEMA"."TBL") + // contains quotes/dots, so TOON emits it as a double-quoted (JSON-escaped) scalar. Capture + // the whole value on the line and JSON-decode it when quoted — a naive non-greedy match + // would truncate at the first inner quote and produce invalid SQL. + const fqnMatch = detail.match(/fullyQualifiedName:\s*(.+?)\s*$/m); + expect(fqnMatch, `Table detail should expose a fullyQualifiedName. Got: ${detail}`).not.toBeNull(); + let fqn = fqnMatch![1]!.trim(); + if (fqn.startsWith('"') && fqn.endsWith('"')) fqn = JSON.parse(fqn) as string; + expect(fqn.length).toBeGreaterThan(0); + + const text = await callToolText(session.client, 'query_data', { + sql_query: `SELECT COUNT(*) as row_count FROM ${fqn}`, + query_name: 'Row Count Query', + }); + expect(text).toContain('Row Count Query'); + // CSV must have a header (ROW_COUNT on Snowflake / row_count on BigQuery) plus a numeric row. + expect(text).toMatch(/row_count/i); + expect(text).toMatch(/\b\d+\b/); + } finally { + await session.close(); + } + }, + ); + + it('runs a literal query and returns CSV data', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const text = await callToolText(session.client, 'query_data', { + sql_query: 'SELECT 1 AS one', + query_name: 'Smoke Query', + }); + expect(text).toContain('Smoke Query'); + expect(text).toMatch(/\b1\b/); + } finally { + await session.close(); + } + }); + + it('reports an error for invalid SQL', async () => { + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const result = await callToolRaw(session.client, 'query_data', { + sql_query: 'INVALID SQL SYNTAX SELECT * FROM', + query_name: 'Invalid Query Test', + }); + expect(result.isError).toBeTruthy(); + expect((result.content as { text: string }[])[0]!.text).toMatch(/Failed to run SQL query/i); + } finally { + await session.close(); + } + }); +}); diff --git a/integtests/tools/storage.test.ts b/integtests/tools/storage.test.ts new file mode 100644 index 000000000..845079748 --- /dev/null +++ b/integtests/tools/storage.test.ts @@ -0,0 +1,319 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolRaw, callToolText, connectMcp } from '../helpers/mcp'; +import { seedProject } from '../helpers/seed'; +import { getTestProjectForTest } from '../testproject/fixture'; +import type { Backend } from '../testproject/types'; + +// Ported from integtests/tools/test_storage.py. Each test leases a fresh project, resets it, +// seeds the standard fixtures (2 input buckets, 1 CSV table with columns id/name/item_count, +// 2 component configs), then exercises the storage tools. +// +// The MCP server returns tool output as TOON text (token-oriented notation), so the assertions +// match substrings / regex / the TOON tabular header rather than reconstructing pydantic models +// the way the Python tests did against the in-process tool functions. +const SEEDED_BUCKET_IDS = ['in.c-test_bucket_01', 'in.c-test_bucket_02']; +const SEEDED_TABLE_ID = 'in.c-test_bucket_01.test_table_01'; +const SEEDED_COLUMNS = ['id', 'name', 'item_count']; + +describe('storage tools (integration)', () => { + // --- get_buckets --------------------------------------------------------- + + it('get_buckets lists the seeded buckets', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_buckets'); + for (const id of SEEDED_BUCKET_IDS) expect(text).toContain(id); + } finally { + await session.close(); + } + }); + + it('get_buckets reports bucket counts by stage (port of test_get_buckets)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_buckets'); + // bucket_counts: both seeded buckets are stage 'in' → total=2, input=2, output=0. + // TOON renders the nested object as indented `key: value` lines. + expect(text).toMatch(/total_buckets:\s*2/); + expect(text).toMatch(/input_buckets:\s*2/); + expect(text).toMatch(/output_buckets:\s*0/); + // Every listed bucket carries an explicit stage column. + expect(text).toContain('stage'); + } finally { + await session.close(); + } + }); + + it('get_buckets returns full detail for specific bucket ids (port of test_get_bucket)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + for (const bucketId of SEEDED_BUCKET_IDS) { + const text = await callToolText(session.client, 'get_buckets', { bucket_ids: [bucketId] }); + expect(text).toContain(bucketId); + } + } finally { + await session.close(); + } + }); + + it('get_buckets emits TOON tabular output (port of test_get_buckets_output_format)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_buckets'); + // Two buckets are presented in TOON's list-of-objects tabular form: + // buckets[2]{id,name,displayName,stage,...}: + expect(text).toMatch(/^buckets\[2\]\{[^}]*\bid\b[^}]*\}:/m); + } finally { + await session.close(); + } + }); + + // --- get_tables ---------------------------------------------------------- + + // FQN + warehouse-native types are now resolved dialect-aware (Snowflake double-quoted + // 3-part db.schema.table; BigQuery backtick-quoted 2-part dataset.table). Both backends + // must expose a fully_qualified_name for the seeded table. + const FQN_QUOTE: Record = { snowflake: '"', bigquery: '`' }; + + it.each(['snowflake', 'bigquery'])( + 'get_tables lists a bucket and returns table detail with an FQN (%s)', + async (backend) => { + const project = await getTestProjectForTest({ backend }); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const listed = await callToolText(session.client, 'get_tables', { + bucket_ids: ['in.c-test_bucket_01'], + }); + expect(listed).toContain(SEEDED_TABLE_ID); + + const detail = await callToolText(session.client, 'get_tables', { + table_ids: [SEEDED_TABLE_ID], + }); + // The FQN must be present and use the backend's quote char. + const fqnMatch = detail.match(/fullyQualifiedName:\s*(.+?)\s*$/m); + expect( + fqnMatch, + `Table detail should expose a fullyQualifiedName. Got: ${detail}`, + ).not.toBeNull(); + let fqn = fqnMatch![1]!.trim(); + if (fqn.startsWith('"') && fqn.endsWith('"')) fqn = JSON.parse(fqn) as string; + expect(fqn).toContain(FQN_QUOTE[backend]); + expect(detail).toContain('item_count'); + } finally { + await session.close(); + } + }, + ); + + it.each(['snowflake', 'bigquery'])( + 'get_tables detail returns the seeded columns with types (%s) (port of test_get_table)', + async (backend) => { + const project = await getTestProjectForTest({ backend }); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const detail = await callToolText(session.client, 'get_tables', { + table_ids: [SEEDED_TABLE_ID], + }); + expect(detail).toContain(SEEDED_TABLE_ID); + expect(detail).toContain('test_table_01'); + // Every CSV column must appear in the detail's column listing. + for (const col of SEEDED_COLUMNS) expect(detail).toContain(col); + // Detail carries database-native type info per column. + expect(detail).toContain('database_native_type'); + } finally { + await session.close(); + } + }, + ); + + it('get_tables listing returns summaries without FQN/columns (port of test_get_tables)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + // Bucket with the seeded table: exactly one table, summary shape. + const withTable = await callToolText(session.client, 'get_tables', { + bucket_ids: ['in.c-test_bucket_01'], + }); + expect(withTable).toContain(SEEDED_TABLE_ID); + // Summaries never resolve the warehouse FQN nor expand columns — those fields must be + // absent from the output entirely (not emitted as a misleading null). + expect(withTable).not.toMatch(/fullyQualifiedName|fully_qualified_name/); + expect(withTable).not.toContain('database_native_type'); + + // The second seeded bucket has no tables → empty table list. + const emptyBucket = await callToolText(session.client, 'get_tables', { + bucket_ids: ['in.c-test_bucket_02'], + }); + expect(emptyBucket).not.toContain(SEEDED_TABLE_ID); + } finally { + await session.close(); + } + }); + + it('get_tables emits TOON tabular output (port of test_get_tables_output_format)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'get_tables', { + bucket_ids: ['in.c-test_bucket_01'], + }); + // One table in the bucket → TOON list-of-objects tabular header `tables[1]{...}:`. + expect(text).toMatch(/^tables\[1\]\{[^}]*\bid\b[^}]*\}:/m); + } finally { + await session.close(); + } + }); + + // --- update_descriptions ------------------------------------------------- + + it('update_descriptions updates a bucket description (port of test_update_descriptions_bucket)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const bucketId = SEEDED_BUCKET_IDS[0]!; + const text = await callToolText(session.client, 'update_descriptions', { + updates: [{ item_id: bucketId, description: 'New Description' }], + }); + expect(text).toMatch(/total_processed:\s*1/); + expect(text).toMatch(/successful:\s*1/); + expect(text).toMatch(/failed:\s*0/); + expect(text).toContain(bucketId); + + // Verify the description actually landed: get_buckets detail surfaces it. + const detail = await callToolText(session.client, 'get_buckets', { bucket_ids: [bucketId] }); + expect(detail).toContain('New Description'); + } finally { + await session.close(); + } + }); + + it('update_descriptions updates a table description (port of test_update_descriptions_table)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const text = await callToolText(session.client, 'update_descriptions', { + updates: [{ item_id: SEEDED_TABLE_ID, description: 'New Table Description' }], + }); + expect(text).toMatch(/total_processed:\s*1/); + expect(text).toMatch(/successful:\s*1/); + expect(text).toMatch(/failed:\s*0/); + expect(text).toContain(SEEDED_TABLE_ID); + + const detail = await callToolText(session.client, 'get_tables', { + table_ids: [SEEDED_TABLE_ID], + }); + expect(detail).toContain('New Table Description'); + } finally { + await session.close(); + } + }); + + it('update_descriptions updates a column description (port of test_update_descriptions_table_column)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const columnName = SEEDED_COLUMNS[0]!; + const columnId = `${SEEDED_TABLE_ID}.${columnName}`; + const text = await callToolText(session.client, 'update_descriptions', { + updates: [{ item_id: columnId, description: 'New Table Column Description' }], + }); + expect(text).toMatch(/total_processed:\s*1/); + expect(text).toMatch(/successful:\s*1/); + expect(text).toMatch(/failed:\s*0/); + expect(text).toContain(columnId); + + // The column description must surface in the table detail's column listing. + const detail = await callToolText(session.client, 'get_tables', { + table_ids: [SEEDED_TABLE_ID], + }); + expect(detail).toContain('New Table Column Description'); + } finally { + await session.close(); + } + }); + + it('update_descriptions handles mixed item types in one call (port of test_update_descriptions_mixed_types)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + const bucketId = SEEDED_BUCKET_IDS[0]!; + const columnName = SEEDED_COLUMNS[0]!; + const text = await callToolText(session.client, 'update_descriptions', { + updates: [ + { item_id: bucketId, description: 'Mixed Bucket Description' }, + { item_id: SEEDED_TABLE_ID, description: 'Mixed Table Description' }, + { item_id: `${SEEDED_TABLE_ID}.${columnName}`, description: 'Mixed Column Description' }, + ], + }); + expect(text).toMatch(/total_processed:\s*3/); + expect(text).toMatch(/successful:\s*3/); + expect(text).toMatch(/failed:\s*0/); + + const bucketDetail = await callToolText(session.client, 'get_buckets', { + bucket_ids: [bucketId], + }); + expect(bucketDetail).toContain('Mixed Bucket Description'); + + const tableDetail = await callToolText(session.client, 'get_tables', { + table_ids: [SEEDED_TABLE_ID], + }); + expect(tableDetail).toContain('Mixed Table Description'); + expect(tableDetail).toContain('Mixed Column Description'); + } finally { + await session.close(); + } + }); + + it('update_descriptions reports failure for an invalid item id (port of test_update_descriptions_invalid_path)', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + // The tool itself succeeds (no isError); the per-item result records the failure. + const text = await callToolText(session.client, 'update_descriptions', { + updates: [{ item_id: 'invalid-path', description: 'This should fail' }], + }); + expect(text).toMatch(/total_processed:\s*1/); + expect(text).toMatch(/successful:\s*0/); + expect(text).toMatch(/failed:\s*1/); + expect(text).toContain('invalid-path'); + expect(text).toContain('Invalid item_id format'); + } finally { + await session.close(); + } + }); + + it('get_tables flags missing tables rather than erroring', async () => { + const project = await getTestProjectForTest(); + await seedProject(project); + const session = await connectMcp(project.config); + try { + // A non-existent table id is reported via tables_not_found, not a tool error. + const result = await callToolRaw(session.client, 'get_tables', { + table_ids: ['in.c-test_bucket_01.does_not_exist'], + }); + expect(result.isError).toBeFalsy(); + const text = (result.content as { text: string }[])[0]!.text; + expect(text).toContain('does_not_exist'); + } finally { + await session.close(); + } + }); +}); diff --git a/integtests/tools/storage_branches.test.ts b/integtests/tools/storage_branches.test.ts new file mode 100644 index 000000000..3e7c9e85f --- /dev/null +++ b/integtests/tools/storage_branches.test.ts @@ -0,0 +1,365 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +import { callToolText, connectMcp, type McpSession } from '../helpers/mcp'; + +import { Config } from '@/config'; + +// Ported from integtests/tools/test_storage_branches.py — validates the storage-branches +// deference mechanism (a dev-branch MCP context sees production buckets/tables deferred into the +// branch, plus the branch's own new objects, and gets a branch-scoped workspace for SQL). +// +// These tests require a project WITH the `storage-branches` feature, which the regular local pool +// projects do NOT have (CI uses a dedicated project — 3055 — supplied out-of-band). The project is +// addressed directly by env vars, mirroring the Python fixtures: +// - INTEGTEST_POOL_STORAGE_API_URL → storage API base URL +// - INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES → token for the storage-branches project +// +// When either is absent the whole suite is skipped (see SKIP_REASON). The setup faithfully mirrors +// the Python harness: it provisions production data idempotently, creates two dev branches, and +// runs Python transformations in each branch so they run unchanged once such a project is wired up. + +const STORAGE_BRANCHES_TOKEN_ENV_VAR = 'INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES'; +const POOL_STORAGE_API_URL_ENV_VAR = 'INTEGTEST_POOL_STORAGE_API_URL'; +const PYTHON_TRANSFORMATION_COMPONENT = 'keboola.python-transformation-v2'; + +const token = (process.env[STORAGE_BRANCHES_TOKEN_ENV_VAR] ?? '').trim(); +const storageApiUrl = (process.env[POOL_STORAGE_API_URL_ENV_VAR] ?? '').trim(); + +// Gate the suite: without the dedicated project's token + URL there is nothing to run against. +// The dedicated storage-branches project is 3055 in CI, not necessarily in the local pool. +const SKIP_REASON = !token + ? `${STORAGE_BRANCHES_TOKEN_ENV_VAR} not set (storage-branches project unavailable in this pool)` + : !storageApiUrl + ? `${POOL_STORAGE_API_URL_ENV_VAR} not set (storage-branches project URL unavailable)` + : null; + +// --- HTTP helpers (ports of the Python _api_request / job-wait helpers) ---------------------- + +const apiRequest = async ( + method: string, + url: string, + init: { json?: unknown; form?: Record } = {}, +): Promise> => { + const headers: Record = { 'X-StorageApi-Token': token }; + let body: string | URLSearchParams | undefined; + if (init.json !== undefined) { + headers['Content-Type'] = 'application/json'; + body = JSON.stringify(init.json); + } else if (init.form !== undefined) { + body = new URLSearchParams(init.form); + } + const resp = await fetch(url, { method, headers, body }); + const text = await resp.text(); + if (!resp.ok) throw new Error(`${method} ${url} failed: ${resp.status} ${text}`); + return text ? (JSON.parse(text) as Record) : {}; +}; + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +const waitForStorageJob = async (jobId: string, timeoutMs = 120_000): Promise => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const job = await apiRequest('GET', `${storageApiUrl}/v2/storage/jobs/${jobId}`); + const status = job.status; + if (status === 'success') return; + if (status === 'error' || status === 'cancelled') { + throw new Error(`Storage job ${jobId} failed: ${JSON.stringify(job)}`); + } + await sleep(2_000); + } + throw new Error(`Storage job ${jobId} did not complete within ${timeoutMs}ms`); +}; + +const waitForQueueJob = async (jobId: string, timeoutMs = 300_000): Promise => { + const queueUrl = storageApiUrl.replace('connection.', 'queue.'); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const job = await apiRequest('GET', `${queueUrl}/jobs/${jobId}`); + const status = job.status; + if (status === 'success') return; + if (status === 'error' || status === 'cancelled' || status === 'terminated') { + throw new Error(`Queue job ${jobId} failed with status=${String(status)}`); + } + await sleep(5_000); + } + throw new Error(`Queue job ${jobId} did not complete within ${timeoutMs}ms`); +}; + +const createBranch = async (name: string): Promise => { + const job = await apiRequest('POST', `${storageApiUrl}/v2/storage/dev-branches`, { json: { name } }); + const jobId = String(job.id); + await waitForStorageJob(jobId); + const done = await apiRequest('GET', `${storageApiUrl}/v2/storage/jobs/${jobId}`); + return String((done.results as Record).id); +}; + +const deleteBranch = async (branchId: string): Promise => { + try { + const job = await apiRequest('DELETE', `${storageApiUrl}/v2/storage/dev-branches/${branchId}`); + await waitForStorageJob(String(job.id)); + } catch { + // Best-effort cleanup, matching the Python teardown. + } +}; + +const ensureBucket = async (name: string, stage = 'in'): Promise => { + const bucketId = `${stage}.c-${name}`; + const resp = await fetch(`${storageApiUrl}/v2/storage/buckets/${bucketId}`, { + headers: { 'X-StorageApi-Token': token }, + }); + if (resp.ok) return bucketId; + if (resp.status !== 404) throw new Error(`GET bucket ${bucketId} failed: ${resp.status}`); + const created = await apiRequest('POST', `${storageApiUrl}/v2/storage/buckets`, { + json: { name, stage, description: 'Integration test bucket' }, + }); + return String(created.id); +}; + +const ensureTable = async (bucketId: string, tableName: string, csvData: string): Promise => { + const tableId = `${bucketId}.${tableName}`; + const resp = await fetch(`${storageApiUrl}/v2/storage/tables/${tableId}`, { + headers: { 'X-StorageApi-Token': token }, + }); + if (resp.ok) return tableId; + if (resp.status !== 404) throw new Error(`GET table ${tableId} failed: ${resp.status}`); + const created = await apiRequest('POST', `${storageApiUrl}/v2/storage/buckets/${bucketId}/tables`, { + form: { name: tableName, delimiter: ',', dataString: csvData }, + }); + return String(created.id); +}; + +const pythonTransformConfig = ( + destination: string, + csvFilename: string, + fieldnames: string[], + row: Record, +): Record => { + const fieldsStr = JSON.stringify(fieldnames).replace(/"/g, "'"); + const rowStr = JSON.stringify(row).replace(/"/g, "'"); + const script = [ + 'import csv', + 'import os', + "os.makedirs('out/tables', exist_ok=True)", + `with open('out/tables/${csvFilename}', mode='wt', encoding='utf-8') as f:`, + ` writer = csv.DictWriter(f, fieldnames=${fieldsStr}, dialect='kbc')`, + ' writer.writeheader()', + ` writer.writerow(${rowStr})`, + ].join('\n'); + return { + storage: { + output: { + tables: [{ source: csvFilename, destination, primary_key: ['id'] }], + }, + }, + parameters: { + blocks: [{ name: 'Generate data', codes: [{ name: 'script', script: [script] }] }], + packages: [], + }, + }; +}; + +const createConfigInBranch = async ( + branchId: string, + componentId: string, + name: string, + config: Record, +): Promise => { + const created = await apiRequest( + 'POST', + `${storageApiUrl}/v2/storage/branch/${branchId}/components/${componentId}/configs`, + { json: { name, description: `Integration test config: ${name}`, configuration: JSON.stringify(config) } }, + ); + return String(created.id); +}; + +const runJobInBranch = async (branchId: string, componentId: string, configId: string): Promise => { + const queueUrl = storageApiUrl.replace('connection.', 'queue.'); + const job = await apiRequest('POST', `${queueUrl}/jobs`, { + json: { component: componentId, config: configId, mode: 'run', branchId }, + }); + await waitForQueueJob(String(job.id)); +}; + +// --- Suite ----------------------------------------------------------------------------------- + +type BranchProject = { branchAId: string; branchBId: string }; + +const describeBranches = SKIP_REASON ? describe.skip : describe; + +describeBranches('storage-branches tools (integration)', () => { + let project: BranchProject; + // Per-branch Config: production-branch alias when branchId is undefined. + const configFor = (branchId?: string): Config => + new Config({ storageApiUrl, storageToken: token, branchId }); + + beforeAll(async () => { + // Confirm the token's project actually carries the storage-branches feature before paying for + // the (slow) branch + transformation-job setup. + const verify = await apiRequest('GET', `${storageApiUrl}/v2/storage/tokens/verify`); + const owner = (verify.owner ?? {}) as Record; + const features = Array.isArray(owner.features) ? (owner.features as string[]) : []; + if (!features.includes('storage-branches')) { + throw new Error( + `project ${String(owner.name)} must have the storage-branches feature enabled to run these tests`, + ); + } + + // Idempotent production data (shared safely across concurrent sessions). + await ensureBucket('test_bucket_01'); + await ensureTable( + 'in.c-test_bucket_01', + 'test_table_01', + '"id","name","item_count"\n1,"item1",10\n2,"item2",20', + ); + + const uid = Math.random().toString(36).slice(2, 10); + const branchAId = await createBranch(`integtest-branch-A-${uid}`); + const branchBId = await createBranch(`integtest-branch-B-${uid}`); + + // Branch A: update the existing production table (creates a branched version). + let cid = await createConfigInBranch( + branchAId, + PYTHON_TRANSFORMATION_COMPONENT, + 'update-tbl', + pythonTransformConfig('in.c-test_bucket_01.test_table_01', 'test_table_01.csv', ['id', 'name', 'item_count'], { + id: '99', + name: 'branched_item', + item_count: '999', + }), + ); + await runJobInBranch(branchAId, PYTHON_TRANSFORMATION_COMPONENT, cid); + + // Branch A: create a new bucket + table that exists only in the branch. + cid = await createConfigInBranch( + branchAId, + PYTHON_TRANSFORMATION_COMPONENT, + 'create-tbl', + pythonTransformConfig('in.c-test_branch.test_table_branch', 'test_table_branch.csv', ['id', 'name', 'value'], { + id: '1', + name: 'branch_a_data', + value: '100', + }), + ); + await runJobInBranch(branchAId, PYTHON_TRANSFORMATION_COMPONENT, cid); + + // Branch B: create a different branch-only bucket + table. + cid = await createConfigInBranch( + branchBId, + PYTHON_TRANSFORMATION_COMPONENT, + 'create-b-tbl', + pythonTransformConfig('in.c-test_branch_2.test_table_branch', 'test_table_branch.csv', ['id', 'name', 'value'], { + id: '1', + name: 'branch_b_data', + value: '200', + }), + ); + await runJobInBranch(branchBId, PYTHON_TRANSFORMATION_COMPONENT, cid); + + project = { branchAId, branchBId }; + }, 600_000); + + afterAll(async () => { + if (project) { + await deleteBranch(project.branchAId); + await deleteBranch(project.branchBId); + } + }); + + const withSession = async (config: Config, fn: (session: McpSession) => Promise): Promise => { + const session = await connectMcp(config); + try { + await fn(session); + } finally { + await session.close(); + } + }; + + it('get_buckets from Branch A includes production + Branch A buckets, not Branch B (test_list_buckets_includes_branch_a_bucket)', async () => { + await withSession(configFor(project.branchAId), async (session) => { + const text = await callToolText(session.client, 'get_buckets'); + expect(text).toContain('in.c-test_bucket_01'); + expect(text).toContain('in.c-test_branch'); + expect(text).not.toContain('in.c-test_branch_2'); + }); + }); + + it('get_tables lists a branch-only bucket with a production-like id (test_list_tables_in_branched_bucket)', async () => { + await withSession(configFor(project.branchAId), async (session) => { + const text = await callToolText(session.client, 'get_tables', { bucket_ids: ['in.c-test_branch'] }); + expect(text).toContain('test_table_branch'); + expect(text).toContain('in.c-test_branch.test_table_branch'); + // branch_id is internal-only and must not leak into the deferred (production-like) output. + expect(text).not.toMatch(/branch_id/); + }); + }); + + it('get_tables for the production bucket sees the branched table (test_deference_branched_table)', async () => { + await withSession(configFor(project.branchAId), async (session) => { + const text = await callToolText(session.client, 'get_tables', { bucket_ids: ['in.c-test_bucket_01'] }); + expect(text).toContain('in.c-test_bucket_01.test_table_01'); + expect(text).not.toMatch(/branch_id/); + }); + }); + + it('get_project_info from a dev branch reports is_development_branch=true (test_get_project_info_reports_dev_branch)', async () => { + await withSession(configFor(project.branchAId), async (session) => { + const text = await callToolText(session.client, 'get_project_info'); + expect(text).toContain(String(project.branchAId)); + expect(text).toMatch(/is_development_branch:\s*true/); + expect(text).toMatch(/branch_name:/); + }); + }); + + it('get_project_info from the default branch reports is_development_branch=false (test_get_project_info_reports_default_branch)', async () => { + await withSession(configFor(undefined), async (session) => { + const text = await callToolText(session.client, 'get_project_info'); + expect(text).toMatch(/is_development_branch:\s*false/); + expect(text).toMatch(/branch_name:/); + // The default branch must not be either dev branch created for this session. + expect(text).not.toContain(String(project.branchAId)); + expect(text).not.toContain(String(project.branchBId)); + }); + }); + + it.each([ + ['in.c-test_branch.test_table_branch', 'branch-only table created via transformation'], + ['in.c-test_bucket_01.test_table_01', 'production table (also branched in Branch A)'], + ])( + 'query_data from a dev branch reaches %s (test_query_data_from_dev_branch_reaches_both_kinds_of_tables)', + async (tableId, description) => { + await withSession(configFor(project.branchAId), async (session) => { + const tablesListing = await callToolText(session.client, 'get_tables', { table_ids: [tableId] }); + // Pull the FQN out of the table detail; without it the table is not queryable. + const fqnMatch = tablesListing.match(/(?:fully_qualified_name|fullyQualifiedName):\s*(\S+)/); + expect(fqnMatch, `${description}: table ${tableId} has no FQN, cannot query`).not.toBeNull(); + const fqn = fqnMatch![1]!; + + const text = await callToolText(session.client, 'query_data', { + sql_query: `SELECT COUNT(*) AS row_count FROM ${fqn}`, + query_name: `Row count for ${tableId}`, + }); + // csv_data holds a header row + a single COUNT(*) data row (a positive integer). + expect(text).toContain('csv_data'); + const countMatch = text.match(/row_count[^\d]*(\d+)/i); + expect(countMatch, `${description}: expected a numeric row count`).not.toBeNull(); + expect(Number(countMatch![1])).toBeGreaterThanOrEqual(1); + }); + }, + ); + + it('dev-branch and default-branch contexts get different workspaces (test_workspace_id_is_branch_aware)', async () => { + let devWorkspace = ''; + let defaultWorkspace = ''; + await withSession(configFor(project.branchAId), async (session) => { + const text = await callToolText(session.client, 'get_project_info'); + devWorkspace = text.match(/workspace_id:\s*(\d+)/)?.[1] ?? ''; + }); + await withSession(configFor(undefined), async (session) => { + const text = await callToolText(session.client, 'get_project_info'); + defaultWorkspace = text.match(/workspace_id:\s*(\d+)/)?.[1] ?? ''; + }); + expect(Number(devWorkspace)).toBeGreaterThan(0); + expect(Number(defaultWorkspace)).toBeGreaterThan(0); + expect(devWorkspace).not.toBe(defaultWorkspace); + }); +}); diff --git a/integtests/tools/test_data_apps.py b/integtests/tools/test_data_apps.py deleted file mode 100644 index 1a9067ec2..000000000 --- a/integtests/tools/test_data_apps.py +++ /dev/null @@ -1,509 +0,0 @@ -import logging -import os -import subprocess -import uuid -from pathlib import Path -from typing import Any, AsyncGenerator, Mapping, cast - -import pytest -import pytest_asyncio -import toon_format -from fastmcp import Client, FastMCP - -from keboola_mcp_server.clients.client import DATA_APP_COMPONENT_ID, KeboolaClient, get_metadata_property -from keboola_mcp_server.config import Config, MetadataField, ServerRuntimeInfo -from keboola_mcp_server.server import create_server -from keboola_mcp_server.tools.data_apps import ( - _DEFAULT_PACKAGES, - DataApp, - DataAppSummary, - GetDataAppsOutput, - ModifiedDataAppOutput, - ModifiedPythonJsDataAppOutput, - _get_query_function_code, -) -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - - -@pytest.fixture -def streamlit_app_imports() -> str: - return 'import streamlit as st\n\n' - - -@pytest.fixture -def streamlit_app_entrypoint() -> str: - return ( - 'def main():\n' - " st.title('Integration Test Data App')\n" - " st.write('Hello from integration test')\n" - ' # Optionally query data (kept commented to avoid side-effects during tests)\n' - " # df = query_data('select 1 as col')\n" - ' # st.dataframe(df)\n\n' - 'if __name__ == "__main__":\n' - ' main()\n' - ) - - -@pytest.fixture -def sample_streamlit_app(streamlit_app_imports: str, streamlit_app_entrypoint: str) -> str: - """Return a minimal Streamlit app template that supports query injection.""" - return f'{streamlit_app_imports}' '{QUERY_DATA_FUNCTION}\n\n' f'{streamlit_app_entrypoint}' - - -@pytest.fixture -def mcp_server(storage_api_url: str, storage_api_token: str, workspace_schema: str) -> FastMCP: - config = Config(storage_api_url=storage_api_url, storage_token=storage_api_token, workspace_schema=workspace_schema) - mcp_server = create_server(config, runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(mcp_server, FastMCP) - return mcp_server - - -@pytest_asyncio.fixture -async def mcp_client(mcp_server: FastMCP) -> AsyncGenerator[Client, None]: - async with Client(mcp_server) as client: - yield client - - -@pytest.fixture -def app_name() -> str: - unique_suffix = uuid.uuid4().hex[:8] - return f'Integration Test Data App {unique_suffix}' - - -@pytest.fixture -def app_description() -> str: - return 'Data app created by integration test' - - -@pytest_asyncio.fixture -async def initial_data_app( - mcp_client: Client, - keboola_client: KeboolaClient, - app_name: str, - app_description: str, - sample_streamlit_app: str, -) -> AsyncGenerator[ModifiedDataAppOutput, None]: - sync_output: ModifiedDataAppOutput | None = None - try: - # Create - created_result = await mcp_client.call_tool( - name='modify_streamlit_data_app', - arguments={ - 'name': app_name, - 'description': app_description, - 'source_code': sample_streamlit_app, - 'packages': ['numpy', 'streamlit'], - 'authentication_type': 'no-auth', - }, - ) - assert created_result.structured_content is not None - sync_output = ModifiedDataAppOutput.model_validate(created_result.structured_content) - yield sync_output - finally: - if sync_output: - try: - # Delete the data app from the data science API and the configuration from the storage API as well. - await keboola_client.data_science_client.delete_data_app(sync_output.data_app.data_app_id) - except Exception as e: - LOG.error(f'Error deleting data app: {e}') - else: - LOG.error('No data app to delete') - - -@pytest.mark.asyncio -async def test_get_data_apps_listing(mcp_client: Client, initial_data_app: ModifiedDataAppOutput) -> None: - """Test listing data apps returns valid TOON formatted output.""" - tool_result = await mcp_client.call_tool(name='get_data_apps', arguments={}) - - # Verify structured content - assert tool_result.structured_content is not None - apps = GetDataAppsOutput.model_validate(tool_result.structured_content) - assert len(apps.data_apps) > 0 - - # Verify TOON formatted text content matches structured content - assert len(tool_result.content) == 1 - assert tool_result.content[0].type == 'text' - toon_decoded = GetDataAppsOutput.model_validate(toon_format.decode(tool_result.content[0].text)) - assert toon_decoded == apps - - -@pytest.mark.asyncio -async def test_data_app_lifecycle( - mcp_client: Client, - keboola_client: KeboolaClient, - workspace_manager: WorkspaceManager, - app_name: str, - app_description: str, - initial_data_app: ModifiedDataAppOutput, - streamlit_app_imports: str, - streamlit_app_entrypoint: str, -) -> None: - """ - End-to-end lifecycle for data apps: - Starts with a created app. - - get details and list of created app - - update app - - get details and list of updated app - Always deletes the data app in teardown. - """ - - # Check created app basic details - assert initial_data_app.response == 'created' - data_app_id = initial_data_app.data_app.data_app_id - configuration_id = initial_data_app.data_app.configuration_id - assert data_app_id - assert configuration_id - - # Verify the metadata - check that KBC.MCP.createdBy is set to 'true' - metadata = await keboola_client.storage_client.configuration_metadata_get( - component_id=DATA_APP_COMPONENT_ID, configuration_id=configuration_id - ) - assert isinstance(metadata, list) - metadata_dict = {item['key']: item['value'] for item in metadata if isinstance(item, dict)} - assert MetadataField.CREATED_BY_MCP in metadata_dict - assert metadata_dict[MetadataField.CREATED_BY_MCP] == 'true' - - # Check created app details by configuration_id - details_result = await mcp_client.call_tool( - name='get_data_apps', arguments={'configuration_ids': [configuration_id]} - ) - assert details_result.structured_content is not None - details = GetDataAppsOutput.model_validate(details_result.structured_content) - assert len(details.data_apps) == 1 - data_app_details = details.data_apps[0] - assert isinstance(data_app_details, DataApp) - - assert data_app_details.configuration_id == configuration_id - assert data_app_details.data_app_id == data_app_id - assert data_app_details.name == app_name - assert data_app_details.description == app_description - # Check code and code injection - data_app_details_parameters = data_app_details.configuration.get('parameters') or {} - assert streamlit_app_imports in data_app_details_parameters['script'][0] - assert streamlit_app_entrypoint in data_app_details_parameters['script'][0] - sql_dialect = await workspace_manager.get_sql_dialect() - assert _get_query_function_code(sql_dialect) in data_app_details_parameters['script'][0] - # Check packages - assert set(data_app_details_parameters['packages']) == set(['numpy', 'streamlit'] + _DEFAULT_PACKAGES) - - # Check listing contains our app - # TODO(REMOVE): Set the limit back to the default value once DSAPI is fixed. The limit is temporarily increased to - # 500 to prevent listing only the leftover data apps from previous tests (100). These apps cannot be deleted - # because their configurations were removed in SAPI first, causing the DSAPI delete endpoint to return a 500 error - # afterward. - listed_result = await mcp_client.call_tool(name='get_data_apps', arguments={'limit': 500}) - assert listed_result.structured_content is not None - listed = GetDataAppsOutput.model_validate(listed_result.structured_content) - assert len(listed.data_apps) > 0 - assert all(isinstance(app, DataAppSummary) for app in listed.data_apps) - assert configuration_id in [a.configuration_id for a in listed.data_apps] - # TODO(REMOVE): Remove this assertion once DSAPI is fixed. This only checks that we do not leave any data apps - # in the CI project after test executions except those which are already there and cannot be deleted. - assert len(listed.data_apps) < 110 - - # Update app - updated_name = f'{app_name} - Updated' - updated_description = 'Data app updated by integration test' - updated_source_code = 'import numpy as np\n\n' - updated_result = await mcp_client.call_tool( - name='modify_streamlit_data_app', - arguments={ - 'name': updated_name, - 'description': updated_description, - 'source_code': updated_source_code, - 'packages': ['streamlit'], - 'authentication_type': 'no-auth', - 'configuration_id': configuration_id, - 'change_description': 'Update Code', - }, - ) - # Check updated app basic details - assert updated_result.structured_content is not None - updated = ModifiedDataAppOutput.model_validate(updated_result.structured_content) - assert updated.response == 'updated' - assert updated.data_app.data_app_id == data_app_id - assert updated.data_app.configuration_id == configuration_id - - # Check that KBC.MCP.updatedBy.version.{version} is set to 'true' - metadata = cast( - list[Mapping[str, Any]], - await keboola_client.storage_client.configuration_metadata_get( - component_id=DATA_APP_COMPONENT_ID, configuration_id=configuration_id - ), - ) - meta_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{updated.data_app.config_version}' - meta_value = get_metadata_property(metadata, meta_key) - assert meta_value == 'true' - # Check that the original creation metadata is still there - assert get_metadata_property(metadata, MetadataField.CREATED_BY_MCP) == 'true' - - # Check updated app details by configuration_id - fetched_app = await mcp_client.call_tool(name='get_data_apps', arguments={'configuration_ids': [configuration_id]}) - assert fetched_app.structured_content is not None - fetched = GetDataAppsOutput.model_validate(fetched_app.structured_content) - assert len(fetched.data_apps) == 1 - assert isinstance(fetched.data_apps[0], DataApp) - assert fetched.data_apps[0].name == updated_name - assert fetched.data_apps[0].description == updated_description - # Check that the source code is updated - fetched_data_app_parameters = fetched.data_apps[0].configuration.get('parameters') or {} - assert _get_query_function_code(sql_dialect) in fetched_data_app_parameters['script'][0] - assert updated_source_code in fetched_data_app_parameters['script'][0] - assert streamlit_app_imports not in fetched_data_app_parameters['script'][0] - assert streamlit_app_entrypoint not in fetched_data_app_parameters['script'][0] - # Check that the packages are updated - assert set(fetched_data_app_parameters['packages']) == set(['streamlit'] + _DEFAULT_PACKAGES) - - -# ===== python-js data app: prod + external-git draft (AI-3005 / AI-3286) ===== - - -@pytest.fixture -def python_js_app_py() -> str: - """Minimal python-js entrypoint: a tiny HTTP server returning a fixed string.""" - return ( - 'from http.server import BaseHTTPRequestHandler, HTTPServer\n' - 'import os\n\n' - 'class H(BaseHTTPRequestHandler):\n' - ' def do_GET(self):\n' - ' self.send_response(200)\n' - ' self.end_headers()\n' - " self.wfile.write(b'integration-test-ok')\n\n" - "if __name__ == '__main__':\n" - " port = int(os.environ.get('PORT', '8000'))\n" - " HTTPServer(('0.0.0.0', port), H).serve_forever()\n" - ) - - -def _git(*args: str, cwd: Path) -> None: - """Run a git subcommand inside `cwd`, failing loudly on non-zero exit.""" - subprocess.run(['git', *args], cwd=cwd, check=True, capture_output=True, text=True) - - -@pytest.mark.asyncio -async def test_python_js_data_app_prod_and_draft_lifecycle( - mcp_client: Client, - keboola_client: KeboolaClient, - tmp_path: Path, - python_js_app_py: str, -) -> None: - """End-to-end on canary-orion: create prod (managed repo), create draft - (external-git pointing at prod's repo), push branch, deploy draft in mode='dev', - merge into main, redeploy prod, then delete the draft via the new MCP tool. - - Also asserts that fetching the prod's detail surfaces the draft in `drafts: [...]` - before deletion and returns an empty `drafts` array after. - """ - - unique = uuid.uuid4().hex[:8] - prod_slug = f'int-prod-{unique}' - draft_slug = f'int-draft-{unique}' - prod_output: ModifiedPythonJsDataAppOutput | None = None - draft_output: ModifiedPythonJsDataAppOutput | None = None - draft_deleted_via_tool = False - - try: - # Step 1: create prod (managed repo). - prod_result = await mcp_client.call_tool( - name='modify_python_js_data_app', - arguments={ - 'name': f'Integration prod {unique}', - 'description': 'AI-3286 prod app integration test', - 'slug': prod_slug, - 'authentication_type': 'no-auth', - }, - ) - assert prod_result.structured_content is not None - prod_output = ModifiedPythonJsDataAppOutput.model_validate(prod_result.structured_content) - assert prod_output.response == 'created' - assert prod_output.repo_url is not None - assert prod_output.repo_url.startswith('https://') - assert prod_output.git_clone_url is None - assert prod_output.branch is None - - # Step 2: create draft pointing at prod's repo. Branch defaults to 'init'. - draft_result = await mcp_client.call_tool( - name='modify_python_js_data_app', - arguments={ - 'name': f'Integration draft {unique}', - 'description': 'AI-3286 draft integration test', - 'slug': draft_slug, - 'parent_configuration_id': prod_output.data_app.configuration_id, - 'authentication_type': 'no-auth', - }, - ) - assert draft_result.structured_content is not None - draft_output = ModifiedPythonJsDataAppOutput.model_validate(draft_result.structured_content) - assert draft_output.response == 'created' - assert draft_output.repo_url == prod_output.repo_url - assert draft_output.git_clone_url is not None - assert draft_output.git_clone_url.startswith('https://kai:') - assert draft_output.branch == 'init' - - # Step 3: clone via the embedded credential. Initialize main if the freshly provisioned - # repo is empty, then branch off and push the draft branch. - repo_dir = tmp_path / 'repo' - env = {**os.environ, 'GIT_TERMINAL_PROMPT': '0'} - subprocess.run( - ['git', 'clone', draft_output.git_clone_url, str(repo_dir)], - check=True, - capture_output=True, - text=True, - env=env, - ) - _git('config', 'user.email', 'mcp-integration@keboola.com', cwd=repo_dir) - _git('config', 'user.name', 'MCP Integration Test', cwd=repo_dir) - # The platform-provisioned repo may be empty. Make sure `main` exists with at least one - # commit so the post-merge push has somewhere to land. - has_main = ( - subprocess.run( - ['git', 'rev-parse', '--verify', 'refs/heads/main'], - cwd=repo_dir, - capture_output=True, - text=True, - ).returncode - == 0 - ) - if not has_main: - _git('checkout', '-b', 'main', cwd=repo_dir) - (repo_dir / 'README.md').write_text(f'# integration test {unique}\n') - _git('add', 'README.md', cwd=repo_dir) - _git('commit', '-m', 'init main', cwd=repo_dir) - subprocess.run( - ['git', 'push', '-u', 'origin', 'main'], - cwd=repo_dir, - check=True, - capture_output=True, - text=True, - env=env, - ) - _git('checkout', '-b', draft_output.branch, 'main', cwd=repo_dir) - (repo_dir / 'app.py').write_text(python_js_app_py) - _git('add', 'app.py', cwd=repo_dir) - _git('commit', '-m', f'AI-3286 integration test commit {unique}', cwd=repo_dir) - subprocess.run( - ['git', 'push', '-u', 'origin', draft_output.branch], - cwd=repo_dir, - check=True, - capture_output=True, - text=True, - env=env, - ) - - # Step 4: deploy draft in mode='dev'. - draft_deploy = await mcp_client.call_tool( - name='deploy_data_app', - arguments={ - 'action': 'deploy', - 'configuration_id': draft_output.data_app.configuration_id, - 'mode': 'dev', - }, - ) - assert draft_deploy.structured_content is not None - # The data-app runtime is async — we only assert the deploy call was accepted; not its - # eventual state, since CI cannot afford to poll the full startup loop. - - # Fetching the prod's detail must now list the draft under `drafts`. - prod_detail_before = await mcp_client.call_tool( - name='get_data_apps', - arguments={'configuration_ids': [prod_output.data_app.configuration_id]}, - ) - assert prod_detail_before.structured_content is not None - prod_apps_before = GetDataAppsOutput.model_validate(prod_detail_before.structured_content) - assert len(prod_apps_before.data_apps) == 1 - prod_app_before = prod_apps_before.data_apps[0] - assert isinstance(prod_app_before, DataApp) - draft_cfg_ids_before = [d.configuration_id for d in prod_app_before.drafts] - assert draft_output.data_app.configuration_id in draft_cfg_ids_before - - # Confirm the draft's stored config carries the external-git block we sent and the - # parent linkage we expect. - draft_detail = await mcp_client.call_tool( - name='get_data_apps', - arguments={'configuration_ids': [draft_output.data_app.configuration_id]}, - ) - assert draft_detail.structured_content is not None - draft_apps = GetDataAppsOutput.model_validate(draft_detail.structured_content) - assert len(draft_apps.data_apps) == 1 - draft_detail_app = draft_apps.data_apps[0] - assert isinstance(draft_detail_app, DataApp) - draft_data_app_block = draft_detail_app.configuration.get('parameters', {}).get('dataApp', {}) - draft_git_block = draft_data_app_block.get('git', {}) - assert draft_git_block.get('repository') == prod_output.repo_url - assert draft_git_block.get('branch') == draft_output.branch - assert draft_git_block.get('username') == 'kai' - encrypted_pw = draft_git_block.get('#password', '') - assert encrypted_pw.startswith('KBC::'), f'expected encrypted #password, got {encrypted_pw!r}' - assert draft_data_app_block.get('isDraft') is True - assert draft_data_app_block.get('parentConfigurationId') == prod_output.data_app.configuration_id - # Drafts of a draft are always empty. - assert draft_detail_app.drafts == [] - - # Step 5: merge into main and push. - _git('checkout', 'main', cwd=repo_dir) - _git('merge', '--no-ff', '-m', f'Merge {draft_output.branch}', draft_output.branch, cwd=repo_dir) - subprocess.run( - ['git', 'push', 'origin', 'main'], - cwd=repo_dir, - check=True, - capture_output=True, - text=True, - env=env, - ) - - # Step 6: redeploy prod (no mode, no branch). - prod_deploy = await mcp_client.call_tool( - name='deploy_data_app', - arguments={ - 'action': 'deploy', - 'configuration_id': prod_output.data_app.configuration_id, - }, - ) - assert prod_deploy.structured_content is not None - - # Step 7: delete the draft via the new MCP tool, then verify it's gone from prod's drafts. - # Stop the draft first — DSAPI delete requires desiredState == currentState. - try: - await keboola_client.data_science_client.suspend_data_app(draft_output.data_app.data_app_id) - except Exception as exc: - LOG.info(f'suspend failed for {draft_output.data_app.data_app_id}: {exc}') - delete_result = await mcp_client.call_tool( - name='delete_python_js_data_app_draft', - arguments={'configuration_id': draft_output.data_app.configuration_id}, - ) - assert delete_result.structured_content is not None - deleted = delete_result.structured_content - assert deleted['response'] == 'deleted' - assert deleted['configuration_id'] == draft_output.data_app.configuration_id - assert deleted['parent_configuration_id'] == prod_output.data_app.configuration_id - draft_deleted_via_tool = True - - prod_detail_after = await mcp_client.call_tool( - name='get_data_apps', - arguments={'configuration_ids': [prod_output.data_app.configuration_id]}, - ) - assert prod_detail_after.structured_content is not None - prod_apps_after = GetDataAppsOutput.model_validate(prod_detail_after.structured_content) - prod_app_after = prod_apps_after.data_apps[0] - assert isinstance(prod_app_after, DataApp) - draft_cfg_ids_after = [d.configuration_id for d in prod_app_after.drafts] - assert draft_output.data_app.configuration_id not in draft_cfg_ids_after - - finally: - # Teardown: best-effort cleanup. Skip draft if the test already deleted it via the tool. - cleanup_targets = [(prod_output, 'prod')] - if not draft_deleted_via_tool: - cleanup_targets.insert(0, (draft_output, 'draft')) - for app, role in cleanup_targets: - if app is None: - continue - try: - await keboola_client.data_science_client.suspend_data_app(app.data_app.data_app_id) - except Exception as exc: - LOG.info(f'suspend failed for {role} {app.data_app.data_app_id}: {exc}') - try: - await keboola_client.data_science_client.delete_data_app(app.data_app.data_app_id) - except Exception as exc: - LOG.error(f'delete failed for {role} {app.data_app.data_app_id}: {exc}') diff --git a/integtests/tools/test_doc.py b/integtests/tools/test_doc.py deleted file mode 100644 index 4f931a657..000000000 --- a/integtests/tools/test_doc.py +++ /dev/null @@ -1,18 +0,0 @@ -import httpx -import pytest -from fastmcp import Context - -from keboola_mcp_server.tools.doc import DocsAnswer, docs_query - - -@pytest.mark.asyncio -@pytest.mark.xfail(raises=httpx.ReadTimeout, strict=False, reason='AI service may exceed read timeout in CI') -async def test_docs_query(mcp_context: Context) -> None: - """Tests that `docs_query` returns a valid `DocsAnswer` with text and source URLs.""" - query = 'What is Keboola Connection?' - - result = await docs_query(ctx=mcp_context, query=query) - - assert isinstance(result, DocsAnswer) - assert len(result.text) > 0, 'Answer text should not be empty' - assert len(result.source_urls) > 0, 'Source URLs list should not be empty' diff --git a/integtests/tools/test_jobs.py b/integtests/tools/test_jobs.py deleted file mode 100644 index b4b66695b..000000000 --- a/integtests/tools/test_jobs.py +++ /dev/null @@ -1,266 +0,0 @@ -import asyncio -import logging - -import pytest -from mcp.server.fastmcp import Context - -from integtests.conftest import ConfigDef, ProjectDef -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.components import create_config -from keboola_mcp_server.tools.jobs import ( - GetJobsDetailOutput, - GetJobsListOutput, - JobDetail, - get_jobs, - run_job, -) - -LOG = logging.getLogger(__name__) - - -async def _wait_for_job_in_list( - mcp_context: Context, - job_id: str, - component_id: str | None, - config_id: str | None, - max_retries: int = 10, - delay: float = 0.5, -) -> GetJobsListOutput: - """ - Wait for a job to appear in the job list with retry mechanism. - - :param mcp_context: MCP context - :param job_id: ID of the job to find - :param component_id: Component ID to filter by (can be None) - :param config_id: Config ID to filter by (can be None) - :param max_retries: Maximum number of retry attempts - :param delay: Delay between retries in seconds - :return: GetJobsListOutput containing the job - :raises AssertionError: If job is not found after all retries - """ - for attempt in range(max_retries): - result = await get_jobs( - ctx=mcp_context, - job_ids=tuple(), - component_id=component_id, - config_id=config_id, - limit=10, - sort_by='startTime', - sort_order='desc', - ) - - assert isinstance(result, GetJobsListOutput) - job_ids = {job.id for job in result.jobs} - if job_id in job_ids: - LOG.info(f'Job {job_id} found in list after {attempt + 1} attempts') - return result - - if attempt < max_retries - 1: - LOG.info(f'Job {job_id} not found in list, attempt {attempt + 1}/{max_retries}, retrying in {delay}s...') - await asyncio.sleep(delay) - - raise AssertionError(f'Job {job_id} not found in job list after {max_retries} attempts') - - -@pytest.mark.asyncio -async def test_get_jobs_listing_with_component_and_config_filter(mcp_context: Context, configs: list[ConfigDef]): - """Tests that `get_jobs` works with component and config filtering when listing.""" - - # Use first config to create jobs for testing - test_config = configs[0] - component_id = test_config.component_id - configuration_id = test_config.configuration_id - - job = await run_job(ctx=mcp_context, component_id=component_id, configuration_id=configuration_id) - - # Wait for the job to appear in the list (handles race condition) - result = await _wait_for_job_in_list( - mcp_context=mcp_context, - job_id=job.id, - component_id=component_id, - config_id=configuration_id, - ) - - assert isinstance(result, GetJobsListOutput) - assert len(result.jobs) >= 1 - - # Verify our created jobs appear in the results - job_ids = {job.id for job in result.jobs} - assert job.id in job_ids - - for job in result.jobs: - assert job.component_id == component_id - assert job.config_id == configuration_id - - -@pytest.mark.asyncio -async def test_run_job_and_get_jobs( - mcp_context: Context, configs: list[ConfigDef], keboola_project: ProjectDef, storage_api_url: str -): - """Tests that `run_job` creates a job and `get_jobs` retrieves its details.""" - - project_id = keboola_project.project_id - - test_config = configs[0] - component_id = test_config.component_id - configuration_id = test_config.configuration_id - - started_job = await run_job(ctx=mcp_context, component_id=component_id, configuration_id=configuration_id) - - # Verify the started job response - assert isinstance(started_job, JobDetail) - assert started_job.id is not None - assert started_job.component_id == component_id - assert started_job.config_id == configuration_id - assert started_job.status is not None - assert frozenset(started_job.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Job: {started_job.id}', - url=f'{storage_api_url}/admin/projects/{project_id}/queue/{started_job.id}', - ), - Link( - type='ui-dashboard', - title='Jobs in the project', - url=f'{storage_api_url}/admin/projects/{project_id}/queue', - ), - ] - ) - - result = await get_jobs(ctx=mcp_context, job_ids=(started_job.id,)) - - # Verify the job detail response - assert isinstance(result, GetJobsDetailOutput) - assert len(result.jobs) == 1 - job_detail = result.jobs[0] - assert isinstance(job_detail, JobDetail) - assert job_detail.id == started_job.id - assert job_detail.component_id == component_id - assert job_detail.config_id == configuration_id - assert job_detail.status is not None - assert job_detail.url is not None - assert frozenset(job_detail.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Job: {job_detail.id}', - url=f'{storage_api_url}/admin/projects/{project_id}/queue/{job_detail.id}', - ), - Link( - type='ui-dashboard', - title='Jobs in the project', - url=f'{storage_api_url}/admin/projects/{project_id}/queue', - ), - ] - ) - - -@pytest.mark.asyncio -async def test_get_jobs_detail( - mcp_context: Context, configs: list[ConfigDef], keboola_project: ProjectDef, storage_api_url: str -): - """Tests `get_jobs` by creating a job and then retrieving its details.""" - - project_id = keboola_project.project_id - - # Use first config to create a specific job - test_config = configs[0] - component_id = test_config.component_id - configuration_id = test_config.configuration_id - - # Create a specific job to test get_jobs with - created_job = await run_job(ctx=mcp_context, component_id=component_id, configuration_id=configuration_id) - - # Now test get_jobs on the job we just created - result = await get_jobs(ctx=mcp_context, job_ids=(created_job.id,)) - - # Verify all expected fields are present - assert isinstance(result, GetJobsDetailOutput) - assert len(result.jobs) == 1 - job_detail = result.jobs[0] - assert job_detail.id == created_job.id - assert job_detail.component_id == component_id - assert job_detail.config_id == configuration_id - assert job_detail.status is not None - assert frozenset(job_detail.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Job: {created_job.id}', - url=f'{storage_api_url}/admin/projects/{project_id}/queue/{created_job.id}', - ), - Link( - type='ui-dashboard', - title='Jobs in the project', - url=f'{storage_api_url}/admin/projects/{project_id}/queue', - ), - ] - ) - - -@pytest.mark.asyncio -async def test_run_job_with_newly_created_config( - mcp_context: Context, configs: list[ConfigDef], keboola_project: ProjectDef, storage_api_url: str -): - """Tests that `run_job` works with a newly created configuration.""" - - project_id = keboola_project.project_id - - test_config = configs[0] - component_id = test_config.component_id - - # Create a new configuration for testing - new_config = await create_config( - ctx=mcp_context, - name='Test Config for Job Run', - description='Test configuration created for job run test', - component_id=component_id, - parameters={}, - storage={}, - ) - - try: - # Run a job on the new configuration - started_job = await run_job( - ctx=mcp_context, component_id=component_id, configuration_id=new_config.configuration_id - ) - - # Verify the job was started successfully - assert isinstance(started_job, JobDetail) - assert started_job.id is not None - assert started_job.component_id == component_id - assert started_job.config_id == new_config.configuration_id - assert started_job.status is not None - assert frozenset(started_job.links) == frozenset( - [ - Link( - type='ui-detail', - title=f'Job: {started_job.id}', - url=f'{storage_api_url}/admin/projects/{project_id}/queue/{started_job.id}', - ), - Link( - type='ui-dashboard', - title='Jobs in the project', - url=f'{storage_api_url}/admin/projects/{project_id}/queue', - ), - ] - ) - - # Verify job can be retrieved - result = await get_jobs(ctx=mcp_context, job_ids=(started_job.id,)) - assert isinstance(result, GetJobsDetailOutput) - assert len(result.jobs) == 1 - job_detail = result.jobs[0] - assert isinstance(job_detail, JobDetail) - assert job_detail.id == started_job.id - assert job_detail.component_id == component_id - assert job_detail.config_id == new_config.configuration_id - - finally: - # Clean up: Delete the configuration - client = KeboolaClient.from_state(mcp_context.session.state) - await client.storage_client.configuration_delete( - component_id=component_id, configuration_id=new_config.configuration_id, skip_trash=True - ) diff --git a/integtests/tools/test_project.py b/integtests/tools/test_project.py deleted file mode 100644 index 7da79b70c..000000000 --- a/integtests/tools/test_project.py +++ /dev/null @@ -1,33 +0,0 @@ -import pytest -from fastmcp import Context - -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.project import ProjectInfo, get_project_info - - -@pytest.mark.asyncio -async def test_get_project_info(mcp_context: Context, keboola_project) -> None: - result = await get_project_info(mcp_context) - - assert isinstance(result, ProjectInfo) - assert str(result.project_id) == str(keboola_project.project_id) - assert isinstance(result.project_name, str) - assert isinstance(result.organization_id, (str, int)) - assert isinstance(result.project_description, str) - assert isinstance(result.sql_dialect, str) - assert result.sql_dialect in {'Snowflake', 'BigQuery'} - assert isinstance(result.workspace_id, int) - assert result.workspace_id > 0 - assert isinstance(result.links, list) - assert result.links, 'Links list should not be empty.' - for link in result.links: - assert isinstance(link, Link) - assert link.type in {'ui-detail', 'ui-dashboard', 'docs'} - assert isinstance(link.title, str) - assert isinstance(link.url, str) - - assert isinstance(result.branch_id, (str, int)) - assert isinstance(result.branch_name, str) - assert result.branch_name - # The pool fixture runs on the default branch — see integtests/conftest.py. - assert result.is_development_branch is False diff --git a/integtests/tools/test_search.py b/integtests/tools/test_search.py deleted file mode 100644 index 9c179c96b..000000000 --- a/integtests/tools/test_search.py +++ /dev/null @@ -1,145 +0,0 @@ -import logging - -import httpx -import pytest -import toon_format -from fastmcp import Client -from fastmcp.exceptions import ToolError - -from integtests.conftest import BucketDef, ConfigDef, TableDef -from keboola_mcp_server.tools.search import SearchHit, SuggestedComponentOutput - -LOG = logging.getLogger(__name__) - - -@pytest.mark.asyncio -@pytest.mark.parametrize('item_type', [None, 'bucket', 'table', 'configuration', 'transformation']) -async def test_search_end_to_end( - item_type: str | None, - mcp_client: Client, - buckets: list[BucketDef], - tables: list[TableDef], - configs: list[ConfigDef], -) -> None: - """ - Test the search tool end-to-end by searching for items that exist in the test project. - This verifies that the search returns expected results for buckets, tables, and configurations. - """ - item_types = (item_type,) if item_type else tuple() - - full_result = await mcp_client.call_tool( - 'search', {'patterns': ['test'], 'item_types': item_types, 'limit': 50, 'offset': 0} - ) - assert full_result.structured_content is not None - LOG.info(f'result: {full_result.structured_content}') - # The search tool returns a SearchOutput envelope: {'hits': [...], 'total', 'by_type', 'branch_scope'}. - result = [SearchHit.model_validate(hit) for hit in full_result.structured_content['hits']] - - # check validity of the TOON formatted unstructured result - assert len(full_result.content) == 1 - assert full_result.content[0].type == 'text' - decoded_toon = toon_format.decode(full_result.content[0].text) - assert isinstance(decoded_toon, dict) - toon_result = [SearchHit.model_validate(hit) for hit in decoded_toon['hits']] - assert toon_result == result - - # filter out data apps that seem to often be left behind in the testing project - result = [hit for hit in result if hit.item_type != 'configuration' or hit.component_id != 'keboola.data-apps'] - - # Verify the result structure - assert isinstance(result, list) - - # Verify we found some results - assert len(result) > 0, 'Should find at least some test items' - - # Create sets of expected IDs for verification - if not item_type or item_type == 'bucket': - expected_bucket_ids = {bucket.bucket_id for bucket in buckets} - actual_bucket_ids = {hit.bucket_id for hit in result if hit.item_type == 'bucket'} - assert actual_bucket_ids == expected_bucket_ids, f'Should find all test buckets. Found: {result}' - - if not item_type or item_type == 'table': - expected_table_ids = {table.table_id for table in tables} - actual_table_ids = {hit.table_id for hit in result if hit.item_type == 'table'} - assert actual_table_ids == expected_table_ids, f'Should find all test tables. Found: {result}' - - if not item_type: - expected_config_ids = {config.configuration_id for config in configs} - actual_config_ids = { - hit.configuration_id for hit in result if hit.item_type in ['configuration', 'transformation'] - } - assert actual_config_ids == expected_config_ids, f'Should find all test configurations. Found: {result}' - - elif item_type == 'configuration': - expected_config_ids = {config.configuration_id for config in configs if config.component_id == 'ex-generic-v2'} - actual_config_ids = {hit.configuration_id for hit in result if hit.item_type == 'configuration'} - assert actual_config_ids == expected_config_ids, f'Should find all test configurations. Found: {result}' - - elif item_type == 'transformation': - expected_config_ids = { - config.configuration_id for config in configs if config.component_id == 'keboola.snowflake-transformation' - } - actual_config_ids = {hit.configuration_id for hit in result if hit.item_type == 'transformation'} - assert actual_config_ids == expected_config_ids, f'Should find all test transformations. Found: {result}' - - -@pytest.mark.asyncio -async def test_find_component_id(mcp_client: Client): - """Tests that `find_component_id` returns relevant component IDs for a query.""" - query = 'generic extractor - extract data from many APIs' - generic_extractor_id = 'ex-generic-v2' - - try: - full_result = await mcp_client.call_tool('find_component_id', {'query': query}) - except (httpx.ReadTimeout, ToolError) as e: - # The AI service backing `find_component_id` can exceed its read timeout in CI. The timeout - # surfaces either as a raw httpx.ReadTimeout or, when it round-trips through the MCP tool, as - # a ToolError carrying an "...timed out..." message. Tolerate only that timeout signature; any - # other ToolError is a real failure and must propagate. - if isinstance(e, httpx.ReadTimeout) or 'timed out' in str(e).lower(): - pytest.xfail(f'AI service exceeded read timeout in CI: {e}') - raise - - assert full_result.structured_content is not None - result = full_result.structured_content['result'] - - assert isinstance(result, list) - assert len(result) > 0 - LOG.info(f'result: {result}') - structured_result = [SuggestedComponentOutput.model_validate(component) for component in result] - assert generic_extractor_id in [component.component_id for component in structured_result] - - # check validity of the TOON formatted unstructured result - assert len(full_result.content) == 1 - assert full_result.content[0].type == 'text' - decoded_toon = toon_format.decode(full_result.content[0].text) - assert decoded_toon == result - - -@pytest.mark.asyncio -async def test_search_config_based_simple_query( - mcp_client: Client, - configs: list[ConfigDef], -) -> None: - """ - Test config-based search with a simple scoped query. - """ - config = next(cfg for cfg in configs if cfg.component_id == 'ex-generic-v2') - full_result = await mcp_client.call_tool( - 'search', - { - 'patterns': ['wttr.in'], - 'item_types': ['configuration'], - 'search_type': 'config-based', - 'scopes': ['parameters.api.baseUrl'], - 'limit': 20, - 'offset': 0, - }, - ) - - assert full_result.structured_content is not None - result = [SearchHit.model_validate(hit) for hit in full_result.structured_content['hits']] - - assert any( - hit.component_id == 'ex-generic-v2' and hit.configuration_id == config.configuration_id for hit in result - ), f'Expected config {config.configuration_id} to be returned. Found: {result}' diff --git a/integtests/tools/test_sql.py b/integtests/tools/test_sql.py deleted file mode 100644 index 54c96d6f6..000000000 --- a/integtests/tools/test_sql.py +++ /dev/null @@ -1,60 +0,0 @@ -import csv -import logging -from io import StringIO - -import pytest -from mcp.server.fastmcp import Context - -from keboola_mcp_server.tools.sql import QueryDataOutput, query_data -from keboola_mcp_server.tools.storage.tools import get_buckets, get_tables -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - - -@pytest.mark.asyncio -async def test_query_data(mcp_context: Context): - """Tests basic functionality of SQL tools: get_sql_dialect and query_data.""" - - buckets_listing = await get_buckets(ctx=mcp_context) - - tables_listing = await get_tables(bucket_ids=[buckets_listing.buckets[0].id], ctx=mcp_context) - tables_listing = await get_tables(table_ids=[tables_listing.tables[0].id], ctx=mcp_context) - table = tables_listing.tables[0] - - assert table.fully_qualified_name is not None, 'Table should have fully qualified name' - - sql_query = f'SELECT COUNT(*) as row_count FROM {table.fully_qualified_name}' - result = await query_data(sql_query=sql_query, query_name='Row Count Query', ctx=mcp_context) - - # Verify result is structured output - assert isinstance(result, QueryDataOutput) - assert result.query_name == 'Row Count Query' - assert isinstance(result.csv_data, str) - assert len(result.csv_data) > 0 - - # Parse the CSV to verify structure - csv_reader = csv.reader(StringIO(result.csv_data)) - rows = list(csv_reader) - - manager = WorkspaceManager.from_state(mcp_context.session.state) - sql_dialect = await manager.get_sql_dialect() - - # Should have a header and one data row - assert len(rows) == 2, 'COUNT query should return header + one data row' - expected_header = 'ROW_COUNT' if sql_dialect == 'Snowflake' else 'row_count' - assert rows[0] == [expected_header], f'Expected [{expected_header!r}], got {rows[0]}' - - # Count should be a number - count_value = rows[1][0] - assert count_value.isdigit(), f'Count value should be numeric, got: {count_value}' - - -@pytest.mark.asyncio -async def test_query_data_invalid_query(mcp_context: Context): - """Tests that `query_data` properly handles invalid SQL queries.""" - - invalid_sql = 'INVALID SQL SYNTAX SELECT * FROM' - - with pytest.raises(ValueError, match='Failed to run SQL query'): - await query_data(sql_query=invalid_sql, query_name='Invalid Query Test', ctx=mcp_context) diff --git a/integtests/tools/test_storage.py b/integtests/tools/test_storage.py deleted file mode 100644 index d20e29599..000000000 --- a/integtests/tools/test_storage.py +++ /dev/null @@ -1,346 +0,0 @@ -import csv -import logging -from typing import Any, cast - -import pytest -import toon_format -from fastmcp import Client, Context - -from integtests.conftest import BucketDef, TableDef -from keboola_mcp_server.clients.client import KeboolaClient, get_metadata_property -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.tools.storage.tools import ( - BucketDetail, - DescriptionUpdate, - GetBucketsOutput, - GetTablesOutput, - TableDetail, - TableSummary, - UpdateDescriptionsOutput, - get_buckets, - get_tables, - update_descriptions, -) - -LOG = logging.getLogger(__name__) - - -@pytest.mark.asyncio -async def test_get_buckets(mcp_context: Context, buckets: list[BucketDef]): - """Tests that `get_buckets` returns a list of `BucketDetail` instances.""" - result = await get_buckets(mcp_context) - - assert isinstance(result, GetBucketsOutput) - for item in result.buckets: - assert isinstance(item, BucketDetail) - - assert len(result.buckets) == len(buckets) - assert result.bucket_counts.total_buckets == len(buckets) - - # Count buckets by stage from the actual result (since BucketDef doesn't have stage info) - actual_input_count = sum(1 for bucket in result.buckets if bucket.stage == 'in') - actual_output_count = sum(1 for bucket in result.buckets if bucket.stage == 'out') - - # Verify our counts match what we calculated - assert result.bucket_counts.input_buckets == actual_input_count - assert result.bucket_counts.output_buckets == actual_output_count - - # Verify the counts add up to the total - assert ( - result.bucket_counts.input_buckets + result.bucket_counts.output_buckets == result.bucket_counts.total_buckets - ) - - -@pytest.mark.asyncio -async def test_get_buckets_output_format(mcp_client: Client, buckets: list[BucketDef]): - """Tests that `get_buckets` returns the tool output in TOON format.""" - result = await mcp_client.call_tool('get_buckets') - assert len(result.content) == 1 - assert result.content[0].type == 'text' - result_text = result.content[0].text - structured_output = GetBucketsOutput.model_validate(result.structured_content) - assert GetBucketsOutput.model_validate(toon_format.decode(result_text)) == structured_output - - # check that the buckets are presented in tabular format - expected_keys = list( - {k: 'foo' for b in structured_output.buckets for k in b.model_dump(exclude_none=True).keys()}.keys() - ) - assert result_text.startswith(f"buckets[2]{{{','.join(expected_keys)}}}:") - - -@pytest.mark.asyncio -async def test_get_bucket(mcp_context: Context, buckets: list[BucketDef]): - """Tests that for each test bucket, `get_bucket` returns a `BucketDetail` instance.""" - for bucket in buckets: - result = await get_buckets(mcp_context, [bucket.bucket_id]) - assert isinstance(result, GetBucketsOutput) - assert len(result.buckets) == 1 - assert result.buckets[0].id == bucket.bucket_id - - -@pytest.mark.asyncio -async def test_get_table(mcp_context: Context, tables: list[TableDef]): - """Tests that for each test table, `get_table` returns a `TableDetail` instance with correct fields.""" - - for table_def in tables: - with table_def.file_path.open('r', encoding='utf-8') as f: - reader = csv.reader(f) - col_names = frozenset(next(reader)) - - result = await get_tables(mcp_context, table_ids=[table_def.table_id]) - assert isinstance(result, GetTablesOutput) - assert len(result.tables) == 1 - assert result.tables[0].id == table_def.table_id - assert result.tables[0].name == table_def.table_name - assert result.tables[0].columns is not None - assert {col.name for col in result.tables[0].columns} == col_names - - -@pytest.mark.asyncio -async def test_get_tables(mcp_context: Context, tables: list[TableDef], buckets: list[BucketDef]): - """Tests that `get_tables` returns the correct tables for each bucket.""" - # Group tables by bucket to verify counts - tables_by_bucket: dict[str, list[TableDef]] = {} - for table_def in tables: - if table_def.bucket_id not in tables_by_bucket: - tables_by_bucket[table_def.bucket_id] = [] - tables_by_bucket[table_def.bucket_id].append(table_def) - - for bucket in buckets: - result = await get_tables(mcp_context, [bucket.bucket_id]) - - assert isinstance(result, GetTablesOutput) - # Listing returns summaries that never resolve the warehouse FQN, so the field must be - # absent from the (structured) output entirely — not emitted as a misleading `null`, which - # the query_data queryability rule would read as "not queryable". - for table, dumped in zip(result.tables, result.model_dump(by_alias=True)['tables']): - assert isinstance(table, TableSummary) - assert not isinstance(table, TableDetail) - assert 'fullyQualifiedName' not in dumped - assert 'columns' not in dumped - - # Verify the count matches expected tables for this bucket - expected_tables = tables_by_bucket.get(bucket.bucket_id, []) - assert len(result.tables) == len(expected_tables) - - # Verify table IDs match - result_table_ids = {table.id for table in cast(list[TableSummary], result.tables)} - expected_table_ids = {table_def.table_id for table_def in expected_tables} - assert result_table_ids == expected_table_ids - - -@pytest.mark.asyncio -async def test_get_tables_output_format(mcp_client: Client, tables: list[TableDef], buckets: list[BucketDef]): - """Tests that `get_tables` returns the tool output in TOON format.""" - result = await mcp_client.call_tool('get_tables', {'bucket_ids': [buckets[0].bucket_id]}) - assert len(result.content) == 1 - assert result.content[0].type == 'text' - result_text = result.content[0].text - structured_output = GetTablesOutput.model_validate(result.structured_content) - assert GetTablesOutput.model_validate(toon_format.decode(result_text)) == structured_output - - first_table = structured_output.tables[0] - expected_keys = list(first_table.model_dump(exclude_none=True).keys()) - assert result_text.startswith(f"tables[1]{{{','.join(expected_keys)}}}:") - - -@pytest.mark.asyncio -async def test_update_descriptions_bucket(mcp_context: Context, buckets: list[BucketDef]): - """Tests that `update_descriptions` updates bucket descriptions correctly.""" - bucket = buckets[0] - client = KeboolaClient.from_state(mcp_context.session.state) - - result = await update_descriptions( - ctx=mcp_context, - updates=[DescriptionUpdate(item_id=bucket.bucket_id, description='New Description')], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 1 - assert result.successful == 1 - assert result.failed == 0 - assert len(result.results) == 1 - - bucket_result = result.results[0] - assert bucket_result.item_id == bucket.bucket_id - assert bucket_result.success is True - assert bucket_result.error is None - assert bucket_result.timestamp is not None - - # Verify the description was actually updated - metadata = await client.storage_client.bucket_metadata_get(bucket.bucket_id) - assert get_metadata_property(metadata, MetadataField.DESCRIPTION) == 'New Description' - - -@pytest.mark.asyncio -async def test_update_descriptions_table(mcp_context: Context, mcp_client: Client, tables: list[TableDef]): - """ - Tests that `update_descriptions` updates table descriptions correctly. - Also tests that the tool output is in TOON format. - """ - table = tables[0] - storage_client = KeboolaClient.from_state(mcp_context.session.state).storage_client - - call_result = await mcp_client.call_tool( - 'update_descriptions', - { - 'updates': [{'item_id': table.table_id, 'description': 'New Table Description'}], - }, - ) - assert len(call_result.content) == 1 - assert call_result.content[0].type == 'text' - - result = UpdateDescriptionsOutput.model_validate(call_result.structured_content) - - toon_result = UpdateDescriptionsOutput.model_validate(toon_format.decode(call_result.content[0].text)) - assert toon_result == result - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 1 - assert result.successful == 1 - assert result.failed == 0 - assert len(result.results) == 1 - - table_result = result.results[0] - assert table_result.item_id == table.table_id - assert table_result.success is True - assert table_result.error is None - assert table_result.timestamp is not None - - # Verify the description was actually updated - metadata = await storage_client.table_metadata_get(table.table_id) - assert get_metadata_property(metadata, MetadataField.DESCRIPTION) == 'New Table Description' - - -@pytest.mark.asyncio -async def test_update_descriptions_table_column(mcp_context: Context, tables: list[TableDef]): - """Tests that `update_descriptions` updates table descriptions correctly.""" - table = tables[0] - - with table.file_path.open('r', encoding='utf-8') as f: - reader = csv.reader(f) - col_names = next(reader) - column_name = col_names[0] - - column_id = f'{table.table_id}.{column_name}' - result = await update_descriptions( - ctx=mcp_context, - updates=[DescriptionUpdate(item_id=column_id, description='New Table Column Description')], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 1 - assert result.successful == 1 - assert result.failed == 0 - assert len(result.results) == 1 - - column_result = result.results[0] - assert column_result.item_id == column_id - assert column_result.success is True - assert column_result.error is None - assert column_result.timestamp is not None - - # Verify the description is available in the table detail - tables_output = await get_tables(mcp_context, table_ids=[table.table_id]) - assert isinstance(tables_output, GetTablesOutput) - assert len(tables_output.tables) == 1 - table_detail = tables_output.tables[0] - assert table_detail.columns is not None - column_detail = next((col for col in table_detail.columns if col.name == column_name), None) - assert column_detail is not None - assert column_detail.description == 'New Table Column Description' - - -@pytest.mark.asyncio -async def test_update_descriptions_mixed_types(mcp_context: Context, buckets: list[BucketDef], tables: list[TableDef]): - """Tests that `update_descriptions` can handle mixed types in a single call.""" - bucket = buckets[0] - table = tables[0] - - # Get the first column name from the table CSV file - with table.file_path.open('r', encoding='utf-8') as f: - reader = csv.reader(f) - columns = next(reader) - column_name = columns[0] - - md_ids: list[tuple[str, str, str]] = [] - client = KeboolaClient.from_state(mcp_context.session.state) - try: - result = await update_descriptions( - ctx=mcp_context, - updates=[ - DescriptionUpdate(item_id=bucket.bucket_id, description='Mixed Bucket Description'), - DescriptionUpdate(item_id=table.table_id, description='Mixed Table Description'), - DescriptionUpdate(item_id=f'{table.table_id}.{column_name}', description='Mixed Column Description'), - ], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 3 - assert result.successful == 3 - assert result.failed == 0 - assert len(result.results) == 3 - - # Verify all results are successful - for item_result in result.results: - assert item_result.success is True - assert item_result.error is None - assert item_result.timestamp is not None - - # Verify bucket description was updated - bucket_metadata = await client.storage_client.bucket_metadata_get(bucket.bucket_id) - bucket_entry = next((entry for entry in bucket_metadata if entry.get('key') == MetadataField.DESCRIPTION), None) - if bucket_entry: - assert bucket_entry['value'] == 'Mixed Bucket Description' - md_ids.append(('bucket', bucket.bucket_id, str(bucket_entry['id']))) - - # Verify table description was updated - table_metadata = await client.storage_client.table_metadata_get(table.table_id) - table_entry = next((entry for entry in table_metadata if entry.get('key') == MetadataField.DESCRIPTION), None) - if table_entry: - assert table_entry['value'] == 'Mixed Table Description' - md_ids.append(('table', table.table_id, str(table_entry['id']))) - - # Verify column description was updated - table_detail = await client.storage_client.table_detail(table.table_id) - assert 'columnMetadata' in table_detail - column_metadata = cast(dict[str, list[dict[str, Any]]], table_detail['columnMetadata']) - assert column_name in column_metadata - column_entry = next( - (entry for entry in column_metadata[column_name] if entry.get('key') == MetadataField.DESCRIPTION), None - ) - if column_entry: - assert column_entry['value'] == 'Mixed Column Description' - md_ids.append(('column', f'{table.table_id}.{column_name}', str(column_entry['id']))) - - finally: - # Clean up metadata - for md_type, item_id, md_id in md_ids: - if md_type == 'bucket': - await client.storage_client.bucket_metadata_delete(bucket_id=item_id, metadata_id=md_id) - elif md_type == 'table': - await client.storage_client.table_metadata_delete(table_id=item_id, metadata_id=md_id) - elif md_type == 'column': - await client.storage_client.column_metadata_delete(column_id=item_id, metadata_id=md_id) - - -@pytest.mark.asyncio -async def test_update_descriptions_invalid_path(mcp_context: Context): - """Tests that `update_descriptions` handles invalid paths gracefully.""" - result = await update_descriptions( - ctx=mcp_context, - updates=[DescriptionUpdate(item_id='invalid-path', description='This should fail')], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 1 - assert result.successful == 0 - assert result.failed == 1 - assert len(result.results) == 1 - - error_result = result.results[0] - assert error_result.item_id == 'invalid-path' - assert error_result.success is False - assert error_result.error is not None - assert 'Invalid item_id format' in error_result.error - assert error_result.timestamp is None diff --git a/integtests/tools/test_storage_branches.py b/integtests/tools/test_storage_branches.py deleted file mode 100644 index 46feacdc5..000000000 --- a/integtests/tools/test_storage_branches.py +++ /dev/null @@ -1,550 +0,0 @@ -"""Integration tests for branched storage — validates the storage-branches deference mechanism.""" - -import csv -import json -import logging -import os -import time -import uuid -from dataclasses import dataclass -from io import StringIO -from typing import Any, Generator - -import httpx -import pytest -import pytest_asyncio -from fastmcp import Context -from mcp.server.session import ServerSession -from mcp.types import ClientCapabilities, InitializeRequestParams - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import Config -from keboola_mcp_server.mcp import ServerRuntimeInfo, ServerState -from keboola_mcp_server.tools.project import get_project_info -from keboola_mcp_server.tools.sql import QueryDataOutput, query_data -from keboola_mcp_server.tools.storage.tools import get_buckets, get_tables -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - -PYTHON_TRANSFORMATION_COMPONENT = 'keboola.python-transformation-v2' -STORAGE_BRANCHES_TOKEN_ENV_VAR = 'INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES' - - -# --- Helper functions --- - - -def _python_transform_config(destination: str, csv_filename: str, fieldnames: list[str], row: dict[str, str]) -> dict: - """Build a Python transformation config that generates a single-row CSV and writes to destination.""" - fields_str = str(fieldnames) - row_str = str(row) - script = ( - 'import csv\n' - 'import os\n' - "os.makedirs('out/tables', exist_ok=True)\n" - f"with open('out/tables/{csv_filename}', mode='wt', encoding='utf-8') as f:\n" - f" writer = csv.DictWriter(f, fieldnames={fields_str}, dialect='kbc')\n" - ' writer.writeheader()\n' - f' writer.writerow({row_str})' - ) - return { - 'storage': { - 'output': { - 'tables': [ - { - 'source': csv_filename, - 'destination': destination, - 'primary_key': ['id'], - } - ] - } - }, - 'parameters': { - 'blocks': [ - { - 'name': 'Generate data', - 'codes': [ - { - 'name': 'script', - 'script': [script], - } - ], - } - ], - 'packages': [], - }, - } - - -def _api_request(method: str, url: str, token: str, **kwargs: Any) -> dict: - """Make a synchronous HTTP request to the Keboola API.""" - headers = {'X-StorageApi-Token': token, 'Content-Type': 'application/json'} - resp = httpx.request(method, url, headers=headers, **kwargs) - resp.raise_for_status() - return resp.json() if resp.content else {} - - -def _wait_for_storage_job(base_url: str, token: str, job_id: str, timeout: int = 120) -> None: - """Wait for a Storage API job to complete.""" - deadline = time.time() + timeout - while time.time() < deadline: - job = _api_request('GET', f'{base_url}/v2/storage/jobs/{job_id}', token) - status = job.get('status') - if status == 'success': - return - if status in ('error', 'cancelled'): - raise RuntimeError(f'Storage job {job_id} failed: {job}') - time.sleep(2) - raise TimeoutError(f'Storage job {job_id} did not complete within {timeout}s') - - -def _wait_for_queue_job(base_url: str, token: str, job_id: str, timeout: int = 300) -> None: - """Wait for a Job Queue job to complete.""" - queue_url = base_url.replace('connection.', 'queue.') - deadline = time.time() + timeout - while time.time() < deadline: - resp = httpx.get(f'{queue_url}/jobs/{job_id}', headers={'X-StorageApi-Token': token}) - resp.raise_for_status() - job = resp.json() - status = job.get('status') - if status == 'success': - return - if status in ('error', 'cancelled', 'terminated'): - raise RuntimeError( - f'Queue job {job_id} failed with status={status}: {json.dumps(job.get("result", {}))[:500]}' - ) - time.sleep(5) - raise TimeoutError(f'Queue job {job_id} did not complete within {timeout}s') - - -def _create_branch(base_url: str, token: str, name: str) -> str: - """Create a dev branch and return its ID.""" - job = _api_request('POST', f'{base_url}/v2/storage/dev-branches', token, json={'name': name}) - job_id = str(job['id']) - _wait_for_storage_job(base_url, token, job_id) - job = _api_request('GET', f'{base_url}/v2/storage/jobs/{job_id}', token) - branch_id = str(job['results']['id']) - LOG.info(f'Created branch {name!r} with id={branch_id}') - return branch_id - - -def _delete_branch(base_url: str, token: str, branch_id: str) -> None: - """Delete a dev branch.""" - try: - job = _api_request('DELETE', f'{base_url}/v2/storage/dev-branches/{branch_id}', token) - job_id = str(job['id']) - _wait_for_storage_job(base_url, token, job_id) - LOG.info(f'Deleted branch {branch_id}') - except Exception: - LOG.exception(f'Failed to delete branch {branch_id}') - - -def _ensure_bucket(base_url: str, token: str, name: str, stage: str = 'in') -> str: - """Create a bucket if it doesn't already exist. Returns the bucket ID.""" - bucket_id = f'{stage}.c-{name}' - try: - _api_request('GET', f'{base_url}/v2/storage/buckets/{bucket_id}', token) - LOG.info(f'Bucket {bucket_id} already exists') - except httpx.HTTPStatusError as e: - if e.response.status_code != 404: - raise - result = _api_request( - 'POST', - f'{base_url}/v2/storage/buckets', - token, - json={'name': name, 'stage': stage, 'description': 'Integration test bucket'}, - ) - bucket_id = result['id'] - LOG.info(f'Created bucket {bucket_id}') - return bucket_id - - -def _ensure_table(base_url: str, token: str, bucket_id: str, table_name: str, csv_data: str) -> str: - """Create a table if it doesn't already exist. Returns the table ID.""" - table_id = f'{bucket_id}.{table_name}' - try: - _api_request('GET', f'{base_url}/v2/storage/tables/{table_id}', token) - LOG.info(f'Table {table_id} already exists') - except httpx.HTTPStatusError as e: - if e.response.status_code != 404: - raise - resp = httpx.post( - f'{base_url}/v2/storage/buckets/{bucket_id}/tables', - headers={'X-StorageApi-Token': token}, - data={'name': table_name, 'delimiter': ',', 'dataString': csv_data}, - ) - resp.raise_for_status() - result = resp.json() - table_id = result['id'] - LOG.info(f'Created table {table_id}') - return table_id - - -def _create_config_in_branch( - base_url: str, token: str, branch_id: str, component_id: str, name: str, config: dict -) -> str: - """Create a component configuration in a specific branch. Returns the config ID.""" - result = _api_request( - 'POST', - f'{base_url}/v2/storage/branch/{branch_id}/components/{component_id}/configs', - token, - json={'name': name, 'description': f'Integration test config: {name}', 'configuration': json.dumps(config)}, - ) - config_id = str(result['id']) - LOG.info(f'Created config {name!r} (id={config_id}) in branch {branch_id}') - return config_id - - -def _run_job_in_branch(base_url: str, token: str, branch_id: str, component_id: str, config_id: str) -> None: - """Run a job in a specific branch and wait for completion.""" - queue_url = base_url.replace('connection.', 'queue.') - resp = httpx.post( - f'{queue_url}/jobs', - headers={'X-StorageApi-Token': token, 'Content-Type': 'application/json'}, - json={'component': component_id, 'config': config_id, 'mode': 'run', 'branchId': branch_id}, - ) - resp.raise_for_status() - job = resp.json() - job_id = str(job['id']) - LOG.info(f'Started job {job_id} for {component_id}/{config_id} in branch {branch_id}') - _wait_for_queue_job(base_url, token, job_id) - LOG.info(f'Job {job_id} completed successfully') - - -# --- Test data setup/teardown --- - - -@dataclass -class BranchTestProject: - """A project with branches set up for testing.""" - - storage_api_url: str - storage_api_token: str - branch_a_id: str - branch_b_id: str - label: str - - -def _setup_branch_test_project( - storage_api_url: str, - token: str, - label: str, -) -> BranchTestProject: - """ - Set up production data, branches, and branched data in a project. - - Production data (bucket + table) is created idempotently so multiple - concurrent sessions can share the same project. - - Branch_A: - - Updates in.c-test_bucket_01.test_table_01 (creates branched version) - - Creates new bucket in.c-test_branch with test_table_branch - - Branch_B: - - Creates new bucket in.c-test_branch_2 with test_table_branch - """ - token_info = _api_request('GET', f'{storage_api_url}/v2/storage/tokens/verify', token) - project_name = token_info['owner']['name'] - features = token_info.get('owner', {}).get('features', []) - if 'storage-branches' not in features: - pytest.fail(f'[{label}] project {project_name!r} must have the storage-branches feature enabled') - LOG.info(f'[{label}] Setting up project {project_name!r}') - - _ensure_bucket(storage_api_url, token, 'test_bucket_01') - _ensure_table( - storage_api_url, - token, - 'in.c-test_bucket_01', - 'test_table_01', - '"id","name","item_count"\n1,"item1",10\n2,"item2",20', - ) - - # Create branches - uid = str(uuid.uuid4())[:8] - branch_a_id = _create_branch(storage_api_url, token, f'integtest-branch-A-{uid}') - branch_b_id = _create_branch(storage_api_url, token, f'integtest-branch-B-{uid}') - - # Branch A: update existing table (creates branched version) - config = _python_transform_config( - destination='in.c-test_bucket_01.test_table_01', - csv_filename='test_table_01.csv', - fieldnames=['id', 'name', 'item_count'], - row={'id': '99', 'name': 'branched_item', 'item_count': '999'}, - ) - cid = _create_config_in_branch( - storage_api_url, token, branch_a_id, PYTHON_TRANSFORMATION_COMPONENT, 'update-tbl', config - ) - _run_job_in_branch(storage_api_url, token, branch_a_id, PYTHON_TRANSFORMATION_COMPONENT, cid) - - # Branch A: create new bucket + table - config = _python_transform_config( - destination='in.c-test_branch.test_table_branch', - csv_filename='test_table_branch.csv', - fieldnames=['id', 'name', 'value'], - row={'id': '1', 'name': 'branch_a_data', 'value': '100'}, - ) - cid = _create_config_in_branch( - storage_api_url, token, branch_a_id, PYTHON_TRANSFORMATION_COMPONENT, 'create-tbl', config - ) - _run_job_in_branch(storage_api_url, token, branch_a_id, PYTHON_TRANSFORMATION_COMPONENT, cid) - - # Branch B: create new bucket + table - config = _python_transform_config( - destination='in.c-test_branch_2.test_table_branch', - csv_filename='test_table_branch.csv', - fieldnames=['id', 'name', 'value'], - row={'id': '1', 'name': 'branch_b_data', 'value': '200'}, - ) - cid = _create_config_in_branch( - storage_api_url, token, branch_b_id, PYTHON_TRANSFORMATION_COMPONENT, 'create-b-tbl', config - ) - _run_job_in_branch(storage_api_url, token, branch_b_id, PYTHON_TRANSFORMATION_COMPONENT, cid) - - LOG.info(f'[{label}] Setup complete: branch_a={branch_a_id}, branch_b={branch_b_id}') - return BranchTestProject( - storage_api_url=storage_api_url, - storage_api_token=token, - branch_a_id=branch_a_id, - branch_b_id=branch_b_id, - label=label, - ) - - -def _teardown_branch_test_project(project: BranchTestProject) -> None: - """Clean up branches only. Production data is kept for reuse across sessions.""" - LOG.info(f'[{project.label}] Tearing down') - _delete_branch(project.storage_api_url, project.storage_api_token, project.branch_a_id) - _delete_branch(project.storage_api_url, project.storage_api_token, project.branch_b_id) - - -# --- Fixtures --- - - -@pytest.fixture(scope='session') -def branch_project( - storage_api_url: str, - env_file_loaded: bool, -) -> Generator[BranchTestProject, Any, None]: - """ - Sets up a dedicated project (outside the pool) with the `storage-branches` - feature enabled. - - Idempotent production data setup and unique branch names allow multiple - concurrent sessions to safely share the same project. - """ - token = os.getenv(STORAGE_BRANCHES_TOKEN_ENV_VAR, '').strip() - if not token: - pytest.fail( - f'{STORAGE_BRANCHES_TOKEN_ENV_VAR} must be set to a storage token ' - f'for a project WITH the storage-branches feature' - ) - - project: BranchTestProject | None = None - try: - project = _setup_branch_test_project(storage_api_url, token, 'storage-branches') - yield project - finally: - if project is not None: - _teardown_branch_test_project(project) - - -async def _build_context( - mocker, - branch_project: BranchTestProject, - *, - branch_id: str | None, -) -> Context: - """Build an MCP context bound to a specific branch (or the default branch when None).""" - keboola_client = KeboolaClient( - storage_api_url=branch_project.storage_api_url, - storage_api_token=branch_project.storage_api_token, - headers={'User-Agent': 'KeboolaMCPServer/integtest'}, - ) - if branch_id is not None: - keboola_client = await keboola_client.with_branch_id(branch_id) - workspace_manager = await WorkspaceManager.create(keboola_client) - - mcp_config = Config( - storage_api_url=branch_project.storage_api_url, - storage_token=branch_project.storage_api_token, - ) - ctx = mocker.MagicMock(Context) - ctx.session = mocker.MagicMock(ServerSession) - ctx.session.state = { - KeboolaClient.STATE_KEY: keboola_client, - WorkspaceManager.STATE_KEY: workspace_manager, - } - ctx.session.client_params = InitializeRequestParams( - protocolVersion='1', - capabilities=ClientCapabilities(), - clientInfo={'name': 'integtest-branches', 'version': '0.0.1'}, - ) - ctx.client_id = 'KeboolaMCPServer/integtest' - ctx.session_id = None - ctx.request_context = mocker.MagicMock() - ctx.request_context.lifespan_context = ServerState(mcp_config, ServerRuntimeInfo(transport='stdio')) - return ctx - - -@pytest_asyncio.fixture -async def branch_context( - mocker, - branch_project: BranchTestProject, -) -> Context: - """MCP context bound to Branch A of the current parametrized project.""" - return await _build_context(mocker, branch_project, branch_id=branch_project.branch_a_id) - - -@pytest_asyncio.fixture -async def default_branch_context( - mocker, - branch_project: BranchTestProject, -) -> Context: - """MCP context bound to the default/production branch of the current parametrized project.""" - return await _build_context(mocker, branch_project, branch_id=None) - - -# --- Tests --- - - -@pytest.mark.asyncio -async def test_list_buckets_includes_branch_a_bucket( - branch_context: Context, - branch_project: BranchTestProject, -) -> None: - """get_buckets from Branch A should include production buckets + Branch A's new bucket, but not Branch B's.""" - result = await get_buckets(branch_context) - bucket_ids = {b.id for b in result.buckets} - - expected_ids = {'in.c-test_bucket_01', 'in.c-test_branch'} - assert bucket_ids == expected_ids, f'Expected exactly {expected_ids}, got {bucket_ids}' - - -@pytest.mark.asyncio -async def test_list_tables_in_branched_bucket( - branch_context: Context, - branch_project: BranchTestProject, -) -> None: - """get_tables for Branch A's new bucket should return the table with a production-like ID.""" - result = await get_tables(branch_context, bucket_ids=['in.c-test_branch']) - - assert len(result.tables) == 1 - table = result.tables[0] - assert table.name == 'test_table_branch' - assert table.id == 'in.c-test_branch.test_table_branch' - assert table.branch_id is None - - -@pytest.mark.asyncio -async def test_deference_branched_table( - branch_context: Context, - branch_project: BranchTestProject, -) -> None: - """get_tables for the production bucket should include the branched version of test_table_01.""" - result = await get_tables(branch_context, bucket_ids=['in.c-test_bucket_01']) - table_ids = {t.id for t in result.tables} - - assert 'in.c-test_bucket_01.test_table_01' in table_ids, f'Branched table missing. Got: {table_ids}' - - table = next(t for t in result.tables if t.id == 'in.c-test_bucket_01.test_table_01') - assert table.branch_id is None - - -@pytest.mark.asyncio -async def test_get_project_info_reports_dev_branch( - branch_context: Context, - branch_project: BranchTestProject, -) -> None: - """get_project_info from a dev-branch context should populate branch fields and is_development_branch=True.""" - result = await get_project_info(branch_context) - - assert str(result.branch_id) == str(branch_project.branch_a_id) - assert isinstance(result.branch_name, str) - assert result.branch_name - assert result.is_development_branch is True - - -@pytest.mark.asyncio -async def test_get_project_info_reports_default_branch( - default_branch_context: Context, - branch_project: BranchTestProject, -) -> None: - """get_project_info from a default-branch context should populate branch fields and is_development_branch=False.""" - result = await get_project_info(default_branch_context) - - assert isinstance(result.branch_id, (str, int)) - assert isinstance(result.branch_name, str) - assert result.branch_name - assert result.is_development_branch is False - # Sanity: the default branch must not be the dev branch we created for this session. - assert str(result.branch_id) != str(branch_project.branch_a_id) - assert str(result.branch_id) != str(branch_project.branch_b_id) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('table_id', 'description'), - [ - ('in.c-test_branch.test_table_branch', 'branch-only table created via transformation'), - ('in.c-test_bucket_01.test_table_01', 'production table (also has a branched version in Branch A)'), - ], - ids=['branch_only_table', 'production_table'], -) -async def test_query_data_from_dev_branch_reaches_both_kinds_of_tables( - branch_context: Context, - branch_project: BranchTestProject, - table_id: str, - description: str, -) -> None: - """ - From a dev-branch MCP context, `query_data` must successfully execute SELECT against - both a table that exists only in the branch and a table that exists in production. - Verifies the branch-aware workspace selection unblocks SQL on both kinds of tables. - """ - tables_listing = await get_tables(branch_context, table_ids=[table_id]) - assert len(tables_listing.tables) == 1, f'Expected exactly one table for {table_id}' - table = tables_listing.tables[0] - assert table.fully_qualified_name, f'{description}: table {table_id} has no FQN, cannot query' - - sql_query = f'SELECT COUNT(*) AS row_count FROM {table.fully_qualified_name}' - result = await query_data( - sql_query=sql_query, - query_name=f'Row count for {table_id}', - ctx=branch_context, - ) - - assert isinstance(result, QueryDataOutput) - assert result.csv_data, f'{description}: query returned no CSV data' - - # Sanity-check: COUNT(*) returns a numeric value. Specific row count varies by data - # (fixture writes 1 row to each branched table; the production source table has 2 rows). - csv_reader = csv.reader(StringIO(result.csv_data)) - rows = list(csv_reader) - assert len(rows) == 2, f'{description}: COUNT query should return header + 1 data row, got {rows}' - count_value = rows[1][0] - assert count_value.isdigit(), f'{description}: expected numeric row count, got {count_value!r}' - assert int(count_value) >= 1, f'{description}: expected positive row count, got {count_value!r}' - - -@pytest.mark.asyncio -async def test_workspace_id_is_branch_aware( - branch_context: Context, - default_branch_context: Context, - branch_project: BranchTestProject, -) -> None: - """ - On a `storage-branches` project, the dev-branch context must return a DIFFERENT - workspace than the default branch. - """ - dev_result = await get_project_info(branch_context) - default_result = await get_project_info(default_branch_context) - - assert isinstance(dev_result.workspace_id, int) - assert dev_result.workspace_id > 0 - assert isinstance(default_result.workspace_id, int) - assert default_result.workspace_id > 0 - - assert dev_result.workspace_id != default_result.workspace_id, ( - f'storage-branches project expected per-branch workspace, ' - f'got the same id {dev_result.workspace_id} from both contexts' - ) diff --git a/integtests/validate.test.ts b/integtests/validate.test.ts new file mode 100644 index 000000000..bd8d07e10 --- /dev/null +++ b/integtests/validate.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; + +import { __testing } from '@/tools/validation'; +import { getTestProjectForTest } from './testproject/fixture'; + +// Ported from integtests/test_validate.py. +// +// The Python test fetches the storage stack index (which lists every component with its +// root/row configuration schema) and runs each schema through KeboolaParametersValidator +// to confirm none of them are *structurally* invalid (jsonschema.SchemaError) — a sanity +// check that the MCP server can sanitize+validate every real component schema. Validation +// errors against dummy parameters are irrelevant and ignored; only schema errors fail. +// +// The TS validator's schema sanitization (sanitizeSchema, the equivalent of +// KeboolaParametersValidator.sanitize_schema) is the structural check: it throws on a +// structurally invalid schema and returns normally otherwise. We run it over every +// component's root and row schema fetched from the live stack index. + +type RawComponent = { + id: string; + type?: string; + name?: string; + configurationSchema?: Record | null; + configurationRowSchema?: Record | null; +}; + +describe('component schema validation (integration)', () => { + it('sanitizes every root and row schema on the stack without a structural error', async () => { + const project = await getTestProjectForTest({ clean: false }); + + // Fetch the storage stack index directly — it carries the full component list with + // their configuration schemas (the same source the Python test reads). + const res = await fetch(`${project.storageApiUrl}/v2/storage`, { + headers: { 'X-StorageApi-Token': project.storageApiToken }, + }); + expect(res.ok, `Storage index fetch failed: ${res.status}`).toBeTruthy(); + const data = (await res.json()) as { components?: RawComponent[] }; + const components = data.components ?? []; + expect(components.length).toBeGreaterThan(0); + + let rootCount = 0; + let rowCount = 0; + const invalidRoot: string[] = []; + const invalidRow: string[] = []; + + for (const component of components) { + if (component.configurationSchema) { + rootCount++; + try { + __testing.sanitizeSchema(component.configurationSchema); + } catch { + invalidRoot.push(component.id); + } + } + if (component.configurationRowSchema) { + rowCount++; + try { + __testing.sanitizeSchema(component.configurationRowSchema); + } catch { + invalidRow.push(component.id); + } + } + } + + expect(invalidRoot, `Invalid root schemas (${invalidRoot.length}): ${invalidRoot}`).toEqual([]); + expect(invalidRow, `Invalid row schemas (${invalidRow.length}): ${invalidRow}`).toEqual([]); + // Sanity: the stack exposed at least some schemas to validate. + expect(rootCount + rowCount).toBeGreaterThan(0); + }); +}); diff --git a/integtests/workspace.test.ts b/integtests/workspace.test.ts new file mode 100644 index 000000000..67b21ba98 --- /dev/null +++ b/integtests/workspace.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; + +import { callToolText, connectMcp } from './helpers/mcp'; +import { getTestProjectForTest } from './testproject/fixture'; + +// Ported from integtests/test_workspace.py. +// +// The Python tests poke WorkspaceManager internals directly: test_static_workspace +// resolves a workspace by its configured schema and reads its backend; test_dynamic_workspace +// creates a workspace on demand and confirms it is recorded in the branch. The TS port has +// no configured workspace_schema in the pooled Config — the WorkspaceManager always resolves +// (or provisions) the read-only SQL workspace on first use. We therefore assert the +// observable end-to-end behavior the manager is responsible for, through the tools that use +// it: the workspace is provisioned, its backend is resolved, and SQL actually runs over it. + +describe('workspace provisioning + SQL execution (integration)', () => { + it('resolves the workspace backend dialect via get_project_info', async () => { + // Port of test_static_workspace's `info.backend in ['snowflake', 'bigquery']`: + // get_project_info surfaces the sql_dialect, which the WorkspaceManager derives from + // the resolved workspace's backend. + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const text = await callToolText(session.client, 'get_project_info'); + expect(text).toMatch(/Snowflake|BigQuery/i); + } finally { + await session.close(); + } + }); + + it('provisions a workspace and runs a SELECT through it', async () => { + // Port of test_dynamic_workspace: with no preconfigured workspace schema, the first + // query_data call must provision/resolve the workspace and execute the SQL over it. + // A literal SELECT proves the workspace is live and queryable end-to-end. + const { config } = await getTestProjectForTest({ clean: false }); + const session = await connectMcp(config); + try { + const text = await callToolText(session.client, 'query_data', { + sql_query: 'SELECT 1 AS one', + query_name: 'Workspace Smoke Query', + }); + expect(text).toContain('Workspace Smoke Query'); + expect(text).toMatch(/\b1\b/); + } finally { + await session.close(); + } + }); +}); diff --git a/logging-json.conf b/logging-json.conf deleted file mode 100644 index 9ecbabbf7..000000000 --- a/logging-json.conf +++ /dev/null @@ -1,40 +0,0 @@ -[loggers] -keys=root,mcp.server,uvicorn.access,uvicorn.error,fastmcp - -[handlers] -keys=root - -[formatters] -keys=json - -[logger_root] -level=INFO -handlers=root - -[logger_mcp.server] -level=WARNING -qualname=mcp.server -handlers= - -[logger_uvicorn.access] -qualname=uvicorn.access -handlers= -propagate=1 - -[logger_uvicorn.error] -qualname=uvicorn.error -handlers= -propagate=1 - -[logger_fastmcp] -qualname=fastmcp -handlers= -propagate=1 - -[handler_root] -class=logging.StreamHandler -formatter=json -args=(sys.stdout,) - -[formatter_json] -class=json_log_formatter.VerboseJSONFormatter \ No newline at end of file diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 09d4a00bc..000000000 --- a/mypy.ini +++ /dev/null @@ -1,30 +0,0 @@ -[mypy] -python_version = 3.10 -warn_return_any = True -warn_unused_configs = True -disallow_untyped_defs = True -disallow_incomplete_defs = True -check_untyped_defs = True -disallow_untyped_decorators = False -no_implicit_optional = True -warn_redundant_casts = True -warn_unused_ignores = True -warn_no_return = True -warn_unreachable = True -show_error_codes = True -ignore_missing_imports = True - -[mypy.plugins.numpy.*] -ignore_errors = True - -[mypy-pandas.*] -ignore_missing_imports = True - -[mypy-kbcstorage.*] -ignore_missing_imports = True - -[mypy-mcp.*] -ignore_missing_imports = True - -[mypy-httpx.*] -ignore_missing_imports = True \ No newline at end of file diff --git a/openwiki/INSTRUCTIONS.md b/openwiki/INSTRUCTIONS.md new file mode 100644 index 000000000..b0e3e331f --- /dev/null +++ b/openwiki/INSTRUCTIONS.md @@ -0,0 +1 @@ +A code wiki for this local repository. Prioritize a concise quickstart, architecture overview, source map, key workflows, domain concepts, operations/runbook notes, testing guidance, and integration points. Inspect git history to understand reasoning behind code changes and the progression of the repository. Keep pages grounded in the repository structure and recent code changes. Prefer practical navigation for engineers over generic summaries. diff --git a/oxlint.config.ts b/oxlint.config.ts new file mode 100644 index 000000000..a3354fc5f --- /dev/null +++ b/oxlint.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'oxlint'; + +import keboolaShared from '@keboola/oxlint-config'; + +export default defineConfig({ + extends: [keboolaShared], + env: { + node: true, + }, + ignorePatterns: ['dist/'], + rules: { + // Not a turborepo; env vars are validated by Config, not turbo.json. + 'turbo/no-undeclared-env-vars': 'off', + }, +}); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 000000000..cd9fdac5d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,10604 @@ +{ + "name": "@keboola/mcp-server", + "version": "2.0.0-alpha.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@keboola/mcp-server", + "version": "2.0.0-alpha.1", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.14", + "@keboola/api-client": "^5.0.0", + "@modelcontextprotocol/sdk": "^1.29.0", + "@toon-format/toon": "^2.3.0", + "ajv": "^8.20.0", + "cheerio": "*", + "dd-trace": "^5.69.0", + "hono": "^4.12.27", + "jose": "^6.2.3", + "pg": "^8.22.0", + "pino": "^9.14.0", + "zod": "^4.3.6" + }, + "bin": { + "keboola-mcp-server": "dist/index.js" + }, + "devDependencies": { + "@keboola/oxlint-config": "^0.1.2", + "@keboola/tsconfig": "^0.1.1", + "@types/node": "^24.0.0", + "@types/pg": "^8.20.0", + "ioredis": "^5.4.1", + "msw": "^2.14.6", + "oxfmt": "^0.44.0", + "oxlint": "^1.56.0", + "tsup": "^8.5.1", + "tsx": "^4.19.4", + "typescript": "^5.9.0", + "vitest": "^4.1.8" + }, + "engines": { + "node": ">=22" + }, + "optionalDependencies": { + "@huggingface/transformers": "^4.2.0", + "cheerio": "^1.2.0" + } + }, + "node_modules/@alloc/quick-lru": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", + "integrity": "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@datadog/flagging-core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@datadog/flagging-core/-/flagging-core-1.2.1.tgz", + "integrity": "sha512-qeDkki9fFlqyoZBrn7tneT6pZ04EKKvf3xxisYw1a74zbJihvQui/ARUsjXCurRpzpFqGGTJw/oz+HnXaKhcdw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "spark-md5": "^3.0.2" + } + }, + "node_modules/@datadog/libdatadog": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/@datadog/libdatadog/-/libdatadog-0.9.4.tgz", + "integrity": "sha512-55wHHAuhHOOrBGoWV+fRuEDypN6PMJZq4LTOwExnhoDMvVcZKtqYT05xea/toRs0o75+A1IeNAQHsRLbJMf5oA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/@datadog/native-appsec": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/@datadog/native-appsec/-/native-appsec-11.0.1.tgz", + "integrity": "sha512-Y/XfknUmmJcw4hhQVhqzgdQvfjy+EGmXuUBgtVkI1r+/qS00egYu+wD/x7pOvjdbZNqN96znVszAnXvDQAzMDQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "node-gyp-build": "^3.9.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@datadog/native-iast-taint-tracking": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-4.2.0.tgz", + "integrity": "sha512-NpZABJQoNMzF6cU521RT4GQ8/FbfFRoDepOLTcLYKyw0DY2WmSpg3iG+PoQNK4O3jPSXC++K3rg59GiQgA3Mog==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "node-gyp-build": "^3.9.0" + } + }, + "node_modules/@datadog/native-metrics": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@datadog/native-metrics/-/native-metrics-3.1.2.tgz", + "integrity": "sha512-7AEWt0ZLr/ogR/9if1DmFBDTg3y67xM+gdhXUXKs+UQMxK0lnjrOHgN7fkpEmUG1uL+EkX2BDE3ENDlQ23J7OQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "node-addon-api": "^6.1.0", + "node-gyp-build": "^3.9.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@datadog/openfeature-node-server": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@datadog/openfeature-node-server/-/openfeature-node-server-2.0.0.tgz", + "integrity": "sha512-Ummu/Bd7ZJpCCNdFnZUt/JI+L1+8OMK53+MyIXbS7dCt4JXWWwelbpzSGqmC2jWbWQ/mTo4hEfnFw3kYOINbXA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@datadog/flagging-core": "1.2.1" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "@openfeature/server-sdk": ">=1.15.1" + } + }, + "node_modules/@datadog/pprof": { + "version": "5.15.1", + "resolved": "https://registry.npmjs.org/@datadog/pprof/-/pprof-5.15.1.tgz", + "integrity": "sha512-4mI750tX6okNROS4YKvGQjyAQ+VqfzDqzysCytOJhL2E2ktK/q4M0PtC7j70tiirGb/fO9fjma3IMNSRLYX+xQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "node-gyp-build": "^4.8.4", + "pprof-format": "^2.2.1", + "source-map": "^0.7.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@datadog/pprof/node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/@datadog/wasm-js-rewriter": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@datadog/wasm-js-rewriter/-/wasm-js-rewriter-5.0.1.tgz", + "integrity": "sha512-EzbV3Lrdt3udQEsbDOVC5gB1y7yxfpBbrSIk4jaEsGjyj0Dbv2HGj7tZjs+qXzIzNonHc8h5El2bYZOGfC2wwg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "js-yaml": "^4.1.0", + "lru-cache": "^7.14.0", + "module-details-from-path": "^1.0.3", + "node-gyp-build": "^4.5.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/@datadog/wasm-js-rewriter/node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint-react/ast": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@eslint-react/ast/-/ast-4.2.3.tgz", + "integrity": "sha512-/XHJPFX8lsp+c/gMzFOnIxqH7YIXVX8SlMHuZ6XTUlYHkGquhydTtgso0VFiLQN1z3dThrybdgBq+JD+LSwK2w==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/types": "^8.58.0", + "@typescript-eslint/typescript-estree": "^8.58.0", + "@typescript-eslint/utils": "^8.58.0", + "string-ts": "^2.3.1" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/@eslint-react/core": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@eslint-react/core/-/core-4.2.3.tgz", + "integrity": "sha512-r0cgJlCemBb61f0qCrXS95hNq2ajIku5V7Tk45fROQu4HIV55ILJeN2ceea1LKmgRWy/pQw8+SvImronwWo16A==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-react/ast": "4.2.3", + "@eslint-react/jsx": "4.2.3", + "@eslint-react/shared": "4.2.3", + "@eslint-react/var": "4.2.3", + "@typescript-eslint/scope-manager": "^8.58.0", + "@typescript-eslint/types": "^8.58.0", + "@typescript-eslint/utils": "^8.58.0", + "ts-pattern": "^5.9.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/@eslint-react/jsx": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@eslint-react/jsx/-/jsx-4.2.3.tgz", + "integrity": "sha512-lSwRo/PAwf1EvXRxpXA5yBhPIxahFuC4uHh84nc5OxE0mJ7YEmzmASR+ug3QOnVnfDsJDVo6AWVR7PSL99YkOQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-react/ast": "4.2.3", + "@eslint-react/shared": "4.2.3", + "@eslint-react/var": "4.2.3", + "@typescript-eslint/types": "^8.58.0", + "@typescript-eslint/utils": "^8.58.0", + "ts-pattern": "^5.9.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/@eslint-react/shared": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@eslint-react/shared/-/shared-4.2.3.tgz", + "integrity": "sha512-6HermdKaTWkID0coAK46ynA9XIwUWGgA2Y+NK6qcmL/qbYzyRYs4hq+SmLMvZZ8DV/SFOaHRXl9iCTvjf6DvXQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/utils": "^8.58.0", + "ts-pattern": "^5.9.0", + "zod": "^4.3.6" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/@eslint-react/var": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@eslint-react/var/-/var-4.2.3.tgz", + "integrity": "sha512-zkQki2eYbQrMW4O6DCZDQzslFvw0sWAlvW/WWjocEIGHqRGC3IHWcRt3xsq8JPNOW4WjF4/LZ8czkyLoINV9rw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-react/ast": "4.2.3", + "@eslint-react/shared": "4.2.3", + "@typescript-eslint/scope-manager": "^8.58.0", + "@typescript-eslint/types": "^8.58.0", + "@typescript-eslint/utils": "^8.58.0", + "ts-pattern": "^5.9.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@eslint/object-schema": "^3.0.5", + "debug": "^4.3.1", + "minimatch": "^10.2.4" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.6.0.tgz", + "integrity": "sha512-ii6Bw9jJ2zi2cWA2Z+9/QZ/+3DX6kwaV5Q986D/CdP3Lap3w/pgQZ373FV7byY/i7L4IRH/G43I5dz1ClsCbpA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@eslint/core": "^1.2.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/object-schema": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.2.tgz", + "integrity": "sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@eslint/core": "^1.2.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@hono/node-server": { + "version": "1.19.14", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz", + "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==", + "license": "MIT", + "engines": { + "node": ">=18.14.1" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@huggingface/jinja": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.9.tgz", + "integrity": "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tokenizers": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.1.3.tgz", + "integrity": "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/@huggingface/transformers": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.2.0.tgz", + "integrity": "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@huggingface/jinja": "^0.5.6", + "@huggingface/tokenizers": "^0.1.3", + "onnxruntime-node": "1.24.3", + "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", + "sharp": "^0.34.5" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@inquirer/ansi": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-2.0.7.tgz", + "integrity": "sha512-3eTuUO1vH2cZm2ZKHeQxnOqlTi9EfZDGgIe3BL3I4u+rJHocr9Fz86M4fjYABPvFnQG/gGK551HqDiIcETwU6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + } + }, + "node_modules/@inquirer/confirm": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-6.1.1.tgz", + "integrity": "sha512-eb8DBZcz/2qHWQda4rk2JiQk5h9QV/cVHi1yjt0f69WFZMRFn0sJTye3EAP8icut8UDMjQPsaH5KbcOogefrFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.2.1", + "@inquirer/type": "^4.0.7" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "11.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-11.2.1.tgz", + "integrity": "sha512-Qd6GJT1yVyrZZCfN8W2qKF5ApmqryXRhRKCuip8h01x2w/esJQ2XIYc6f9abMIHgKQdBfFTSOdbHRLAhuM09UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.7", + "@inquirer/figures": "^2.0.7", + "@inquirer/type": "^4.0.7", + "cli-width": "^4.1.0", + "fast-wrap-ansi": "^0.2.0", + "mute-stream": "^3.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/figures": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-2.0.7.tgz", + "integrity": "sha512-aJ8TBPOGB6f/2qziPfElISTCEd5XOYTFckA2SGjhNmiKzfK/u4ot3v0DUzGVdUnKjN10EqnnEPck36BkyfLnJw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + } + }, + "node_modules/@inquirer/type": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-4.0.7.tgz", + "integrity": "sha512-t28inv14nMQ1PhKpsJPY+kEs/c00qzeCOS2gTNRyTjG5d6qsVA2fItxW4hkvGZ5lvanGLdtCzVIx5dwdRpN1+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@ioredis/commands": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", + "integrity": "sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@keboola/api-client": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@keboola/api-client/-/api-client-5.0.0.tgz", + "integrity": "sha512-e1Qt6cusBltMQD6cVPG5J7dtu8bbLXftT5/x8wouD8oOWzr05YvMmr9SoxyS4mi2HBYnDoZAJ3qZomZ15n32sw==", + "license": "MIT", + "dependencies": { + "dayjs": "^1.11.19", + "qs": "^6.15.2", + "zod": "^4.3.6" + }, + "engines": { + "node": "24.x" + } + }, + "node_modules/@keboola/oxlint-config": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@keboola/oxlint-config/-/oxlint-config-0.1.2.tgz", + "integrity": "sha512-TjCOk3u18ct7xrDhA4JQ5rqk+96Kqjd4DgVXESCu7/sMNjErCoRl3dGKRKVg1yBqm666XxC4gwozRg1wTH1+nA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@keboola/tailwind-config": "^0.1.1", + "eslint": "^10.2.0", + "eslint-plugin-react-dom": "^4.2.3", + "eslint-plugin-react-jsx": "^4.2.3", + "eslint-plugin-react-naming-convention": "^4.2.3", + "eslint-plugin-react-web-api": "^4.2.3", + "eslint-plugin-react-x": "^4.2.3", + "eslint-plugin-simple-import-sort": "^13.0.0", + "eslint-plugin-tailwindcss": "^3.18.2", + "eslint-plugin-turbo": "^2.9.5", + "oxlint": "^1.56.0" + } + }, + "node_modules/@keboola/tailwind-config": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@keboola/tailwind-config/-/tailwind-config-0.1.1.tgz", + "integrity": "sha512-PdHfIWd8Mkm2ZydMGriUtj3t/2K8S5cwBdwPwGxNA1zQPzggAKqLNMJ/L3c2IuS8117BUGKloPL7N0a1bMP/aQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@tailwindcss/container-queries": "^0.1.1", + "@tailwindcss/typography": "^0.5.19", + "tailwindcss-animate": "1.0.7", + "tailwindcss-scoped-preflight": "3.5.2" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "tailwindcss": "^3.4.0" + } + }, + "node_modules/@keboola/tsconfig": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@keboola/tsconfig/-/tsconfig-0.1.1.tgz", + "integrity": "sha512-29aoQJclUv7wEbc+4hJNMK83896G8ObXwJcSnO4bzk5ZOxxC88fRhELkBsb9no3rp5zQsV0iLg2uiG1h51Rlug==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "typescript": "^5 || ^6" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.29.0.tgz", + "integrity": "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==", + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/@mswjs/interceptors": { + "version": "0.41.9", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.41.9.tgz", + "integrity": "sha512-VVPPgHyQ6ShqnrmDWuxjmUIsO9gWyOZFmuOfLd9LfBGQJwZfy0gvv9pbHSJuoFNIYC7ZDX9aoFwowjcdSC4E8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/logger": "^0.3.0", + "@open-draft/until": "^2.0.0", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.3", + "strict-event-emitter": "^0.5.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@mswjs/interceptors/node_modules/@open-draft/deferred-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@open-draft/deferred-promise": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-3.0.0.tgz", + "integrity": "sha512-XW375UK8/9SqUVNVa6M0yEy8+iTi4QN5VZ7aZuRFQmy76LRwI9wy5F4YIBU6T+eTe2/DNDo8tqu8RHlwLHM6RA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@open-draft/logger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", + "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-node-process": "^1.2.0", + "outvariant": "^1.4.0" + } + }, + "node_modules/@open-draft/until": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", + "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@openfeature/core": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@openfeature/core/-/core-1.11.0.tgz", + "integrity": "sha512-P0u3/ht/oZCQT89fOed+laLk0kZR529a825cS02uPDglxXbE97irWYpDAeRGGVETIzKfuy+H2g8c3Ccv/tXJNQ==", + "license": "Apache-2.0", + "optional": true, + "peer": true + }, + "node_modules/@openfeature/server-sdk": { + "version": "1.22.0", + "resolved": "https://registry.npmjs.org/@openfeature/server-sdk/-/server-sdk-1.22.0.tgz", + "integrity": "sha512-YBrf6SQkn0FNB/dRAtLEs41dvFMUE8CrQTwI+iLaMFUIqWlqGNJfGnulKSneEKS+2OgKTAC6DdmKcZ6tK7kBcg==", + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@openfeature/core": "^1.11.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.219.0.tgz", + "integrity": "sha512-FFx7YnaYJlIjqWW/AG/yAZ0L/NEY724PipXXXQLdtZPbLwBGbUMTGL1i/esI56TWfTUXxhLfpgrnWJCG8aUJyg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@oxc-parser/binding-android-arm-eabi": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.132.0.tgz", + "integrity": "sha512-KrLaPWa5c9Y7LkW+rKkaUE3y7DBDrQtaf7rlsSDfv6KAHUjgzAIRA761Lrrp6//Yd/Rlie/yEOt9YENCoJnOcw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-android-arm64": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.132.0.tgz", + "integrity": "sha512-SThDrSeamB/kG2+NxcJ5/wSLcV6dUqDknrPLqFYQ0ST/55mtBP4M7Q/f3QbubH6aAd11wpzZn/nwbVRSdobOpg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-arm64": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.132.0.tgz", + "integrity": "sha512-Lc0f/TYoKBghE5/2Gsv7bLXk+TJZunx2Tf61X8hG4ARXdc8UYI26dCGccFSd1AyFbK3jfaNXtMnupggDbjPXdQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-x64": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.132.0.tgz", + "integrity": "sha512-RG2eJIpf7C21z9HSSXFw1bTArdpKe7Y4fwcJTwRq1yCSe1vSavaN9GA1sm9KqzemTLAGVktQ+7qBTGp0vQeUZg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-freebsd-x64": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.132.0.tgz", + "integrity": "sha512-wQIPntPLtJ8NcBpvKPbEv3NqzV6k8eP8tP/jE9Rg8HTg/j7urZGFSsTCPCW5k77Qfw2DM4vRvc9p3I4yq/Shvw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-gnueabihf": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.132.0.tgz", + "integrity": "sha512-PixKEpeSe3yxQWqNyOCBALRYc72+Tj7ILDofUl3iXo25cVOzLA6jHUhmOINRtWIPh7dbUie3QNeabwaQpZTw6w==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-musleabihf": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.132.0.tgz", + "integrity": "sha512-sCR+DzGHlyHKnbA2z9zWjTUhIo8Sy0enJl4RDsBwPmkxYynPatpwOAWe8W5127SlW0boqUWHGtr1NWn5UwIhXQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-gnu": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.132.0.tgz", + "integrity": "sha512-sQBix5P2cW+IpzTcCwYxnh9yALrKSIkKJThspBvMGcygSMnbzkSvhN7SfuX1hvBk8y1XEChsdkU3ET0V5DmzUw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-musl": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.132.0.tgz", + "integrity": "sha512-WozHg3Kc//8Sk756HXXgMbEAvqtG+Lzb9JOojwQzIGDtN78Az2dLttkb71akWYUF/8IgYfDSlfKh4Uot8is5Vw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-ppc64-gnu": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.132.0.tgz", + "integrity": "sha512-CmX/ulNBOEwWTyVRmcpYKAcAizW6+OjtLJgo7fXoL9OqQvjF4VER8tPomv44vwzfSCy1BHbsB0ZlZYzYJNj4cA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-gnu": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.132.0.tgz", + "integrity": "sha512-j9oQS+hM90SdhviNGWbPgT4+Rlq+ac++q/zjgwPD1mVHgxHzATvoRGtDx0sXGmFOQ9J9YkwAhYGb5MAHL6TAsA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-musl": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.132.0.tgz", + "integrity": "sha512-bLz+Xi+Agnfmd7kWPEsSVwCn2k4EyIalZkNBcQ0OGIv9rqn8VgCPLNd03tM9mKX/5TdlvDXalz0q71BIrOPNqg==", + "cpu": [ + "riscv64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-s390x-gnu": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.132.0.tgz", + "integrity": "sha512-U6t2qbJU0ypTfyj9QV3W1Y6mITDTL8ai/OR6NUn85vyHthOvobKWgXzU4tu0EskSzlpuVFz1g0jFGulDIUKHxQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-gnu": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.132.0.tgz", + "integrity": "sha512-WcEaSNHFk8yz5YFlQQAlhq6jOFmZBB/RKE7uzhyCIf+pF1Lmv9gUH4221mle2Gd9iHyWT3ySNph8yZgb1xYdWg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-musl": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.132.0.tgz", + "integrity": "sha512-iQrV4iJzQgRwK3BWRmQl1C3C6g3wYpXN2WLdQdyR+efoUnncdShZAVp9OgcojtlD3MDRbuOMGG3SjxF4fL4nlQ==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-openharmony-arm64": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.132.0.tgz", + "integrity": "sha512-FWzmUGrZ6GUby4U7WIwcCtab6tdmlTO3xTRRKyb5kjIJVEiaUAT8animUG/nK8ZCA8gkRkPOTId4rl6uTqUmJQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-wasm32-wasi": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.132.0.tgz", + "integrity": "sha512-TlbMppxJI5CjWDes0QaP6G3aneVg1yikBu5QYI+DUShF9WDL66ccgKFNNGmi/Wybtszw6hxwAvv76T4DaPKnHw==", + "cpu": [ + "wasm32" + ], + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@oxc-parser/binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@oxc-parser/binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@oxc-parser/binding-win32-arm64-msvc": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.132.0.tgz", + "integrity": "sha512-RH/NbFjGKqdUAUi7Oh3LQPxUk2hsWFEEQ38HSnbRQT8QjBZFKqL1fMbmsB3N4jy/KPh9iX94+9dmkEMBBbambw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-ia32-msvc": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.132.0.tgz", + "integrity": "sha512-JUr4jQY9jxoIB/YTLXr6XofSi5xikj6p5/Ns1h0VOBDT0j1jKU+kMsv2xxv51RwnETcXpA1Yw/9oUAfcqfaqEA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-x64-msvc": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.132.0.tgz", + "integrity": "sha512-2dapgHpA5X8DSXF4AU36hJWYf6zP0tKjMXFRAZFBD62pkevW/uhFDXoFH9Y/3Fd2EtDrw5ByNnR1wVE9X9y0SQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.137.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.137.0.tgz", + "integrity": "sha512-WT+Gb24i8hmvo85AIv2oEYouEXkRlKAlT9WaCa3TfLgNCN+GhrJOGZuIlMouAh38Qe4QOx26eUOVsq70qXrywA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@oxfmt/binding-android-arm-eabi": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm-eabi/-/binding-android-arm-eabi-0.44.0.tgz", + "integrity": "sha512-5UvghMd9SA/yvKTWCAxMAPXS1d2i054UeOf4iFjZjfayTwCINcC3oaSXjtbZfCaEpxgJod7XiOjTtby5yEv/BQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-android-arm64": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm64/-/binding-android-arm64-0.44.0.tgz", + "integrity": "sha512-IVudM1BWfvrYO++Khtzr8q9n5Rxu7msUvoFMqzGJVdX7HfUXUDHwaH2zHZNB58svx2J56pmCUzophyaPFkcG/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-darwin-arm64": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-arm64/-/binding-darwin-arm64-0.44.0.tgz", + "integrity": "sha512-eWCLAIKAHfx88EqEP1Ga2yz7qVcqDU5lemn4xck+07bH182hDdprOHjbogyk0In1Djys3T0/pO2JepFnRJ41Mg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-darwin-x64": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-x64/-/binding-darwin-x64-0.44.0.tgz", + "integrity": "sha512-eHTBznHLM49++dwz07MblQ2cOXyIgeedmE3Wgy4ptUESj38/qYZyRi1MPwC9olQJWssMeY6WI3UZ7YmU5ggvyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-freebsd-x64": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-freebsd-x64/-/binding-freebsd-x64-0.44.0.tgz", + "integrity": "sha512-jLMmbj0u0Ft43QpkUVr/0v1ZfQCGWAvU+WznEHcN3wZC/q6ox7XeSJtk9P36CCpiDSUf3sGnzbIuG1KdEMEDJQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-arm-gnueabihf": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.44.0.tgz", + "integrity": "sha512-n+A/u/ByK1qV8FVGOwyaSpw5NPNl0qlZfgTBqHeGIqr8Qzq1tyWZ4lAaxPoe5mZqE3w88vn3+jZtMxriHPE7tg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-arm-musleabihf": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.44.0.tgz", + "integrity": "sha512-5eax+FkxyCqAi3Rw0mrZFr7+KTt/XweFsbALR+B5ljWBLBl8nHe4ADrUnb1gLEfQCJLl+Ca5FIVD4xEt95AwIw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-arm64-gnu": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.44.0.tgz", + "integrity": "sha512-58l8JaHxSGOmOMOG2CIrNsnkRJAj0YcHQCmvNACniOa/vd1iRHhlPajczegzS5jwMENlqgreyiTR9iNlke8qCw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-arm64-musl": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.44.0.tgz", + "integrity": "sha512-AlObQIXyVRZ96LbtVljtFq0JqH5B92NU+BQeDFrXWBUWlCKAM0wF5GLfIhCLT5kQ3Sl+U0YjRJ7Alqj5hGQaCg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-ppc64-gnu": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.44.0.tgz", + "integrity": "sha512-YcFE8/q/BbrCiIiM5piwbkA6GwJc5QqhMQp2yDrqQ2fuVkZ7CInb1aIijZ/k8EXc72qXMSwKpVlBv1w/MsGO/A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-riscv64-gnu": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.44.0.tgz", + "integrity": "sha512-eOdzs6RqkRzuqNHUX5C8ISN5xfGh4xDww8OEd9YAmc3OWN8oAe5bmlIqQ+rrHLpv58/0BuU48bxkhnIGjA/ATQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-riscv64-musl": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.44.0.tgz", + "integrity": "sha512-YBgNTxntD/QvlFUfgvh8bEdwOhXiquX8gaofZJAwYa/Xp1S1DQrFVZEeck7GFktr24DztsSp8N8WtWCBwxs0Hw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-s390x-gnu": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.44.0.tgz", + "integrity": "sha512-GLIh1R6WHWshl/i4QQDNgj0WtT25aRO4HNUWEoitxiywyRdhTFmFEYT2rXlcl9U6/26vhmOqG5cRlMLG3ocaIA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-x64-gnu": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.44.0.tgz", + "integrity": "sha512-gZOpgTlOsLcLfAF9qgpTr7FIIFSKnQN3hDf/0JvQ4CIwMY7h+eilNjxq/CorqvYcEOu+LRt1W4ZS7KccEHLOdA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-linux-x64-musl": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-musl/-/binding-linux-x64-musl-0.44.0.tgz", + "integrity": "sha512-1CyS9JTB+pCUFYFI6pkQGGZaT/AY5gnhHVrQQLhFba6idP9AzVYm1xbdWfywoldTYvjxQJV6x4SuduCIfP3W+A==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-openharmony-arm64": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-openharmony-arm64/-/binding-openharmony-arm64-0.44.0.tgz", + "integrity": "sha512-bmEv70Ak6jLr1xotCbF5TxIKjsmQaiX+jFRtnGtfA03tJPf6VG3cKh96S21boAt3JZc+Vjx8PYcDuLj39vM2Pw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-win32-arm64-msvc": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.44.0.tgz", + "integrity": "sha512-yWzB+oCpSnP/dmw85eFLAT5o35Ve5pkGS2uF/UCISpIwDqf1xa7OpmtomiqY/Vzg8VyvMbuf6vroF2khF/+1Vg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-win32-ia32-msvc": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.44.0.tgz", + "integrity": "sha512-TcWpo18xEIE3AmIG2kpr3kz5IEhQgnx0lazl2+8L+3eTopOAUevQcmlr4nhguImNWz0OMeOZrYZOhJNCf16nlQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxfmt/binding-win32-x64-msvc": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.44.0.tgz", + "integrity": "sha512-oj8aLkPJZppIM4CMQNsyir9ybM1Xw/CfGPTSsTnzpVGyljgfbdP0EVUlURiGM0BDrmw5psQ6ArmGCcUY/yABaQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm-eabi": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.71.0.tgz", + "integrity": "sha512-ImGmd1njEg4FEJH03jhRnveEegtO3czCtfptvaHivKAZQIYATbVFBrrzbaYMYv0oJioTnxZAZVSyV+oL7W8S2g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-android-arm64": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.71.0.tgz", + "integrity": "sha512-4A5BEexBrwY1YFF8Kiq/lp/wQPRG79G3BWIE1FuWaM5MvmpYSd+7ZySVcKkHdwo0UDzdQGddp6pD9mpctMqLnw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-arm64": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.71.0.tgz", + "integrity": "sha512-9wJA9GJulLwS2usU3CEisI/ESDO1n1z9eyTCvApMDrAkbJ1ve0mORgTMjcWWsKxkzkeZ2N/Gpra5IQE7x8tYgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-darwin-x64": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.71.0.tgz", + "integrity": "sha512-PlLCjS06V0PeJMAJwzjrExw1sYNW9Gch3JtNlcwwZDXGlTYDuwHNN89zYH8LTXFfgkVtsYvs2nv0FqrzyuFDzg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-freebsd-x64": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.71.0.tgz", + "integrity": "sha512-Lhil7bWre0ncxbUoDoxfS0JzpTz17BRQKW7iwoAUY8GJ66+WwJEfYPCFJ1P0WgVZR5/O/b3Q2pENlHOjeXLOGQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-gnueabihf": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.71.0.tgz", + "integrity": "sha512-Oo9/L58PYD3RC0x05d2upAPLllHytTjHQGsnC06P6Ynn7jKkp5mdImQxXdJ3+FnBaKspNpGogzgVsi6g872LiA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm-musleabihf": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.71.0.tgz", + "integrity": "sha512-mSHfyfgJrEbyIR29ejaeS50BdPk+GoNPlC1dckpDiUZbJAIel68sjSMdOt4WY0/gva+ECC7FNITQkxMJU+vSBw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-gnu": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.71.0.tgz", + "integrity": "sha512-n9yY4M2tiy3aij4AqtlnspzpfdpeT5JQfK2/w2d8oyp5W0FRwOb1dIeX99nORNcxGr08iD9bH8N5XFz3I2iy8w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-arm64-musl": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.71.0.tgz", + "integrity": "sha512-fJZrs5sDZtTaPIOiemRQQmo82Ezy+vOGXemPc4Ok7iVVsYsFa7SlW6Z5XN819VfsqBHRm3NJ3rTdnR8+bJYJdQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-ppc64-gnu": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.71.0.tgz", + "integrity": "sha512-cwl7VKGERIy9p+G+AvZdfy/06q0aHXaTt/mMRReC751iuNYJgqKjB7NydXSS30nBT9vtr2tunciOtrR4fD6FUA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-gnu": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.71.0.tgz", + "integrity": "sha512-eZ8ieVXvzGi8jr7+ybQGPK2STw3mldfxZlgA2738iflfB/rzA69sE6m5rDRpQaxC7dpm745Enlh1Tod0QAk9Gg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-riscv64-musl": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.71.0.tgz", + "integrity": "sha512-puMDbQYe6+NXwfMusojoA7CXGn2b3utukmd23PQqc1E3XhVCwyZ+FueSMzDYeNgDV2dUfIVXAAKZBcFDeCL6sA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-s390x-gnu": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.71.0.tgz", + "integrity": "sha512-4NJLxBs1ujISCt3L/1FcywLs73PWtJuw+piD6feK2V6h6OS6P7xu9/sWt1DTRLibe6QCzmfZzmM/2HPORoV/Lg==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-gnu": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.71.0.tgz", + "integrity": "sha512-cFDaiR8L3430qp88tfZnvFlt3KotFhR/DlbIL0nHOMMYiG/9Wy4l+6f7t8G8pTa9bd8Lt8+M0y/qjRQ/xcB74g==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-linux-x64-musl": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.71.0.tgz", + "integrity": "sha512-orfixdt76KlpNly9z0PkWBBNfwjKz+JFVLP/7wnVchlKNU9Dpt9InU/ZggeSej6fC7qwHmHNOGlhLnQXcYoGuA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-openharmony-arm64": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.71.0.tgz", + "integrity": "sha512-9emQu2lAp6yhPB3XuI+++vR+l/o6JR1X+EpxwcumPdQXBWXEPAsquPGL7l158EqU8SebQMXTUa/S5zN98juyHw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-arm64-msvc": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.71.0.tgz", + "integrity": "sha512-bd5kI8spYwTm3BILDtGhi73zoup5dw8MlPQNT8YB3BD5UIsjNe3K9/4ctrzQMX4SZMoK5HgzVLkLJzacEXB7fA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-ia32-msvc": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.71.0.tgz", + "integrity": "sha512-W4HvOHGzVLHcrmFu+bMrJlho+/yrlX5ZNdJZqGe8MEldkQG+RHYhxxad9P4jvWAYFmIqUA5i9DQ8QsJqSU9GIw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxlint/binding-win32-x64-msvc": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.71.0.tgz", + "integrity": "sha512-D2kyEIPHk/G/wiZLnwTVC/sVst+T/lKldVOjAFpgTIBUAOlry72e5OiapDbDBF4LfJLkN5ypJb/8Eu6yJzkveQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, + "node_modules/@pkgr/core": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.3.6.tgz", + "integrity": "sha512-SEeaJLb3qBNF/OaXnaR1NmmBbFYk1zC0ZH/52fATcRPLFg/p791YrcyFFy44Bo9sLaGuSuLp5Q6axbb/O+v/RA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/pkgr" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.3.tgz", + "integrity": "sha512-DT6Z3PhvioeHMvxo+xHc3KtqggrI7CCTXCmC2h/5zUlp5jVitv7XEy+9q5/7v8IolhlioawpMo8Kg0EEBy7J0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.3.tgz", + "integrity": "sha512-0NwgwsjM7LrsuVnXMK3koTpagBNOhloc/BNjKqZjv4V5zI5r13qx69uVhRx+o5Z0yy4Hzq+lpy7TAgUG/ocvrw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.3.tgz", + "integrity": "sha512-YtiBp4disu6V560loT6PjMdiRaWmVvDNrUunAalbiFx2ggeJwxdAsgZMcoGP17uyAsTwAj5V1niksxlHnVQ1Sw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.3.tgz", + "integrity": "sha512-yD3EkEdXk2LypPxnf/kSZHirarsI8gcPzc62SukhR9VJTyvV+F9Q/GxWNuCojc7sXyuVC4DxRGhdDK4X8VSsbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.3.tgz", + "integrity": "sha512-c+8vieQbsD7HNAHKIA34w0GJ9FedFFuJGD+7E6vz7Q3uqAIugL5p45fhlsj4UaAsHpcmlqugBWMhA0/j7o0sIg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.3.tgz", + "integrity": "sha512-50jD0uUwLvur7Zz9LHz17kaAdTPjn5wN93hEgjvmYFRZwiR7ZJYovTd5ipyWJDAnXKvZ+wgc+/Ika6dwSF5OcA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.3.tgz", + "integrity": "sha512-BO9+oPL8K9poZJBfYPsXNtYjPE5uM3qeehT3aFcW4LITOl+iSqhp0abzjR2nWBUNjIZeKXjAEWBZ64WjNoHd6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.3.tgz", + "integrity": "sha512-f3VpLB1vQ0Eo6ecr/6cekLnvYMFF4YBFoVGkfkvPLq1bAkbAwHYQPZKoAmG6OJyTcxxoC+AvezGx/S1obNC0Mw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.3.tgz", + "integrity": "sha512-AmurZ26Pqx/RI9N1gzEOCklkKXl927yjfXWUUS0O7Puh8ARM/Ob8qfrD3qnWksScdw6cSrW5PSHE9DyLu7+PtA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.3.tgz", + "integrity": "sha512-JJpqs8bRGITDOdbkNKnlojzBabbOHrqjSvDr0IVsZObE1lBcPjxItUEY9eWIDbxaJ3cGrXPWGfGkIxFijg/URg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.3.tgz", + "integrity": "sha512-rSJcdjPxzA/by/6/rYs+v+bXU7UjvnbUWz8MJb6kh6+knqB1dCrtHg0uu7C/4haqJvqdkYHQ5IGn+tCH9GLW/g==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.3.tgz", + "integrity": "sha512-hQ3/PYkDJICgevvyNcVrihVeqq7k1Pp3VZ9lY+dauAYUJKO+auqApvANhvR1An9BhmqYKvW2Mu1F9u4DXSMLxQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.3.tgz", + "integrity": "sha512-Elcv/BtML9lXrV6JuKITc/grN2kYV9gjsQpW8Jfw4ioK0TOkjBjye0nnyqQNy9STNaI20lXNaQBRrD5gSgR0Yg==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.3.tgz", + "integrity": "sha512-2DrEfhluH9yhiaFApmsjsjwrSYbNcY1oFTzYSP1a535jDbV98zCFanA/96TBUd0iDFcxGmw9QRExwGCXz3U+/g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.3.tgz", + "integrity": "sha512-OL4OMk7UPXOeVGGd3qo5zJyPIljf4AFgk5QAkPPS+OoLuOOozhuaQGC18MxVTnw/06q93gShAJzlwnSCY9YtqA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", + "integrity": "sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.2.tgz", + "integrity": "sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.2.tgz", + "integrity": "sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.2.tgz", + "integrity": "sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.2.tgz", + "integrity": "sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.2.tgz", + "integrity": "sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.2.tgz", + "integrity": "sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.2.tgz", + "integrity": "sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.2.tgz", + "integrity": "sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.2.tgz", + "integrity": "sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.2.tgz", + "integrity": "sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.2.tgz", + "integrity": "sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.2.tgz", + "integrity": "sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.2.tgz", + "integrity": "sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.2.tgz", + "integrity": "sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.2.tgz", + "integrity": "sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.2.tgz", + "integrity": "sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.2.tgz", + "integrity": "sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.2.tgz", + "integrity": "sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.2.tgz", + "integrity": "sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.2.tgz", + "integrity": "sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.2.tgz", + "integrity": "sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.2.tgz", + "integrity": "sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.2.tgz", + "integrity": "sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.2.tgz", + "integrity": "sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tailwindcss/container-queries": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/container-queries/-/container-queries-0.1.1.tgz", + "integrity": "sha512-p18dswChx6WnTSaJCSGx6lTmrGzNNvm2FtXmiO6AuA1V4U5REyoqwmT6kgAsIMdjo07QdAfYXHJ4hnMtfHzWgA==", + "dev": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "tailwindcss": ">=3.2.0" + } + }, + "node_modules/@tailwindcss/typography": { + "version": "0.5.20", + "resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.20.tgz", + "integrity": "sha512-hwbzQuNUfcPvbegQFatVPl/MY/tcM9KLl963hQ5laJKPh81TEZ1+dNG9PirGvcaDBkp+BCshExAyKVPW91dozw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "postcss-selector-parser": "6.0.10" + }, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || >=4.0.0 || insiders" + } + }, + "node_modules/@toon-format/toon": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@toon-format/toon/-/toon-2.3.0.tgz", + "integrity": "sha512-/Ew9etdRQKVMnm9fDaCG0JjyAOK/O7T0M97oum1aW4W+UR8ZhVVPBanIV7oWgHBiGlnVxV9M55PWQCHofDV07w==", + "license": "MIT" + }, + "node_modules/@turbo/darwin-64": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@turbo/darwin-64/-/darwin-64-2.10.0.tgz", + "integrity": "sha512-EwvHThXzpY0KGd1/NAmuewI5D+aVa3Rl/OlxE36yfjUKb/+ySrfJrSlEFt8aD1OXwnnaHnQnPKHFndor0Zxlsg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true + }, + "node_modules/@turbo/darwin-arm64": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@turbo/darwin-arm64/-/darwin-arm64-2.10.0.tgz", + "integrity": "sha512-9d2fTyyG0lf5Wq1bwJA9qUaeecViMkLcdctWaMMmCkxZ/JqypmqOwK3W6vmejeKVgkr06gSoiX8bD+xN5Jpxcg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true + }, + "node_modules/@turbo/linux-64": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@turbo/linux-64/-/linux-64-2.10.0.tgz", + "integrity": "sha512-sZBtjMuufitanjzi6UssoUpJMnnPlLMcdcJj3m3ptNsSq31Xh7MnjhwA5nWvLDTfEFg8GPcbYFXMo8vSdKRfqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true + }, + "node_modules/@turbo/linux-arm64": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@turbo/linux-arm64/-/linux-arm64-2.10.0.tgz", + "integrity": "sha512-vkq/Z8R+1DQ+kifWFa810IjRy2NNBVvha3cg9sWA3nFh6nnGrHSMnnJKrzH7c/No9kq4Jb55Ru44YKsCSBgrKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true + }, + "node_modules/@turbo/windows-64": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@turbo/windows-64/-/windows-64-2.10.0.tgz", + "integrity": "sha512-CRUEguLWxFQHptYZS7HjPhNhAFawfea07iR+xAQ5e4klgLrPCMdexBkXwSCwOxqTFknJ7RZFN3gOaADsw+Gttg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true + }, + "node_modules/@turbo/windows-arm64": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@turbo/windows-arm64/-/windows-arm64-2.10.0.tgz", + "integrity": "sha512-dVHGaf9F8twzgibcBqKoADT/LLqf9++jDb+hq/LPWWaOmRpp4M+/pVOm7vy4z9D++xg8eaxWLT0+wQxFwhYu9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/@types/node": { + "version": "24.13.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.2.tgz", + "integrity": "sha512-fRa09kZTgu8o71KFcDjUFuc7F+dEbZYZmkI0mg5YBTRs0yMKjYHsq/c0urDKeDb+D5qVgXOdFcuu+DZPKOITwA==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, + "node_modules/@types/set-cookie-parser": { + "version": "2.4.10", + "resolved": "https://registry.npmjs.org/@types/set-cookie-parser/-/set-cookie-parser-2.4.10.tgz", + "integrity": "sha512-GGmQVGpQWUe5qglJozEjZV/5dyxbOOZ0LHe/lqyWssB88Y4svNfst0uqBVscdDeIKl5Jy5+aPSvy7mI9tYRguw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/statuses": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.6.tgz", + "integrity": "sha512-xMAgYwceFhRA2zY+XbEA7mxYbA093wdiW8Vu6gZPGWy9cmOyU9XesH1tNcEWsKFd5Vzrqx5T3D38PWx1FIIXkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.62.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.62.0.tgz", + "integrity": "sha512-wexnCqiTg7BOGtbLDftYpRWlmLq4xfoMd7BKFR6Y75sZS3QmRKLdN3yWLhmIYgqMmP/OXWpj3H8odkb5nGURCQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.62.0", + "@typescript-eslint/types": "^8.62.0", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.62.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.62.0.tgz", + "integrity": "sha512-1lX38kNxXIRb8mEc3lbq5mdHq1Pf2+U0nFU65KfT18mtPxxl0fvjuEE92mHuXPuCtElJhOrddOpyMlM3Z0umEA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/types": "8.62.0", + "@typescript-eslint/visitor-keys": "8.62.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.62.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.62.0.tgz", + "integrity": "sha512-y2GAdB6ykaXUvuspbYnizQc4oDDz0Tz/Yc7iWrXf9mx8vm/L/0vLHCe0tS2boG96Zy+DivnVDQ9ZUEWoHqqx1g==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.62.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.62.0.tgz", + "integrity": "sha512-+g5O3j0w2ldzC86Pv6fvbO/xhAonbJFIdf/MKQ1d30gndlsVzUOE83ldfSE15Qrl9fhFjK6AovHs5Wpp6vx86w==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/types": "8.62.0", + "@typescript-eslint/typescript-estree": "8.62.0", + "@typescript-eslint/utils": "8.62.0", + "debug": "^4.4.3", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.62.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.62.0.tgz", + "integrity": "sha512-KvAclkktORPvM54TgLgA4z9HIV1M8zOgw9ZVNXl9f/8dLYfXYX1wkMXP7qmabpijQRV5bHJLOmoyGQbLMaUYeg==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.62.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.62.0.tgz", + "integrity": "sha512-+hVbNxtW64pIcZWDPGbyaKF7vp2IBTVY5ma1blwwksrjdsbdqqEKvJWMGbBofei4F6Dovx1M0RJgoFeNu2279A==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/project-service": "8.62.0", + "@typescript-eslint/tsconfig-utils": "8.62.0", + "@typescript-eslint/types": "8.62.0", + "@typescript-eslint/visitor-keys": "8.62.0", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.62.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.62.0.tgz", + "integrity": "sha512-82r66fi9zYwZ+mTq3vKgwjbZ1PVk/DJzrXFLpG6RnBbdvH8TEGVHIs9H4d2drhkOzf0syZuD/OZvvlu6GDbP4g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.62.0", + "@typescript-eslint/types": "8.62.0", + "@typescript-eslint/typescript-estree": "8.62.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.62.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.62.0.tgz", + "integrity": "sha512-CY3uyFSRbcQv3nnSv8S0+lDftMVz6P963PoRlxrV7ew/Md564g9ut60PYzdLM5qW4jFn93GBF+Soi90ISAN+GQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@typescript-eslint/types": "8.62.0", + "eslint-visitor-keys": "^5.0.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@vitest/expect": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.9.tgz", + "integrity": "sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.9.tgz", + "integrity": "sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.9", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.9.tgz", + "integrity": "sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.9.tgz", + "integrity": "sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.9", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.9.tgz", + "integrity": "sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.9", + "@vitest/utils": "4.1.9", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.9.tgz", + "integrity": "sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.9.tgz", + "integrity": "sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.9", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", + "integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^8" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/adm-zip": { + "version": "0.5.18", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.18.tgz", + "integrity": "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=12.0" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true, + "license": "MIT" + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0", + "optional": true + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/birecord": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/birecord/-/birecord-0.1.1.tgz", + "integrity": "sha512-VUpsf/qykW0heRlC8LooCq28Kxn3mAqKohhDG/49rrsQ1dT1CXyj/pgXS+5BSRzFTR/3DyIBOqQOrGyZOh71Aw==", + "dev": true, + "license": "(MIT OR Apache-2.0)", + "peer": true + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC", + "optional": true + }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT", + "optional": true + }, + "node_modules/brace-expansion": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/bundle-require": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-5.1.0.tgz", + "integrity": "sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "load-tsconfig": "^0.2.3" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "peerDependencies": { + "esbuild": ">=0.18" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/camelcase-css": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", + "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "license": "MIT", + "optional": true, + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.19.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "optional": true, + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/cjs-module-lexer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", + "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", + "license": "MIT" + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.1.tgz", + "integrity": "sha512-rwHwUfXL40Chm1r08yrhU3qpUvdVlgkKNeyeGPOxnW8/SyVDvgRaed/Uz54AqWNaTCAThlj6QAs3TZcKI0xDEw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/compare-versions": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-6.1.1.tgz", + "integrity": "sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/confbox": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.4.tgz", + "integrity": "sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/consola": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz", + "integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.18.0 || >=16.10.0" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "optional": true, + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "optional": true, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/dayjs": { + "version": "1.11.21", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.21.tgz", + "integrity": "sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==", + "license": "MIT" + }, + "node_modules/dc-polyfill": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/dc-polyfill/-/dc-polyfill-0.1.11.tgz", + "integrity": "sha512-TyyeGcjx0YeThAI9fTFtgsvj5qd4R+aGfVmXiUhevbgzWFDr7IK4tv4YjE6jaGzLHQTchk4h7DHdr5q4WGgaZw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/dd-trace": { + "version": "5.110.0", + "resolved": "https://registry.npmjs.org/dd-trace/-/dd-trace-5.110.0.tgz", + "integrity": "sha512-/PeVFa9lSbaJ2b1M233nX0LF+5RMRwdjv22uotNbdtWz7lQk1om8U8ngm07Cf/hrzVMMPvA3Cufy3iZgZOTNkA==", + "license": "(Apache-2.0 OR BSD-3-Clause)", + "dependencies": { + "dc-polyfill": "^0.1.11", + "import-in-the-middle": "^3.1.0", + "opentracing": ">=0.14.7" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@datadog/libdatadog": "0.9.4", + "@datadog/native-appsec": "11.0.1", + "@datadog/native-iast-taint-tracking": "4.2.0", + "@datadog/native-metrics": "3.1.2", + "@datadog/openfeature-node-server": "2.0.0", + "@datadog/pprof": "5.15.1", + "@datadog/wasm-js-rewriter": "5.0.1", + "@opentelemetry/api": ">=1.0.0 <1.10.0", + "@opentelemetry/api-logs": "<1.0.0", + "oxc-parser": "^0.132.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "optional": true, + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "optional": true, + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "devOptional": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT", + "optional": true + }, + "node_modules/didyoumean": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", + "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", + "dev": true, + "license": "Apache-2.0", + "peer": true + }, + "node_modules/dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "optional": true, + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause", + "optional": true + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "optional": true, + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "optional": true, + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dotenv": { + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz", + "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/encoding-sniffer": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", + "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/encoding-sniffer/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/enhanced-resolve": { + "version": "5.24.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.24.1.tgz", + "integrity": "sha512-7DdUaTjmNwMcH2gLr1qycesKII3BK4RLy/mdAb7x10Lq7bR4aNKHt1BR1ZALSv0rPM/hF5wYF0PhGop/rJm8vw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "optional": true, + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz", + "integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT", + "optional": true + }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "devOptional": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.5.0.tgz", + "integrity": "sha512-1y+7C+vi12bUK1IpZeaV3gsH9fHLBmPvYmPx42pvT/E9yG0IC8g3PUZZgp0+JLJl7ZDK0flc2gc+Aw9dpCvIsQ==", + "dev": true, + "license": "MIT", + "peer": true, + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.6.0", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.2", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "minimatch": "^10.2.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-plugin-react-dom": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-dom/-/eslint-plugin-react-dom-4.2.3.tgz", + "integrity": "sha512-7FCB+kx0iwWw2OOb0aDrXU4Eds5ihrq6UACNVMmtv5c4qd82n+wRGQwXBQKlTbwR9gpfn3HRDlaofZX93gShlA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-react/ast": "4.2.3", + "@eslint-react/core": "4.2.3", + "@eslint-react/jsx": "4.2.3", + "@eslint-react/shared": "4.2.3", + "@eslint-react/var": "4.2.3", + "@typescript-eslint/scope-manager": "^8.58.0", + "@typescript-eslint/types": "^8.58.0", + "@typescript-eslint/utils": "^8.58.0", + "compare-versions": "^6.1.1", + "ts-pattern": "^5.9.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/eslint-plugin-react-jsx": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-jsx/-/eslint-plugin-react-jsx-4.2.3.tgz", + "integrity": "sha512-IUiYO1Qm/NDo/CVBa/nOP6lKJvPtDz7ucKsfcmrDYFS7NGyLJedubB4vFtMGZ2XGBFJeEYLnSo7Y+89b0qdynA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-react/ast": "4.2.3", + "@eslint-react/core": "4.2.3", + "@eslint-react/jsx": "4.2.3", + "@eslint-react/shared": "4.2.3", + "@eslint-react/var": "4.2.3", + "@typescript-eslint/scope-manager": "^8.58.0", + "@typescript-eslint/types": "^8.58.0", + "@typescript-eslint/utils": "^8.58.0", + "compare-versions": "^6.1.1", + "ts-pattern": "^5.9.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/eslint-plugin-react-naming-convention": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-naming-convention/-/eslint-plugin-react-naming-convention-4.2.3.tgz", + "integrity": "sha512-H4eq0ajs+K+tgn6/eeglkLN3HBWm4QyWbJ2jbwPo75gyPmEP7Xvr0jslcnAwnmQfiiKX+KqKuiOMAqOr0SXubg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-react/ast": "4.2.3", + "@eslint-react/core": "4.2.3", + "@eslint-react/shared": "4.2.3", + "@eslint-react/var": "4.2.3", + "@typescript-eslint/scope-manager": "^8.58.0", + "@typescript-eslint/type-utils": "^8.58.0", + "@typescript-eslint/types": "^8.58.0", + "@typescript-eslint/utils": "^8.58.0", + "compare-versions": "^6.1.1", + "string-ts": "^2.3.1", + "ts-pattern": "^5.9.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/eslint-plugin-react-web-api": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-web-api/-/eslint-plugin-react-web-api-4.2.3.tgz", + "integrity": "sha512-iHXFiURfokcTicZ9DZsQHCV9BuVRqve7GFYNBBD5AVFzEWseCV+lXc6y2EoXxsQW8WfYhAwTZ5Yhr+fKJR7t1w==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-react/ast": "4.2.3", + "@eslint-react/core": "4.2.3", + "@eslint-react/shared": "4.2.3", + "@eslint-react/var": "4.2.3", + "@typescript-eslint/scope-manager": "^8.58.0", + "@typescript-eslint/types": "^8.58.0", + "@typescript-eslint/utils": "^8.58.0", + "birecord": "^0.1.1", + "ts-pattern": "^5.9.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/eslint-plugin-react-x": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-x/-/eslint-plugin-react-x-4.2.3.tgz", + "integrity": "sha512-kJZXa5QsGA4FzuTyKLKjFt9nm78CZcfHshfgfSXjVOshvlVGeg1RWyNZnXDW3hASdZ/REsPg2mGFYqwUPXnJ5Q==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@eslint-react/ast": "4.2.3", + "@eslint-react/core": "4.2.3", + "@eslint-react/jsx": "4.2.3", + "@eslint-react/shared": "4.2.3", + "@eslint-react/var": "4.2.3", + "@typescript-eslint/scope-manager": "^8.58.0", + "@typescript-eslint/type-utils": "^8.58.0", + "@typescript-eslint/types": "^8.58.0", + "@typescript-eslint/utils": "^8.58.0", + "compare-versions": "^6.1.1", + "string-ts": "^2.3.1", + "ts-api-utils": "^2.5.0", + "ts-pattern": "^5.9.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "eslint": "^10.0.0", + "typescript": "*" + } + }, + "node_modules/eslint-plugin-simple-import-sort": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-simple-import-sort/-/eslint-plugin-simple-import-sort-13.0.0.tgz", + "integrity": "sha512-McAc+/Nlvcg4byY/CABGH8kqnefWBj8s3JA2okEtz8ixbECQgU46p0HkTUKa4YS7wvgGceimlc34p1nXqbWqtA==", + "dev": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "eslint": ">=5.0.0" + } + }, + "node_modules/eslint-plugin-tailwindcss": { + "version": "3.18.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-tailwindcss/-/eslint-plugin-tailwindcss-3.18.3.tgz", + "integrity": "sha512-lqjNX7mt1Ip2qR236hvhbZ9ff2TFLUWou+tBHz82SA1nWFzOZSoEOI+9UBZmuf2977r2MMp9/y3/broyz8AYig==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "fast-glob": "^3.2.5", + "postcss": "^8.4.4", + "synckit": "^0.11.4", + "tailwind-api-utils": "^1.0.3" + }, + "engines": { + "node": ">=18.12.0" + }, + "peerDependencies": { + "tailwindcss": "^3.4.0 || ^4.0.0" + } + }, + "node_modules/eslint-plugin-turbo": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-turbo/-/eslint-plugin-turbo-2.10.0.tgz", + "integrity": "sha512-0HoDr8jdJ5oN9n2ip1DMDtFwmkXYc1ry/JsqGLA1fv6w3kdpvUinPm1FiJPcNnXLxO38N6It//YhW3tfk5j8TQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "dotenv": "16.0.3" + }, + "peerDependencies": { + "eslint": ">6.6.0", + "turbo": ">2.0.0" + } + }, + "node_modules/eslint-scope": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/espree": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "acorn": "^8.16.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^5.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "peer": true, + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.0.tgz", + "integrity": "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", + "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/exsolve": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.1.0.tgz", + "integrity": "sha512-D+42+T12DdIlJM3uepa55qGiL3sYdLBOxIl2ifQCzCHz4c7eiolaHsi3BIqEr7JxBzxv2pYZQX9kw16ziMcEmw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/fast-string-truncated-width": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz", + "integrity": "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-string-width": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz", + "integrity": "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-truncated-width": "^3.0.2" + } + }, + "node_modules/fast-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", + "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fast-wrap-ansi": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz", + "integrity": "sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-width": "^3.0.2" + } + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fix-dts-default-cjs-exports": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/fix-dts-default-cjs-exports/-/fix-dts-default-cjs-exports-1.0.1.tgz", + "integrity": "sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "magic-string": "^0.30.17", + "mlly": "^1.7.4", + "rollup": "^4.34.8" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/flatted": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", + "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", + "dev": true, + "license": "ISC", + "peer": true + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "peer": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC", + "peer": true + }, + "node_modules/graphql": { + "version": "16.14.2", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.14.2.tgz", + "integrity": "sha512-Chq1s4CY7jmh8gO2qvLIJyfCDIN+EHLFW/9iShnp1z8FjBQMoodWP1kDC36VAMXXIvAjj4ARa7ntfAV2BrjsbA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC", + "optional": true + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "optional": true, + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/headers-polyfill": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-5.0.1.tgz", + "integrity": "sha512-1TJ6Fih/b8h5TIcv+1+Hw0PDQWJTKDKzFZzcKOiW1wJza3XoAQlkCuXLbymPYB8+ZQyw8mHvdw560e8zVFIWyA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/set-cookie-parser": "^2.4.10", + "set-cookie-parser": "^3.0.1" + } + }, + "node_modules/hono": { + "version": "4.12.27", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.27.tgz", + "integrity": "sha512-1yrb/+w6HWQJrUCLkJ2IF5jNIPvvFkblV5RNOYl6bV+OA6p9GLcMpHFFGTosSvHvcAUibuUukRqhlYI4z32C7Q==", + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "optional": true, + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-in-the-middle": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-3.2.0.tgz", + "integrity": "sha512-vR2B6HKIhaBjcZr2bLpFiJ1VbzOlRQ7aby4/gw5WPIzToLjqpfWw3VJ4sk1uDchoOODEirvO2jyrSPtUSL5CrQ==", + "license": "Apache-2.0", + "dependencies": { + "acorn": "^8.15.0", + "acorn-import-attributes": "^1.9.5", + "cjs-module-lexer": "^2.2.0", + "module-details-from-path": "^1.0.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ioredis": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.11.1.tgz", + "integrity": "sha512-ehuGcf94bQXhfagULNXrJdfnWO38v070jxSx/qE87Kjzmu2fU7ro5EFAb+OPituLqgfyuQaym5DlrNydW2sJ9A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ioredis/commands": "1.10.0", + "cluster-key-slot": "1.1.1", + "debug": "4.4.3", + "denque": "2.1.0", + "redis-errors": "1.2.0", + "redis-parser": "3.0.0", + "standard-as-callback": "2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ip-address": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.16.2", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", + "integrity": "sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "hasown": "^2.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-node-process": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/jose": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz", + "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/js-yaml": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz", + "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "optional": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "license": "BSD-2-Clause" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC", + "optional": true + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/load-tsconfig": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/load-tsconfig/-/load-tsconfig-0.2.5.tgz", + "integrity": "sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/local-pkg": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-1.2.1.tgz", + "integrity": "sha512-++gUqRDEvcnN6Zhqrr+y/CkVEHhlrR96vZn3nZZPYzMcBUyBtTKzB9NadClFIsIVSsu+3i9tfk/erqy9kAmt7Q==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "mlly": "^1.7.4", + "pkg-types": "^2.3.0", + "quansync": "^0.2.11" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "optional": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "optional": true, + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "dev": true, + "license": "BlueOak-1.0.0", + "peer": true, + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/mlly": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.2.tgz", + "integrity": "sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.16.0", + "pathe": "^2.0.3", + "pkg-types": "^1.3.1", + "ufo": "^1.6.3" + } + }, + "node_modules/mlly/node_modules/confbox": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", + "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/mlly/node_modules/pkg-types": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", + "integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "confbox": "^0.1.8", + "mlly": "^1.7.4", + "pathe": "^2.0.1" + } + }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/msw": { + "version": "2.14.6", + "resolved": "https://registry.npmjs.org/msw/-/msw-2.14.6.tgz", + "integrity": "sha512-ALe+N10S72cyx94cMcy3Zs4HhXCj35sgeAL4c+WTvKi0zWnbd8/h0lcFqv0mb2P+aSgAdD7p9HzvA0DiUPxsyg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "@inquirer/confirm": "^6.0.11", + "@mswjs/interceptors": "^0.41.3", + "@open-draft/deferred-promise": "^3.0.0", + "@types/statuses": "^2.0.6", + "cookie": "^1.1.1", + "graphql": "^16.13.2", + "headers-polyfill": "^5.0.1", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.3", + "path-to-regexp": "^6.3.0", + "picocolors": "^1.1.1", + "rettime": "^0.11.11", + "statuses": "^2.0.2", + "strict-event-emitter": "^0.5.1", + "tough-cookie": "^6.0.1", + "type-fest": "^5.5.0", + "until-async": "^3.0.2", + "yargs": "^17.7.2" + }, + "bin": { + "msw": "cli/index.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/mswjs" + }, + "peerDependencies": { + "typescript": ">= 4.8.x" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/msw/node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/msw/node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/mute-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-3.0.0.tgz", + "integrity": "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", + "license": "MIT", + "optional": true + }, + "node_modules/node-gyp-build": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-3.9.0.tgz", + "integrity": "sha512-zLcTg6P4AbcHPq465ZMFNXx7XpKKJh+7kkN699NiQWisR2uWYOWNWqRHAmbnmKiL4e9aLSlmy5U7rEMUXV59+A==", + "license": "MIT", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "optional": true, + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/obug": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.3.tgz", + "integrity": "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz", + "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==", + "license": "MIT", + "optional": true + }, + "node_modules/onnxruntime-node": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz", + "integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.5.16", + "global-agent": "^3.0.0", + "onnxruntime-common": "1.24.3" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.26.0-dev.20260416-b7804b056c", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.26.0-dev.20260416-b7804b056c.tgz", + "integrity": "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==", + "license": "MIT", + "optional": true, + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.24.0-dev.20251116-b39e144322", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.0-dev.20251116-b39e144322.tgz", + "integrity": "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==", + "license": "MIT", + "optional": true + }, + "node_modules/opentracing": { + "version": "0.14.7", + "resolved": "https://registry.npmjs.org/opentracing/-/opentracing-0.14.7.tgz", + "integrity": "sha512-vz9iS7MJ5+Bp1URw8Khvdyw1H/hGvzHWlKQ7eRrQojSCDL1/SrWfrY9QebLw97n2deyRtzHRC3MkQfVNUCo91Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/outvariant": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.3.tgz", + "integrity": "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA==", + "dev": true, + "license": "MIT" + }, + "node_modules/oxc-parser": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.132.0.tgz", + "integrity": "sha512-+0LAPHaqtfQlvWdpaAa09SmOaZZgP8C552xosEkGJ4+ruEwP1Vgx+sqBgcBCNfR6KDCmagGOZTde8wmAvcI/Hg==", + "license": "MIT", + "optional": true, + "dependencies": { + "@oxc-project/types": "^0.132.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxc-parser/binding-android-arm-eabi": "0.132.0", + "@oxc-parser/binding-android-arm64": "0.132.0", + "@oxc-parser/binding-darwin-arm64": "0.132.0", + "@oxc-parser/binding-darwin-x64": "0.132.0", + "@oxc-parser/binding-freebsd-x64": "0.132.0", + "@oxc-parser/binding-linux-arm-gnueabihf": "0.132.0", + "@oxc-parser/binding-linux-arm-musleabihf": "0.132.0", + "@oxc-parser/binding-linux-arm64-gnu": "0.132.0", + "@oxc-parser/binding-linux-arm64-musl": "0.132.0", + "@oxc-parser/binding-linux-ppc64-gnu": "0.132.0", + "@oxc-parser/binding-linux-riscv64-gnu": "0.132.0", + "@oxc-parser/binding-linux-riscv64-musl": "0.132.0", + "@oxc-parser/binding-linux-s390x-gnu": "0.132.0", + "@oxc-parser/binding-linux-x64-gnu": "0.132.0", + "@oxc-parser/binding-linux-x64-musl": "0.132.0", + "@oxc-parser/binding-openharmony-arm64": "0.132.0", + "@oxc-parser/binding-wasm32-wasi": "0.132.0", + "@oxc-parser/binding-win32-arm64-msvc": "0.132.0", + "@oxc-parser/binding-win32-ia32-msvc": "0.132.0", + "@oxc-parser/binding-win32-x64-msvc": "0.132.0" + } + }, + "node_modules/oxc-parser/node_modules/@oxc-project/types": { + "version": "0.132.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.132.0.tgz", + "integrity": "sha512-FESMOxil5Se014ui/Eq8fT5uHJo6nIRwH0PfJrZJXs6Gek3ZVFOrpUv3YIZT20m+extU98Hg1Ym72U58rlsxUQ==", + "license": "MIT", + "optional": true, + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/oxfmt": { + "version": "0.44.0", + "resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.44.0.tgz", + "integrity": "sha512-lnncqvHewyRvaqdrnntVIrZV2tEddz8lbvPsQzG/zlkfvgZkwy0HP1p/2u1aCDToeg1jb9zBpbJdfkV73Itw+w==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinypool": "2.1.0" + }, + "bin": { + "oxfmt": "bin/oxfmt" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxfmt/binding-android-arm-eabi": "0.44.0", + "@oxfmt/binding-android-arm64": "0.44.0", + "@oxfmt/binding-darwin-arm64": "0.44.0", + "@oxfmt/binding-darwin-x64": "0.44.0", + "@oxfmt/binding-freebsd-x64": "0.44.0", + "@oxfmt/binding-linux-arm-gnueabihf": "0.44.0", + "@oxfmt/binding-linux-arm-musleabihf": "0.44.0", + "@oxfmt/binding-linux-arm64-gnu": "0.44.0", + "@oxfmt/binding-linux-arm64-musl": "0.44.0", + "@oxfmt/binding-linux-ppc64-gnu": "0.44.0", + "@oxfmt/binding-linux-riscv64-gnu": "0.44.0", + "@oxfmt/binding-linux-riscv64-musl": "0.44.0", + "@oxfmt/binding-linux-s390x-gnu": "0.44.0", + "@oxfmt/binding-linux-x64-gnu": "0.44.0", + "@oxfmt/binding-linux-x64-musl": "0.44.0", + "@oxfmt/binding-openharmony-arm64": "0.44.0", + "@oxfmt/binding-win32-arm64-msvc": "0.44.0", + "@oxfmt/binding-win32-ia32-msvc": "0.44.0", + "@oxfmt/binding-win32-x64-msvc": "0.44.0" + } + }, + "node_modules/oxlint": { + "version": "1.71.0", + "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.71.0.tgz", + "integrity": "sha512-U1m1X+C0vDj7DC1e13IoZULzEcPczE7UOMTs8VlZGHUEIUaSTZKo5qkPsQEfzpgnQ29Pea/w3Xntk62UCecxZw==", + "dev": true, + "license": "MIT", + "bin": { + "oxlint": "bin/oxlint" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxlint/binding-android-arm-eabi": "1.71.0", + "@oxlint/binding-android-arm64": "1.71.0", + "@oxlint/binding-darwin-arm64": "1.71.0", + "@oxlint/binding-darwin-x64": "1.71.0", + "@oxlint/binding-freebsd-x64": "1.71.0", + "@oxlint/binding-linux-arm-gnueabihf": "1.71.0", + "@oxlint/binding-linux-arm-musleabihf": "1.71.0", + "@oxlint/binding-linux-arm64-gnu": "1.71.0", + "@oxlint/binding-linux-arm64-musl": "1.71.0", + "@oxlint/binding-linux-ppc64-gnu": "1.71.0", + "@oxlint/binding-linux-riscv64-gnu": "1.71.0", + "@oxlint/binding-linux-riscv64-musl": "1.71.0", + "@oxlint/binding-linux-s390x-gnu": "1.71.0", + "@oxlint/binding-linux-x64-gnu": "1.71.0", + "@oxlint/binding-linux-x64-musl": "1.71.0", + "@oxlint/binding-openharmony-arm64": "1.71.0", + "@oxlint/binding-win32-arm64-msvc": "1.71.0", + "@oxlint/binding-win32-ia32-msvc": "1.71.0", + "@oxlint/binding-win32-x64-msvc": "1.71.0" + }, + "peerDependencies": { + "oxlint-tsgolint": ">=0.22.1", + "vite-plus": "*" + }, + "peerDependenciesMeta": { + "oxlint-tsgolint": { + "optional": true + }, + "vite-plus": { + "optional": true + } + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "optional": true, + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "optional": true, + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", + "optional": true, + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "optional": true, + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/pg": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz", + "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.15.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz", + "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz", + "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pino": { + "version": "9.14.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.14.0.tgz", + "integrity": "sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^3.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz", + "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", + "license": "MIT" + }, + "node_modules/pirates": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", + "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/pkg-types": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-2.3.1.tgz", + "integrity": "sha512-y+ichcgc2LrADuhLNAx8DFjVfgz91pRxfZdI3UDhxHvcVEZsenLO+7XaU5vOp0u/7V/wZ+plyuQxtrDlZJ+yeg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "confbox": "^0.2.4", + "exsolve": "^1.0.8", + "pathe": "^2.0.3" + } + }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT", + "optional": true + }, + "node_modules/postcss": { + "version": "8.5.15", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", + "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-import": { + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-15.1.0.tgz", + "integrity": "sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "postcss-value-parser": "^4.0.0", + "read-cache": "^1.0.0", + "resolve": "^1.1.7" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "postcss": "^8.0.0" + } + }, + "node_modules/postcss-js": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.1.0.tgz", + "integrity": "sha512-oIAOTqgIo7q2EOwbhb8UalYePMvYoIeRY2YKntdpFQXNosSu3vLrniGgmH9OKs/qAkfoj5oB3le/7mINW1LCfw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "camelcase-css": "^2.0.1" + }, + "engines": { + "node": "^12 || ^14 || >= 16" + }, + "peerDependencies": { + "postcss": "^8.4.21" + } + }, + "node_modules/postcss-load-config": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-6.0.1.tgz", + "integrity": "sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "lilconfig": "^3.1.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "jiti": ">=1.21.0", + "postcss": ">=8.0.9", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + }, + "postcss": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/postcss-nested": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.2.0.tgz", + "integrity": "sha512-HQbt28KulC5AJzG+cZtj9kvKB93CFCdLvog1WFLf1D+xmMvPGlBstkpTEZfK5+AN9hfJocyBFCNiqyS48bpgzQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "postcss-selector-parser": "^6.1.1" + }, + "engines": { + "node": ">=12.0" + }, + "peerDependencies": { + "postcss": "^8.2.14" + } + }, + "node_modules/postcss-nested/node_modules/postcss-selector-parser": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.0.10", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz", + "integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pprof-format": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/pprof-format/-/pprof-format-2.2.2.tgz", + "integrity": "sha512-hd90rHVDhNOhgHTmazVzDSVwTLOBjpZQ26AO/0j46sAFZ9uSWY0DcK2zJcwnuo2R6EzufBbOoWlPazA5nSyHcg==", + "license": "MIT", + "optional": true + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/protobufjs": { + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", + "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/qs": { + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/quansync": { + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/quansync/-/quansync-0.2.11.tgz", + "integrity": "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT", + "peer": true + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "peer": true + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/read-cache": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", + "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "pify": "^2.3.0" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dev": true, + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.12", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", + "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "es-errors": "^1.3.0", + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/rettime": { + "version": "0.11.11", + "resolved": "https://registry.npmjs.org/rettime/-/rettime-0.11.11.tgz", + "integrity": "sha512-ILJRqVWBCTlg9r42fFgwVZx1gnFAcQF8mRoMkbgQfIrjEDf9nbBFDFx00oloOa+Q869FUtaYDXZvEfnecQSCoQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/rolldown": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.3.tgz", + "integrity": "sha512-1F1eEtUBtFvcGm1HQ9TiUIUHPQG7mSAODrhIzjxoUEFuo8OcbrGLiVLkevNgj84TE4lnHvnumwFjhJO5Eu135g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.137.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.1.3", + "@rolldown/binding-darwin-arm64": "1.1.3", + "@rolldown/binding-darwin-x64": "1.1.3", + "@rolldown/binding-freebsd-x64": "1.1.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.3", + "@rolldown/binding-linux-arm64-gnu": "1.1.3", + "@rolldown/binding-linux-arm64-musl": "1.1.3", + "@rolldown/binding-linux-ppc64-gnu": "1.1.3", + "@rolldown/binding-linux-s390x-gnu": "1.1.3", + "@rolldown/binding-linux-x64-gnu": "1.1.3", + "@rolldown/binding-linux-x64-musl": "1.1.3", + "@rolldown/binding-openharmony-arm64": "1.1.3", + "@rolldown/binding-wasm32-wasi": "1.1.3", + "@rolldown/binding-win32-arm64-msvc": "1.1.3", + "@rolldown/binding-win32-x64-msvc": "1.1.3" + } + }, + "node_modules/rollup": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.2.tgz", + "integrity": "sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.62.2", + "@rollup/rollup-android-arm64": "4.62.2", + "@rollup/rollup-darwin-arm64": "4.62.2", + "@rollup/rollup-darwin-x64": "4.62.2", + "@rollup/rollup-freebsd-arm64": "4.62.2", + "@rollup/rollup-freebsd-x64": "4.62.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.62.2", + "@rollup/rollup-linux-arm-musleabihf": "4.62.2", + "@rollup/rollup-linux-arm64-gnu": "4.62.2", + "@rollup/rollup-linux-arm64-musl": "4.62.2", + "@rollup/rollup-linux-loong64-gnu": "4.62.2", + "@rollup/rollup-linux-loong64-musl": "4.62.2", + "@rollup/rollup-linux-ppc64-gnu": "4.62.2", + "@rollup/rollup-linux-ppc64-musl": "4.62.2", + "@rollup/rollup-linux-riscv64-gnu": "4.62.2", + "@rollup/rollup-linux-riscv64-musl": "4.62.2", + "@rollup/rollup-linux-s390x-gnu": "4.62.2", + "@rollup/rollup-linux-x64-gnu": "4.62.2", + "@rollup/rollup-linux-x64-musl": "4.62.2", + "@rollup/rollup-openbsd-x64": "4.62.2", + "@rollup/rollup-openharmony-arm64": "4.62.2", + "@rollup/rollup-win32-arm64-msvc": "4.62.2", + "@rollup/rollup-win32-ia32-msvc": "4.62.2", + "@rollup/rollup-win32-x64-gnu": "4.62.2", + "@rollup/rollup-win32-x64-msvc": "4.62.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "devOptional": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT", + "optional": true + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "optional": true, + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serialize-error/node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "optional": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/set-cookie-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.1.1.tgz", + "integrity": "sha512-vM9SUhjsUYs6UeJUmygc5Ofm5eQGe85riob5ju6XCgFGJI5PLV4nrDAQpQjd+LkFBpAkADn5BQQpZ9EUNkyLuA==", + "dev": true, + "license": "MIT" + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/source-map": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", + "integrity": "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==", + "devOptional": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">= 12" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/spark-md5": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/spark-md5/-/spark-md5-3.0.2.tgz", + "integrity": "sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw==", + "license": "(WTFPL OR MIT)", + "optional": true + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause", + "optional": true + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "dev": true, + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/std-env": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/strict-event-emitter": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", + "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-ts": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/string-ts/-/string-ts-2.3.1.tgz", + "integrity": "sha512-xSJq+BS52SaFFAVxuStmx6n5aYZU571uYUnUrPXkPFCfdHyZMMlbP2v2Wx5sNBnAVzq/2+0+mcBLBa3Xa5ubYw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/sucrase": { + "version": "3.35.1", + "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", + "integrity": "sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.2", + "commander": "^4.0.0", + "lines-and-columns": "^1.1.6", + "mz": "^2.7.0", + "pirates": "^4.0.1", + "tinyglobby": "^0.2.11", + "ts-interface-checker": "^0.1.9" + }, + "bin": { + "sucrase": "bin/sucrase", + "sucrase-node": "bin/sucrase-node" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/synckit": { + "version": "0.11.13", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.13.tgz", + "integrity": "sha512-eNRKgb3z66Yp3D2CixVujOUvXLFUTij/zVnV8KRyvFdQwpz7I5DS8UfRkTeLzb64u+dkzDSdelE24izu+zSSUg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@pkgr/core": "^0.3.6" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/synckit" + } + }, + "node_modules/tagged-tag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz", + "integrity": "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tailwind-api-utils": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tailwind-api-utils/-/tailwind-api-utils-1.0.3.tgz", + "integrity": "sha512-KpzUHkH1ug1sq4394SLJX38ZtpeTiqQ1RVyFTTSY2XuHsNSTWUkRo108KmyyrMWdDbQrLYkSHaNKj/a3bmA4sQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "enhanced-resolve": "^5.18.1", + "jiti": "^2.4.2", + "local-pkg": "^1.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/hyoban" + }, + "peerDependencies": { + "tailwindcss": "^3.3.0 || ^4.0.0 || ^4.0.0-beta" + } + }, + "node_modules/tailwindcss": { + "version": "3.4.19", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.19.tgz", + "integrity": "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@alloc/quick-lru": "^5.2.0", + "arg": "^5.0.2", + "chokidar": "^3.6.0", + "didyoumean": "^1.2.2", + "dlv": "^1.1.3", + "fast-glob": "^3.3.2", + "glob-parent": "^6.0.2", + "is-glob": "^4.0.3", + "jiti": "^1.21.7", + "lilconfig": "^3.1.3", + "micromatch": "^4.0.8", + "normalize-path": "^3.0.0", + "object-hash": "^3.0.0", + "picocolors": "^1.1.1", + "postcss": "^8.4.47", + "postcss-import": "^15.1.0", + "postcss-js": "^4.0.1", + "postcss-load-config": "^4.0.2 || ^5.0 || ^6.0", + "postcss-nested": "^6.2.0", + "postcss-selector-parser": "^6.1.2", + "resolve": "^1.22.8", + "sucrase": "^3.35.0" + }, + "bin": { + "tailwind": "lib/cli.js", + "tailwindcss": "lib/cli.js" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tailwindcss-animate": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/tailwindcss-animate/-/tailwindcss-animate-1.0.7.tgz", + "integrity": "sha512-bl6mpH3T7I3UFxuvDEXLxy/VuFxBk5bbzplh7tXI68mwMokNYd1t9qPBHlnyTwfa4JGC4zP516I1hYYtQ/vspA==", + "dev": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || insiders" + } + }, + "node_modules/tailwindcss-scoped-preflight": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/tailwindcss-scoped-preflight/-/tailwindcss-scoped-preflight-3.5.2.tgz", + "integrity": "sha512-W+aWoA1Ia7M/Gaw5EZ5wN+OplbSgOkS4UKfJOZqTqKm4XN2Sv8c5gPaNavxIKwaPKX9A62kP/9Y7xELe5tHvNA==", + "dev": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "postcss": "^8", + "tailwindcss": "^3 || ^4" + } + }, + "node_modules/tailwindcss/node_modules/jiti": { + "version": "1.21.7", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", + "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "jiti": "bin/jiti.js" + } + }, + "node_modules/tailwindcss/node_modules/postcss-selector-parser": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.4.tgz", + "integrity": "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/tapable": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/thread-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.2.0.tgz", + "integrity": "sha512-zLBvqpwr4Esa0kRjcrzGU6zL25lePWaCLMx0RQFrmteozIfeNdaMLpG5U7PeHzvlFkAWaRKA9/KVW4F60iB+qw==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/tinypool": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-2.1.0.tgz", + "integrity": "sha512-Pugqs6M0m7Lv1I7FtxN4aoyToKg1C4tu+/381vH35y8oENM/Ai7f7C4StcoK4/+BSw9ebcS8jRiVrORFKCALLw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.0.0 || >=22.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tldts": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.4.tgz", + "integrity": "sha512-kFXFK7O4WPextIUAOk8qtnw9dxR9UIXP9CjuH1cTBVBZMDeQcUPgr/IazGiw1B0Yiw5L75gHLWeW4iD793r90g==", + "dev": true, + "license": "MIT", + "dependencies": { + "tldts-core": "^7.4.4" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.4.tgz", + "integrity": "sha512-vwVLJVvvpslm7vqAH7+XNj/neA/Ynq7DT2EEcMuwc5YzN5XaMyRAqxwU+uX3azZ1FQtB2gvrvnLnAEkvYlVdfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tough-cookie": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", + "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^7.0.5" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "dev": true, + "license": "MIT", + "bin": { + "tree-kill": "cli.js" + } + }, + "node_modules/ts-api-utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", + "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/ts-interface-checker": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", + "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/ts-pattern": { + "version": "5.9.0", + "resolved": "https://registry.npmjs.org/ts-pattern/-/ts-pattern-5.9.0.tgz", + "integrity": "sha512-6s5V71mX8qBUmlgbrfL33xDUwO0fq48rxAu2LBE11WBeGdpCPOsXksQbZJHvHwhrd3QjUusd3mAOM5Gg0mFBLg==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/tsup": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/tsup/-/tsup-8.5.1.tgz", + "integrity": "sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==", + "dev": true, + "license": "MIT", + "dependencies": { + "bundle-require": "^5.1.0", + "cac": "^6.7.14", + "chokidar": "^4.0.3", + "consola": "^3.4.0", + "debug": "^4.4.0", + "esbuild": "^0.27.0", + "fix-dts-default-cjs-exports": "^1.0.0", + "joycon": "^3.1.1", + "picocolors": "^1.1.1", + "postcss-load-config": "^6.0.1", + "resolve-from": "^5.0.0", + "rollup": "^4.34.8", + "source-map": "^0.7.6", + "sucrase": "^3.35.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.11", + "tree-kill": "^1.2.2" + }, + "bin": { + "tsup": "dist/cli-default.js", + "tsup-node": "dist/cli-node.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@microsoft/api-extractor": "^7.36.0", + "@swc/core": "^1", + "postcss": "^8.4.12", + "typescript": ">=4.5.0" + }, + "peerDependenciesMeta": { + "@microsoft/api-extractor": { + "optional": true + }, + "@swc/core": { + "optional": true + }, + "postcss": { + "optional": true + }, + "typescript": { + "optional": true + } + } + }, + "node_modules/tsup/node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/tsup/node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/tsx": { + "version": "4.22.4", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz", + "integrity": "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/turbo": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/turbo/-/turbo-2.10.0.tgz", + "integrity": "sha512-o016H9PPtuH2deb3mh3Vci3Avfi9UYgM/RONQisY7HnloupP0IFSbFS3gFYJgFJP8nwBrByHWFQIDa8T2zIXPw==", + "dev": true, + "license": "MIT", + "peer": true, + "bin": { + "turbo": "bin/turbo" + }, + "optionalDependencies": { + "@turbo/darwin-64": "2.10.0", + "@turbo/darwin-arm64": "2.10.0", + "@turbo/linux-64": "2.10.0", + "@turbo/linux-arm64": "2.10.0", + "@turbo/windows-64": "2.10.0", + "@turbo/windows-arm64": "2.10.0" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.7.0.tgz", + "integrity": "sha512-1URUxUqfHFM1c+zfSPsa3gnkO7Aq21qyH75SIduNYz4SzY964rn1X2vCMQaHSHhktiw+0kPa2iyb6PUpXqB6Vg==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "dependencies": { + "tagged-tag": "^1.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/ufo": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.4.tgz", + "integrity": "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==", + "dev": true, + "license": "MIT" + }, + "node_modules/undici": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "devOptional": true, + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/until-async": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/until-async/-/until-async-3.0.2.tgz", + "integrity": "sha512-IiSk4HlzAMqTUseHHe3VhIGyuFmN90zMTpD3Z3y8jeQbzLIq500MVM7Jq2vUAnTKAFPJrqwkzr6PoTcPhGcOiw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/kettanaito" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vite": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.0.tgz", + "integrity": "sha512-BuJcQK/56NQTWDGn4ABea3q4SSBdNPWwNZKTkkUpcMPnLoquSYH8llRtSUIgoL1KSCpHt5eghLShn50mH36y7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.15", + "rolldown": "~1.1.2", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.3.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/vitest": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.9.tgz", + "integrity": "sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.9", + "@vitest/mocker": "4.1.9", + "@vitest/pretty-format": "4.1.9", + "@vitest/runner": "4.1.9", + "@vitest/snapshot": "4.1.9", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.9", + "@vitest/browser-preview": "4.1.9", + "@vitest/browser-webdriverio": "4.1.9", + "@vitest/coverage-istanbul": "4.1.9", + "@vitest/coverage-v8": "4.1.9", + "@vitest/ui": "4.1.9", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/vitest/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/vitest/node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 000000000..5a2776159 --- /dev/null +++ b/package.json @@ -0,0 +1,91 @@ +{ + "name": "@keboola/mcp-server", + "version": "2.0.0-alpha.2", + "description": "MCP server for interacting with Keboola Connection", + "keywords": [ + "ai", + "data-platform", + "keboola", + "mcp", + "model-context-protocol" + ], + "homepage": "https://github.com/keboola/keboola-mcp-server#readme", + "bugs": { + "url": "https://github.com/keboola/keboola-mcp-server/issues" + }, + "license": "MIT", + "author": "Keboola ", + "repository": { + "type": "git", + "url": "https://github.com/keboola/keboola-mcp-server.git" + }, + "bin": { + "keboola-mcp-server": "dist/index.js" + }, + "files": [ + "dist", + "README.md" + ], + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsup", + "dev": "tsx watch src/index.ts", + "start": "node dist/index.js", + "type-check": "tsc --noEmit", + "lint": "oxlint . --quiet && oxfmt --check src __tests__ *.ts *.json", + "lint:fix": "oxlint --fix . --quiet && oxfmt src __tests__ *.ts *.json", + "test": "vitest run", + "test:watch": "vitest", + "test:integ": "vitest run --config vitest.integ.config.ts", + "test:integ:watch": "vitest --config vitest.integ.config.ts", + "docs:build": "tsx scripts/docs-build.ts", + "docs:crawl": "tsx scripts/docs-crawl.ts", + "gen:tools-docs": "tsx scripts/gen-tools-docs.ts", + "check:tools-docs": "tsx scripts/gen-tools-docs.ts --check" + }, + "dependencies": { + "@hono/node-server": "^1.19.14", + "@keboola/api-client": "^5.0.0", + "@modelcontextprotocol/sdk": "^1.29.0", + "@toon-format/toon": "^2.3.0", + "ajv": "^8.20.0", + "dd-trace": "^5.69.0", + "hono": "^4.12.27", + "jose": "^6.2.3", + "pg": "^8.22.0", + "pino": "^9.14.0", + "zod": "^4.3.6" + }, + "devDependencies": { + "@keboola/oxlint-config": "^0.1.2", + "@keboola/tsconfig": "^0.1.1", + "@types/node": "^24.0.0", + "@types/pg": "^8.20.0", + "ioredis": "^5.4.1", + "msw": "^2.14.6", + "oxfmt": "^0.44.0", + "oxlint": "^1.56.0", + "tsup": "^8.5.1", + "tsx": "^4.19.4", + "typescript": "^5.9.0", + "vitest": "^4.1.8" + }, + "engines": { + "node": ">=22" + }, + "optionalDependencies": { + "@huggingface/transformers": "^4.2.0", + "cheerio": "^1.2.0" + } +} diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index aa137d0bf..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,223 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "keboola-mcp-server" -version = "1.72.8" -description = "MCP server for interacting with Keboola Connection" -readme = "README.md" -requires-python = ">=3.10" -license = "MIT" -authors = [{ name = "Keboola", email = "devel@keboola.com" }] -dependencies = [ - "fastmcp == 3.4.2", - "mcp == 1.27.2", - "httpx ~= 0.28", - "httpx-retries~=0.5", - "jsonpath-ng ~= 1.8", - "jsonschema ~= 4.26", - "pyjwt ~= 2.13", - "json-log-formatter ~= 1.1", - "cryptography ~= 48.0", - "pydantic ~= 2.13.0", - "sqlglot ~= 30.0", - "toon-format ~= 0.9.0b1", - "pyyaml ~= 6.0", -] -[project.optional-dependencies] -codestyle = [ - "black ~= 26.3", - "isort ~= 8.0", - "flake8 ~= 7.3", - "Flake8-pyproject ~= 1.2", - "flake8-bugbear ~= 25.11", - "flake8-colors ~= 0.1", - "flake8-isort ~= 7.0", - "flake8-pytest-style ~= 2.2", - "flake8-quotes ~= 3.4", - "flake8-typing-imports ~= 1.17", - "pep8-naming ~= 0.15", -] -tests = [ - "pytest ~= 9.0", - "pytest-asyncio ~= 1.4", - "pytest-cov ~= 7.0", - "pytest-datadir ~= 1.8", - "pytest-mock ~= 3.15", - "python-dateutil ~= 2.9", - "python-dotenv ~= 1.2", -] -integtests = [ - "kbcstorage ~= 0.9", - "requests ~= 2.34", -] -dev = [ - "tox ~= 4.35", -] - -[project.scripts] -keboola_mcp_server = "keboola_mcp_server.cli:main" -keboola-mcp-server = "keboola_mcp_server.cli:main" - -[tool.setuptools.package-data] -"keboola_mcp_server.resources" = ["storage-schema.json"] - -[tool.black] -target-version = ["py310"] -skip-string-normalization = true -line-length = 120 -extend-exclude = ''' -/( - # directories - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist -)/ -''' - -[tool.isort] -profile = "black" -line_length = 120 -multi_line_output = 3 -use_parentheses = true - -[tool.uv] -# Dependency cooldown (supply-chain hardening): never resolve a package version -# published within the last 7 days. Freshly published malicious releases are -# typically detected and yanked within hours to a few days, so a 7-day buffer -# removes most of that attack window before a version can reach our lock file. -# This is a resolution-time filter — it applies whenever `uv lock` regenerates -# the lock; CI installs with `uv sync --frozen` and is unaffected. -# Temporarily relax this (e.g. `--exclude-newer` on the CLI) when an urgent -# security fix needs to be picked up before the window elapses. -exclude-newer = "7 days" - -[tool.hatch.build.targets.wheel] -packages = ["src/keboola_mcp_server"] - -[tool.pytest.ini_options] -addopts = [ - "--import-mode=importlib", -] -asyncio_default_fixture_loop_scope = "function" -asyncio_mode = "strict" -log_cli = false -log_level = "INFO" -log_cli_format = "%(asctime)s [%(levelname)8s] %(name)s: %(message)s (%(filename)s:%(lineno)s)" -log_cli_date_format = "%Y-%m-%d %H:%M:%S" - -[tool.flake8] -max-line-length = 120 -import-order-style = "edited" -application-package-names = ["keboola_mcp_server"] -min-python-version = "3.10.0" -pytest-fixture-no-parentheses = true -# Skip unused import checks F401 in __init__.py files; re-exported imports are intentional. -per-file-ignores = [ - "__init__.py: F401", -] -# For Compatibility with black we skip: E203 whitespace before ':', W503 checks line break before binary operator -# For Compatibility with isort we skip: I101 Imported names are in the wrong order since we follow isort format using -# alphabetical case-sensitive sort A, B, a, b -extend-ignore = ["E203", "W503", "I101"] - -[tool.tox] -requires = ["tox>=4.23"] -env_list = ["python", "black", "isort", "flake8", "check-tools-docs"] -labels = { cs-fix = ["black", "isort"], cs = ["isort", "flake8"] } - -[tool.tox.env_run_base] -description = "Run tests" -extras = ["tests"] -package = "wheel" -wheel_build_env = ".pkg" -commands = [ - [ - "pytest", - { replace = "posargs", default = [ - "tests", - "--cov=keboola_mcp_server", - "--cov-report=term-missing", - "--cov-report=xml", - "--junitxml", - "./test-results.xml", - ], extend = true }, - ], -] - -[tool.tox.env.integtests] -description = "Run integration tests" -extras = ["tests", "integtests"] -pass_env = [ - "INTEGTEST_STORAGE_API_URL", - "INTEGTEST_POOL_STORAGE_API_URL", - "INTEGTEST_STORAGE_TOKENS", - "INTEGTEST_WORKSPACE_SCHEMAS", - "INTEGTEST_STORAGE_TOKEN_PRJ2", - "INTEGTEST_WORKSPACE_SCHEMA_PRJ2", - "INTEGTEST_STORAGE_TOKEN_STORAGE_BRANCHES", - "INTEGTEST_LOCK_TTL_MINUTES", - "INTEGTEST_LOCK_POLL_INTERVAL_SECONDS", - "INTEGTEST_LOCK_MAX_WAIT_MINUTES", -] -commands = [ - [ - "pytest", - { replace = "posargs", default = [ - "integtests", - "--junitxml", - "./integtest-results.xml", - ], extend = true }, - ], -] - -[tool.tox.env.black] -description = "Fix code formatting using black" -package = "skip" -deps = [ - "black ~= 26.3", -] -commands = [["black", "src/", "tests/", "integtests/"]] - -[tool.tox.env.isort] -description = "Sort imports using isort" -package = "skip" -deps = [ - "isort ~= 8.0", -] -commands = [["isort", "src/", "tests/", "integtests/"]] - -[tool.tox.env.flake8] -description = "Run code style check using flake8" -package = "skip" -deps = [ - "flake8 ~= 7.3", - "Flake8-pyproject ~= 1.2", - "flake8-bugbear ~= 25.11", - "flake8-colors ~= 0.1", - "flake8-isort ~= 7.0", - "flake8-pytest-style ~= 2.2", - "flake8-quotes ~= 3.4", - "flake8-typing-imports ~= 1.17", - "pep8-naming ~= 0.15", -] -commands = [["flake8", "src/", "tests/", "integtests/"]] - -[tool.tox.env.check-tools-docs] -description = "Check if TOOLS.md is up-to-date with tool definitions" -extras = [] -package = "wheel" -wheel_build_env = ".pkg" -allowlist_externals = ["git"] -commands = [ - ["python", "-m", "keboola_mcp_server.generate_tool_docs"], - ["git", "diff", "--exit-code", "TOOLS.md"], -] diff --git a/scripts/docs-build.ts b/scripts/docs-build.ts new file mode 100644 index 000000000..c5a8a797f --- /dev/null +++ b/scripts/docs-build.ts @@ -0,0 +1,52 @@ +/** + * Local docs-index builder — `npm run docs:build`. + * + * Populates the pgvector docs index at DATABASE_URL with a small fixture corpus so a + * developer can run `docs_query` / `find_component_id` against a real local index. This is + * the dev mirror of the production out-of-band build (see + * feature_spec/docs-search-pgvector/architecture.md); it does NOT fetch real docs. + * + * Local quickstart: + * docker compose up -d pgvector + * DATABASE_URL=postgres://mcp:mcp@localhost:5432/docs DOCS_EMBEDDER_MODEL=stub \ + * npm run docs:build + * + * With DOCS_EMBEDDER_MODEL=stub the deterministic offline embedder is used (no API key). + * Point DOCS_EMBEDDER_* at a real embedding endpoint for semantically meaningful vectors. + */ +import { Pool } from 'pg'; + +import { createEmbedderFromEnv } from '@/clients/docsSearch'; +import { parseEnv } from '@/env'; +import { logger } from '@/logger'; +import { FIXTURE_SOURCES, migrateDocsIndex, seedDocsIndex } from './docsIndex'; + +const main = async (): Promise => { + const env = parseEnv(); + if (!env.DATABASE_URL) { + throw new Error('DATABASE_URL is required (e.g. postgres://mcp:mcp@localhost:5432/docs).'); + } + const embedder = createEmbedderFromEnv(env); + if (!embedder) { + throw new Error( + 'No embedder configured. Set DOCS_EMBEDDER_MODEL to: "stub" (offline CI), "local" ' + + '(in-process HuggingFace, no key), or a remote model name with ' + + 'DOCS_EMBEDDER_ENDPOINT/API_KEY.', + ); + } + + const pool = new Pool({ connectionString: env.DATABASE_URL }); + try { + logger.info(`Building docs index with embedder "${embedder.model}" (dim ${embedder.dim})…`); + await migrateDocsIndex(pool, embedder.dim); + const { docCount, chunkCount } = await seedDocsIndex(pool, embedder, FIXTURE_SOURCES); + logger.info(`Docs index built: ${docCount} docs, ${chunkCount} chunks.`); + } finally { + await pool.end(); + } +}; + +main().catch((err) => { + logger.error({ err }, 'docs:build failed'); + process.exitCode = 1; +}); diff --git a/scripts/docs-crawl.ts b/scripts/docs-crawl.ts new file mode 100644 index 000000000..2113aa036 --- /dev/null +++ b/scripts/docs-crawl.ts @@ -0,0 +1,154 @@ +/** + * Build a REAL local docs index from the public Keboola documentation — `npm run docs:crawl`. + * + * A simplified, local stand-in for the production out-of-band index builder (whose full + * connectors live on the @keboola/docs-search side, keboola/ui#6672). It crawls the public + * help + developer docs sitemaps, extracts each page's main text, chunks it, embeds with the + * configured embedder, and writes the pgvector index the MCP reads. No Keboola stack / token + * needed — only public HTTP + your local Postgres. + * + * Quickstart: + * docker compose up -d --wait pgvector + * DATABASE_URL=postgres://mcp:mcp@localhost:5432/docs DOCS_EMBEDDER_MODEL=local DOCS_EMBEDDER_DIM=384 \ + * npm run docs:crawl -- --limit 50 # omit --limit for the full crawl + * + * Then point the MCP at the same DATABASE_URL + embedder and query docs_query / find_component_id. + * Flags: --limit N (cap pages, for a quick run), --source help|dev|all (default all). + */ +import type * as CheerioNS from 'cheerio'; // type-only (erased) — cheerio is a runtime-optional dep +import { parseArgs } from 'node:util'; +import { Pool } from 'pg'; + +import { createEmbedderFromEnv } from '@/clients/docsSearch'; +import { parseEnv } from '@/env'; +import { logger } from '@/logger'; +import { migrateDocsIndex, seedDocsIndex, type SourceDoc } from './docsIndex'; + +type CheerioLoad = typeof CheerioNS.load; + +type Source = { type: 'help' | 'dev'; sitemap: string }; + +const SOURCES: Record = { + help: { type: 'help', sitemap: 'https://help.keboola.com/sitemap-index.xml' }, + dev: { type: 'dev', sitemap: 'https://developers.keboola.com/sitemap.xml' }, +}; + +const CONCURRENCY = 6; + +const fetchText = async (url: string): Promise => { + const res = await fetch(url, { headers: { 'user-agent': 'keboola-mcp-docs-crawl' } }); + if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); + return res.text(); +}; + +/** Recursively resolve a sitemap or sitemap-index into a flat list of page URLs. */ +const sitemapUrls = async (sitemapUrl: string): Promise => { + const xml = await fetchText(sitemapUrl); + const locs = [...xml.matchAll(/([^<]+)<\/loc>/g)].map((m) => m[1]!.trim()); + const isIndex = / sitemapUrls(u).catch(() => []))); + return nested.flat(); +}; + +/** Extract a page's title + main content text, dropping nav/script/style/etc. */ +const extractPage = (html: string, load: CheerioLoad): { title: string; content: string } => { + const $ = load(html); + $('script, style, nav, header, footer, aside, noscript, svg').remove(); + const main = $('main, article, .sl-markdown-content, [role="main"]').first(); + const root = main.length ? main : $('body'); + const content = root.text().replace(/\s+/g, ' ').trim(); + const title = ($('h1').first().text() || $('title').text() || '').replace(/\s+/g, ' ').trim(); + return { title, content }; +}; + +/** Run `worker` over `items` with bounded concurrency, collecting non-null results. */ +const mapPool = async ( + items: T[], + limit: number, + worker: (item: T, i: number) => Promise, +): Promise => { + const out: R[] = []; + let cursor = 0; + const runners = Array.from({ length: Math.min(limit, items.length) }, async () => { + for (let i = cursor++; i < items.length; i = cursor++) { + const r = await worker(items[i]!, i); + if (r !== null) out.push(r); + } + }); + await Promise.all(runners); + return out; +}; + +const main = async (): Promise => { + const { values } = parseArgs({ + options: { limit: { type: 'string' }, source: { type: 'string', default: 'all' } }, + allowPositionals: false, + }); + const env = parseEnv(); + if (!env.DATABASE_URL) throw new Error('DATABASE_URL is required.'); + const embedder = createEmbedderFromEnv(env); + if (!embedder) { + throw new Error('No embedder configured (set DOCS_EMBEDDER_MODEL=local for a no-key local build).'); + } + const limit = values.limit ? Number(values.limit) : Infinity; + const chosen: Source[] = + values.source === 'all' + ? Object.values(SOURCES) + : [SOURCES[values.source ?? 'all']].filter((s): s is Source => s !== undefined); + if (chosen.length === 0) throw new Error(`Unknown --source "${values.source}" (help|dev|all).`); + + const { load } = await import('cheerio').catch(() => { + throw new Error("docs:crawl needs the optional 'cheerio' package (`npm i cheerio`)."); + }); + + // 1) Collect page URLs from the sitemaps. + logger.info(`Resolving sitemaps for: ${chosen.map((s) => s.type).join(', ')}…`); + const urlsBySource = await Promise.all( + chosen.map(async (s) => ({ type: s.type, urls: await sitemapUrls(s.sitemap) })), + ); + let pages = urlsBySource.flatMap((s) => s.urls.map((url) => ({ type: s.type, url }))); + if (Number.isFinite(limit)) pages = pages.slice(0, limit); + logger.info(`Fetching ${pages.length} page(s) with concurrency ${CONCURRENCY}…`); + + // 2) Fetch + extract each page into a SourceDoc. + let done = 0; + const sources = await mapPool<{ type: 'help' | 'dev'; url: string }, SourceDoc>( + pages, + CONCURRENCY, + async ({ type, url }) => { + try { + const { title, content } = extractPage(await fetchText(url), load); + if (content.length < 200) return null; // skip near-empty pages (nav-only, redirects) + if (++done % 25 === 0) logger.info(` …extracted ${done}/${pages.length}`); + return { + sourceKey: `${type}:${new URL(url).pathname}`, + sourceType: type, + title: title || url, + content, + sourceUrl: url, + componentType: null, + }; + } catch (err) { + logger.warn(` skip ${url}: ${(err as Error).message}`); + return null; + } + }, + ); + logger.info(`Extracted ${sources.length} docs. Embedding with "${embedder.model}" + indexing…`); + + // 3) Migrate + seed (chunk + embed + store). Full rebuild. + const pool = new Pool({ connectionString: env.DATABASE_URL }); + try { + await migrateDocsIndex(pool, embedder.dim); + const { docCount, chunkCount } = await seedDocsIndex(pool, embedder, sources); + logger.info(`Docs index built from live docs: ${docCount} docs, ${chunkCount} chunks.`); + } finally { + await pool.end(); + } +}; + +main().catch((err) => { + logger.error({ err }, 'docs:crawl failed'); + process.exitCode = 1; +}); diff --git a/scripts/docsIndex.ts b/scripts/docsIndex.ts new file mode 100644 index 000000000..a73542117 --- /dev/null +++ b/scripts/docsIndex.ts @@ -0,0 +1,242 @@ +/** + * Local/CI docs-index provisioning: schema + a small fixture corpus + a minimal seeder. + * + * This is the *local* mirror of the production index build. In production the index is + * built out-of-band by the `@keboola/docs-search` indexer (`runIndexBuild` + the source + * connectors that clone the help/dev repos and the component catalog) — see + * feature_spec/docs-search-pgvector/architecture.md. That machinery is out of scope for + * this repo; here we only need *some* reproducible content so a developer (and CI) can run + * `docs_query` / `find_component_id` end-to-end against a real pgvector index. + * + * The seeder is deliberately minimal (one chunk per doc, no incremental diffing/gates) — + * enough for retrieval to work. Paired with the deterministic StubEmbedder, build and + * query embed identically, so retrieval is reproducible offline. + */ +import { randomUUID } from 'node:crypto'; +import type { Pool } from 'pg'; + +import type { Embedder } from '@/clients/docsSearch'; + +/** A source document to index (subset of @keboola/docs-search `SourceDoc`). */ +export type SourceDoc = { + sourceKey: string; + sourceType: 'help' | 'dev' | 'component'; + title: string; + content: string; + sourceUrl: string; + componentType: string | null; +}; + +/** Idempotent schema (mirrors @keboola/docs-search migrations/001_init.sql). */ +/** Default embedding dimension (text-embedding-3-large / stub). The local embedder is 384. */ +export const DEFAULT_DIM = 3072; + +/** + * Idempotent schema at a given embedding dimension (mirrors @keboola/docs-search + * migrations/001_init.sql, but the `halfvec(N)` size is parametrized). `halfvec` covers + * up to 4000 dims, so the same type works for 384 / 768 / 1024 / 1536 / 3072. + */ +export const migrationSql = (dim: number): string => ` +create extension if not exists vector; + +create table if not exists doc ( + id uuid primary key, + source_key text not null unique, + source_type text not null, + source_url text not null, + title text, + content text not null, + component_type text, + content_hash text not null +); + +create table if not exists doc_chunk ( + id uuid primary key, + doc_id uuid not null references doc(id) on delete cascade, + ordinal int not null, + embed_input text not null, + embedding halfvec(${dim}) not null +); + +create table if not exists index_manifest ( + source_key text primary key, + content_hash text not null, + doc_id uuid not null, + indexed_at timestamptz not null default now() +); + +create table if not exists index_meta ( + id boolean primary key default true check (id), + embedding_model text not null, + embedding_dim int not null, + last_success_at timestamptz, + doc_count int, + chunk_count int +); + +create index if not exists doc_chunk_embedding_hnsw + on doc_chunk using hnsw (embedding halfvec_cosine_ops); +create index if not exists doc_component_type_idx on doc (component_type); +create index if not exists doc_source_type_idx on doc (source_type); +`; + +/** A tiny but representative corpus: a couple of help/dev pages + a few component docs. */ +export const FIXTURE_SOURCES: SourceDoc[] = [ + { + sourceKey: 'connection-docs:overview', + sourceType: 'help', + title: 'What is Keboola Connection?', + content: + 'Keboola Connection is a data operations platform that lets you extract, store, ' + + 'transform, and write data across many services. Storage holds your data in buckets ' + + 'and tables; components run extractions, transformations, and writers.', + sourceUrl: 'https://help.keboola.com/overview/', + componentType: null, + }, + { + sourceKey: 'developers-docs:api', + sourceType: 'dev', + title: 'Storage API basics', + content: + 'The Storage API manages buckets, tables, and file uploads. Authenticate with a ' + + 'Storage API token via the X-StorageApi-Token header.', + sourceUrl: 'https://developers.keboola.com/integrate/storage/api/', + componentType: null, + }, + { + sourceKey: 'component:keboola.ex-db-mysql', + sourceType: 'component', + title: 'MySQL extractor', + content: + 'The MySQL extractor loads data from a MySQL database into Keboola Storage. Configure ' + + 'host, port, database, user, password, and the tables or queries to extract.', + sourceUrl: 'https://components.keboola.com/components/keboola.ex-db-mysql', + componentType: 'extractor', + }, + { + sourceKey: 'component:keboola.wr-db-snowflake', + sourceType: 'component', + title: 'Snowflake writer', + content: + 'The Snowflake writer loads tables from Keboola Storage into a Snowflake database. ' + + 'Configure the connection and the input mapping of tables to write.', + sourceUrl: 'https://components.keboola.com/components/keboola.wr-db-snowflake', + componentType: 'writer', + }, + { + sourceKey: 'component:keboola.ex-google-analytics-v4', + sourceType: 'component', + title: 'Google Analytics extractor', + content: + 'The Google Analytics extractor pulls reports and metrics from Google Analytics 4 ' + + 'into Keboola Storage using OAuth authorization.', + sourceUrl: 'https://components.keboola.com/components/keboola.ex-google-analytics-v4', + componentType: 'extractor', + }, +]; + +const vectorLiteral = (vec: number[]): string => `[${vec.join(',')}]`; + +/** + * Applies the idempotent schema at `dim`. If the index already exists at a *different* + * embedding dimension, the tables are dropped and recreated — switching embedder/dim + * requires a full reindex anyway, and the seeder rebuilds from scratch. (In production the + * dim is stable; a planned dim change is an intentional reindex.) + */ +export const migrateDocsIndex = async (pool: Pool, dim: number = DEFAULT_DIM): Promise => { + const { rows } = await pool.query<{ type: string }>( + `SELECT format_type(a.atttypid, a.atttypmod) AS type + FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid + WHERE c.relname = 'doc_chunk' AND a.attname = 'embedding' AND NOT a.attisdropped`, + ); + const current = rows[0]?.type; // e.g. "halfvec(3072)" + if (current && current !== `halfvec(${dim})`) { + await pool.query('DROP TABLE IF EXISTS doc_chunk, doc, index_manifest, index_meta CASCADE'); + } + await pool.query(migrationSql(dim)); +}; + +/** + * Splits `text` into ~`size`-char windows with `overlap` chars of carry-over, breaking on + * whitespace so words aren't cut. Short text yields a single chunk. Keeps long real docs + * retrievable (each chunk is embedded + searched independently, then collapsed to its parent). + */ +export const chunkText = (text: string, size = 1000, overlap = 100): string[] => { + const clean = text.replace(/\s+/g, ' ').trim(); + if (clean.length <= size) return clean ? [clean] : []; + const chunks: string[] = []; + let start = 0; + while (start < clean.length) { + let end = Math.min(start + size, clean.length); + if (end < clean.length) { + const nextSpace = clean.lastIndexOf(' ', end); + if (nextSpace > start) end = nextSpace; + } + chunks.push(clean.slice(start, end).trim()); + if (end >= clean.length) break; + start = Math.max(end - overlap, start + 1); + } + return chunks; +}; + +/** + * Wipes and re-seeds the index from `sources` using `embedder`: each doc is chunked + * (see {@link chunkText}) into one-or-more `doc_chunk` rows, all pointing at the parent + * `doc`. Stamps `index_meta` so the availability probe reports ready. Returns counts. + */ +export const seedDocsIndex = async ( + pool: Pool, + embedder: Embedder, + sources: SourceDoc[] = FIXTURE_SOURCES, +): Promise<{ docCount: number; chunkCount: number }> => { + const client = await pool.connect(); + try { + await client.query('BEGIN'); + await client.query('TRUNCATE doc, doc_chunk, index_manifest, index_meta RESTART IDENTITY'); + + let chunkCount = 0; + for (let i = 0; i < sources.length; i++) { + const doc = sources[i]!; + const chunks = chunkText(doc.content); + if (chunks.length === 0) continue; + // Prefix each chunk with the title (parity with the SDK's embed_input) and embed the batch. + const embedInputs = chunks.map((c) => `${doc.title}\n${c}`); + const vectors = await embedder.embed(embedInputs); + + const docId = randomUUID(); + await client.query( + `INSERT INTO doc (id, source_key, source_type, source_url, title, content, component_type, content_hash) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + [docId, doc.sourceKey, doc.sourceType, doc.sourceUrl, doc.title, doc.content, doc.componentType, String(i)], + ); + for (let j = 0; j < chunks.length; j++) { + await client.query( + `INSERT INTO doc_chunk (id, doc_id, ordinal, embed_input, embedding) + VALUES ($1, $2, $3, $4, $5::halfvec)`, + [randomUUID(), docId, j, embedInputs[j]!, vectorLiteral(vectors[j]!)], + ); + chunkCount++; + } + await client.query( + `INSERT INTO index_manifest (source_key, content_hash, doc_id) VALUES ($1, $2, $3)`, + [doc.sourceKey, String(i), docId], + ); + } + + await client.query( + `INSERT INTO index_meta (id, embedding_model, embedding_dim, last_success_at, doc_count, chunk_count) + VALUES (true, $1, $2, now(), $3, $4)`, + [embedder.model, embedder.dim, sources.length, chunkCount], + ); + await client.query('COMMIT'); + return { docCount: sources.length, chunkCount }; + } catch (err) { + await client.query('ROLLBACK'); + throw err; + } finally { + client.release(); + } +}; + +/** Convenience: the exact text used as a chunk's embed input (deterministic-query helper). */ +export const embedInputFor = (doc: SourceDoc): string => `${doc.title}\n${doc.content}`; diff --git a/scripts/gen-tools-docs.ts b/scripts/gen-tools-docs.ts new file mode 100644 index 000000000..b2f54b2b7 --- /dev/null +++ b/scripts/gen-tools-docs.ts @@ -0,0 +1,274 @@ +/** + * TOOLS.md generator — TypeScript port of `keboola_mcp_server.generate_tool_docs`. + * + * Builds the MCP server with a dummy Config, lists every registered tool over an + * in-memory MCP client (which gives us the same zod->JSON-schema conversion used on + * the wire), and renders `TOOLS.md` in the exact format the Python generator used. + * + * TS tools carry no FastMCP-style tags, so the category + tag metadata that the + * Python doc derived from tool tags is reproduced here from a name-keyed map. The + * map is the single source of truth for both the category grouping and the + * `**Tags**:` lines — keep it in sync when adding/removing tools. + * + * Usage: + * tsx scripts/gen-tools-docs.ts # write TOOLS.md + * tsx scripts/gen-tools-docs.ts --check # diff against committed TOOLS.md (CI gate) + */ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import type { ToolAnnotations } from '@modelcontextprotocol/sdk/types.js'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { Config } from '@/config'; +import { createServer } from '@/server'; + +const OUTPUT_PATH = resolve(import.meta.dirname, '..', 'TOOLS.md'); + +/** Tool name kept out of the docs (scaffold/diagnostic tool, no public category). */ +const HIDDEN_TOOLS = new Set(['get_server_info']); + +/** + * Category + tags per tool name, mirroring the FastMCP tags the Python tools carried. + * Tools whose category is not one of the listed categories fall into "Other Tools" + * (parity with the Python `OTHER_CATEGORY`), matching the data-app tools' placement. + */ +type ToolMeta = { category: string; tags: string[] }; + +const TOOL_META: Record = { + // Component Tools + add_config_row: { category: 'Component Tools', tags: ['components'] }, + create_config: { category: 'Component Tools', tags: ['components'] }, + create_sql_transformation: { category: 'Component Tools', tags: ['components'] }, + get_components: { category: 'Component Tools', tags: ['components'] }, + get_config_examples: { category: 'Component Tools', tags: ['components'] }, + get_configs: { category: 'Component Tools', tags: ['components'] }, + run_sync_action: { category: 'Component Tools', tags: ['components'] }, + update_config: { category: 'Component Tools', tags: ['components', 'config-diff-preview'] }, + update_config_row: { category: 'Component Tools', tags: ['components', 'config-diff-preview'] }, + update_sql_transformation: { + category: 'Component Tools', + tags: ['components', 'config-diff-preview'], + }, + // Other Tools (data apps) + create_python_js_data_app_git_credential: { category: 'Other Tools', tags: ['data-apps'] }, + delete_python_js_data_app_draft: { category: 'Other Tools', tags: ['data-apps'] }, + deploy_data_app: { category: 'Other Tools', tags: ['data-apps'] }, + get_data_apps: { category: 'Other Tools', tags: ['data-apps'] }, + modify_python_js_data_app: { category: 'Other Tools', tags: ['data-apps'] }, + modify_streamlit_data_app: { + category: 'Other Tools', + tags: ['config-diff-preview', 'data-apps'], + }, + // Documentation Tools + docs_query: { category: 'Documentation Tools', tags: ['docs'] }, + // Flow Tools + create_conditional_flow: { category: 'Flow Tools', tags: ['flows'] }, + create_flow: { category: 'Flow Tools', tags: ['flows'] }, + get_flow_examples: { category: 'Flow Tools', tags: ['flows'] }, + get_flow_schema: { category: 'Flow Tools', tags: ['flows'] }, + get_flows: { category: 'Flow Tools', tags: ['flows'] }, + modify_flow: { category: 'Flow Tools', tags: ['config-diff-preview', 'flows'] }, + update_flow: { category: 'Flow Tools', tags: ['config-diff-preview', 'flows'] }, + // Jobs Tools + get_jobs: { category: 'Jobs Tools', tags: ['jobs'] }, + run_job: { category: 'Jobs Tools', tags: ['jobs'] }, + // OAuth Tools + create_oauth_url: { category: 'OAuth Tools', tags: ['oauth'] }, + // Project Tools + get_project_info: { category: 'Project Tools', tags: ['project'] }, + update_project_description: { category: 'Project Tools', tags: ['project'] }, + // Search Tools + find_component_id: { category: 'Search Tools', tags: ['search'] }, + search: { category: 'Search Tools', tags: ['search'] }, + // Semantic Tools + get_semantic_context: { category: 'Semantic Tools', tags: ['semantic'] }, + get_semantic_schema: { category: 'Semantic Tools', tags: ['semantic'] }, + search_semantic_context: { category: 'Semantic Tools', tags: ['semantic'] }, + validate_semantic_query: { category: 'Semantic Tools', tags: ['semantic'] }, + // SQL Tools + query_data: { category: 'SQL Tools', tags: ['sql'] }, + // Storage Tools + get_buckets: { category: 'Storage Tools', tags: ['storage'] }, + get_tables: { category: 'Storage Tools', tags: ['storage'] }, + update_descriptions: { category: 'Storage Tools', tags: ['storage'] }, +}; + +const OTHER_CATEGORY = 'Other Tools'; + +/** + * Detail-section category order — the order categories first appear when the Python + * generator walked `list_tools()`. Reproduced verbatim to keep the committed file + * stable; tools whose category is missing here are appended in first-seen order. + */ +const DETAIL_CATEGORY_ORDER = [ + 'Component Tools', + 'Other Tools', + 'Documentation Tools', + 'Flow Tools', + 'Jobs Tools', + 'OAuth Tools', + 'Project Tools', + 'Search Tools', + 'Semantic Tools', + 'SQL Tools', + 'Storage Tools', +]; + +type ListedTool = { + name: string; + description?: string; + inputSchema?: unknown; + annotations?: ToolAnnotations; +}; + +/** Lists all registered tools over an in-memory MCP client, gating bypassed. */ +const listAllTools = async (): Promise => { + const config = new Config({ + storageApiUrl: 'https://connection.test', + storageToken: 'tok', + }); + // skipGating: docs must list every tool regardless of project features/role. + const server = createServer(config, { skipGating: true }); + + const client = new Client({ name: 'tools-docs-generator', version: '0.0.0' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await Promise.all([client.connect(clientTransport), server.server.connect(serverTransport)]); + + try { + const { tools } = await client.listTools(); + return tools as ListedTool[]; + } finally { + await client.close(); + await server.close(); + } +}; + +const annotationsLabel = (annotations: ToolAnnotations | undefined): string => { + if (!annotations) return ''; + const labels: string[] = []; + if (annotations.readOnlyHint) labels.push('read-only'); + if (annotations.destructiveHint) labels.push('destructive'); + if (annotations.idempotentHint) labels.push('idempotent'); + return labels.length ? `\`${labels.sort().join(', ')}\`` : ''; +}; + +const tagsLabel = (tags: string[]): string => + tags.length ? `\`${[...tags].sort().join(', ')}\`` : ''; + +const firstSentence = (text: string | undefined): string => { + if (!text) return 'No description available.'; + return `${text.split('.')[0]}.`.trim(); +}; + +/** GitHub-style markdown anchor (port of `_generate_anchor`). */ +const anchor = (text: string): string => + text + .toLowerCase() + .replace(/[^\w\s-]/g, '') + .replace(/\s+/g, '-'); + +const categoryOf = (name: string): string => TOOL_META[name]?.category ?? OTHER_CATEGORY; +const tagsOf = (name: string): string[] => TOOL_META[name]?.tags ?? []; + +/** Codepoint string comparison, matching Python's `sorted` (not locale-aware). */ +const cmp = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0); + +const byName = (a: ListedTool, b: ListedTool): number => cmp(a.name, b.name); + +const render = (tools: ListedTool[]): string => { + const docTools = tools.filter((t) => !HIDDEN_TOOLS.has(t.name)); + + const byCategory = new Map(); + for (const tool of docTools) { + const cat = categoryOf(tool.name); + (byCategory.get(cat) ?? byCategory.set(cat, []).get(cat)!).push(tool); + } + + const out: string[] = []; + + // Header + out.push('# Tools Documentation'); + out.push( + 'This document provides details about the tools available in the Keboola MCP server.', + ); + out.push(''); + + // Index — categories sorted by name, tools sorted by name. + out.push('## Index'); + const indexCategories = [...byCategory.keys()].sort(cmp); + for (const category of indexCategories) { + const catTools = [...byCategory.get(category)!].sort(byName); + out.push(''); + out.push(`### ${category}`); + for (const tool of catTools) { + out.push(`- [${tool.name}](#${anchor(tool.name)}): ${firstSentence(tool.description)}`); + } + } + out.push(''); + out.push('---'); + + // Detail — categories in first-seen order, tools sorted by name. + const detailCategories = [ + ...DETAIL_CATEGORY_ORDER.filter((c) => byCategory.has(c)), + ...[...byCategory.keys()].filter((c) => !DETAIL_CATEGORY_ORDER.includes(c)), + ]; + for (const category of detailCategories) { + const catTools = [...byCategory.get(category)!].sort(byName); + out.push(''); + out.push(`# ${category}`); + for (const tool of catTools) { + const a = anchor(tool.name); + out.push(``); + out.push(`## ${tool.name}`); + out.push(`**Annotations**: ${annotationsLabel(tool.annotations)}`); + out.push(''); + out.push(`**Tags**: ${tagsLabel(tagsOf(tool.name))}`); + out.push(''); + out.push('**Description**:'); + out.push(''); + out.push(tool.description ?? ''); + out.push(''); + out.push(''); + out.push('**Input JSON Schema**:'); + out.push('```json'); + out.push(JSON.stringify(tool.inputSchema ?? {}, null, 2)); + out.push('```'); + out.push(''); + out.push('---'); + } + } + + return `${out.join('\n')}\n`; +}; + +const main = async (): Promise => { + const check = process.argv.includes('--check'); + const tools = await listAllTools(); + const content = render(tools); + + if (check) { + let committed = ''; + try { + committed = readFileSync(OUTPUT_PATH, 'utf-8'); + } catch { + committed = ''; + } + if (committed !== content) { + process.stderr.write( + 'TOOLS.md is out of date. Run `npm run gen:tools-docs` and commit the result.\n', + ); + process.exit(1); + } + process.stdout.write('TOOLS.md is up to date.\n'); + return; + } + + writeFileSync(OUTPUT_PATH, content, 'utf-8'); + process.stdout.write(`Wrote ${OUTPUT_PATH} (${tools.length} tools).\n`); +}; + +main().catch((error) => { + process.stderr.write(`Failed to generate TOOLS.md: ${String(error)}\n`); + process.exit(1); +}); diff --git a/server.json.template b/server.json.template index 56c635f74..0c3eb026b 100644 --- a/server.json.template +++ b/server.json.template @@ -9,8 +9,8 @@ }, "packages": [ { - "registryType": "pypi", - "identifier": "keboola-mcp-server", + "registryType": "npm", + "identifier": "@keboola/mcp-server", "version": "__MCP_SERVER_VERSION__", "transport": { "type": "stdio" diff --git a/src/clients/docsSearch.ts b/src/clients/docsSearch.ts new file mode 100644 index 000000000..3b6e07099 --- /dev/null +++ b/src/clients/docsSearch.ts @@ -0,0 +1,435 @@ +/** + * Docs-search retrieval client (pgvector-backed). + * + * Serves the `docs_query` and `find_component_id` tools from a prebuilt pgvector + * documentation index (RFC: feature_spec/docs-search-pgvector/RFC.md). The MCP only + * ever *reads* the index — it is built out-of-band by a cron job (see the architecture + * doc); a missing/stale index degrades the two docs tools gracefully, nothing else. + * + * VENDORED, TEMPORARILY: the retrieval tier below is a verbatim copy of + * `@keboola/docs-search` (keboola/ui#6672). That package is a private workspace package + * not yet on the registry `@keboola/api-client` comes from, so it cannot be installed + * here. Once it publishes, delete the vendored functions and replace the body of + * `createDocsSearchFromEnv` with: + * + * import { createDocsSearch, OpenAIEmbedder } from '@keboola/docs-search'; + * const sdk = createDocsSearch({ pool, embedder, llm }); + * + * keeping the exported {@link DocsSearch} interface identical so the tools don't change. + * + * `recommendComponents` exposes each result's `sourceKey` so `find_component_id` can + * recover the component id (it lives in `source_key = 'component:'`). This was added + * to the SDK on keboola/ui#6672, so the vendored SELECT here matches the published shape + * and the swap will be a clean drop-in. + */ +import { Pool } from 'pg'; + +import { type Env, parseEnv } from '@/env'; +import { logger } from '@/logger'; + +// --------------------------------------------------------------------------- +// Contract (mirrors @keboola/docs-search public types) +// --------------------------------------------------------------------------- + +export type RetrievedDoc = { + id: string; + /** Stable natural key, e.g. `component:keboola.ex-salesforce`. */ + sourceKey: string; + sourceUrl: string; + title: string | null; + content: string; + componentType: string | null; + /** Cosine similarity in [0, 1]. */ + score: number; +}; + +export type DocsAnswer = { text: string; sourceUrls: string[] }; + +export type SearchOptions = { + k?: number; + minSimilarity?: number; + componentType?: string | null; + componentOnly?: boolean; +}; + +export type Embedder = { + readonly model: string; + readonly dim: number; + /** Returns one unit-normalized vector per input text. */ + embed(texts: string[]): Promise; +}; + +export type Llm = { + answer(input: { question: string; context: string }): Promise<{ answer: string }>; +}; + +export type DocsSearch = { + search(query: string, opts?: SearchOptions): Promise; + answerQuestion(question: string, opts?: SearchOptions): Promise; + recommendComponents(query: string, opts?: SearchOptions): Promise; + /** True when the index is reachable and has at least one successfully-built doc. */ + isReady(): Promise; + close(): Promise; +}; + +const DEFAULT_K = 15; +const DEFAULT_MIN_SIMILARITY = 0.25; +// LLM-less `docs_query`: how many top docs to extract and how much of each, so the tool +// payload stays bounded (~a few KB) instead of dumping full pages (RFC trade-off). +const EXTRACT_DOC_COUNT = 5; +const EXTRACT_SNIPPET_CHARS = 800; + +// --------------------------------------------------------------------------- +// Retrieval (vendored from @keboola/docs-search) +// --------------------------------------------------------------------------- + +const toVectorLiteral = (vec: number[]): string => `[${vec.join(',')}]`; + +type DocResultRow = { + id: string; + source_key: string; + source_url: string; + title: string | null; + content: string; + component_type: string | null; + sim: number; +}; + +/** + * ANN retrieval: rank child chunks by cosine distance (HNSW), collapse to best-scoring + * parent doc, apply the similarity threshold, return top-k parents in score order. + */ +const search = async ( + pool: Pool, + embedder: Embedder, + query: string, + opts: SearchOptions = {}, +): Promise => { + const k = opts.k ?? DEFAULT_K; + const minSim = opts.minSimilarity ?? DEFAULT_MIN_SIMILARITY; + const [vec] = await embedder.embed([query]); + if (!vec) return []; + const literal = toVectorLiteral(vec); + const candidateLimit = k * 4; // over-fetch chunks to survive parent dedup + + const { rows } = await pool.query( + `WITH cand AS ( + SELECT c.doc_id, (c.embedding <=> $1::halfvec) AS dist + FROM doc_chunk c + JOIN doc d ON d.id = c.doc_id + WHERE ($3::text IS NULL OR d.component_type = $3) + AND ($4::bool IS NOT TRUE OR d.component_type IS NOT NULL) + ORDER BY c.embedding <=> $1::halfvec + LIMIT $5 + ), best AS ( + SELECT DISTINCT ON (doc_id) doc_id, dist FROM cand ORDER BY doc_id, dist + ) + SELECT d.id, d.source_key, d.source_url, d.title, d.content, d.component_type, + (1 - b.dist) AS sim + FROM best b JOIN doc d ON d.id = b.doc_id + WHERE (1 - b.dist) >= $2 + ORDER BY sim DESC + LIMIT $6`, + [literal, minSim, opts.componentType ?? null, opts.componentOnly ?? false, candidateLimit, k], + ); + + return rows.map((r) => ({ + id: r.id, + sourceKey: r.source_key, + sourceUrl: r.source_url, + title: r.title, + content: r.content, + componentType: r.component_type, + score: Number(r.sim), + })); +}; + +const formatContext = (docs: { content: string }[]): string => + docs.map((d) => `\n${d.content}\n`).join('\n\n'); + +// --------------------------------------------------------------------------- +// Embedder / LLM (OpenAI-compatible in prod; deterministic stub for local/CI) +// --------------------------------------------------------------------------- + +/** Sentinel `DOCS_EMBEDDER_MODEL` value selecting the offline {@link StubEmbedder}. */ +export const STUB_EMBEDDER_MODEL = 'stub'; +/** Sentinel `DOCS_EMBEDDER_MODEL` value selecting the in-process {@link LocalEmbedder}. */ +export const LOCAL_EMBEDDER_MODEL = 'local'; + +/** Default HuggingFace model + dim for the local embedder (small, fast, CPU-friendly). */ +export const DEFAULT_LOCAL_MODEL = 'Xenova/all-MiniLM-L6-v2'; +export const DEFAULT_LOCAL_DIM = 384; +/** Default dim for the stub / remote embedders (text-embedding-3-large native size). */ +export const DEFAULT_EMBEDDER_DIM = 3072; + +/** + * Deterministic, offline embedder (vendored from @keboola/docs-search): the same text + * always yields the same unit vector, with no network calls. NOT for production retrieval + * quality — it exists so `docs:build` + the integ test can seed and query a real pgvector + * index reproducibly (build and query embed identically). Selected via + * `DOCS_EMBEDDER_MODEL=stub`. + */ +export class StubEmbedder implements Embedder { + readonly model = STUB_EMBEDDER_MODEL; + readonly dim: number; + + constructor(dim = 3072) { + this.dim = dim; + } + + embed(texts: string[]): Promise { + return Promise.resolve(texts.map((t) => this.vector(t))); + } + + private vector(text: string): number[] { + // Seed an LCG from a rolling hash of the text, fill dim floats, L2-normalize. + let seed = 2166136261; + for (let i = 0; i < text.length; i++) { + seed ^= text.charCodeAt(i); + seed = Math.imul(seed, 16777619); + } + let state = seed >>> 0; + const next = () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 0xffffffff - 0.5; + }; + const v = Array.from({ length: this.dim }, next); + const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1; + return v.map((x) => x / norm); + } +} + +/** + * In-process embedder running a HuggingFace model as ONNX on CPU via transformers.js — + * no external service, no API key (selected via `DOCS_EMBEDDER_MODEL=local`). The model + * id (`DOCS_EMBEDDER_LOCAL_MODEL`) and dim (`DOCS_EMBEDDER_DIM`) are configurable; the dim + * MUST match the model's output size and the pgvector column. `@huggingface/transformers` + * is an optional dependency, dynamically imported so stub/remote users don't need it + * installed; a clear error is thrown if it is missing. + */ +export class LocalEmbedder implements Embedder { + readonly model: string; + readonly dim: number; + // Lazily-loaded feature-extraction pipeline (model weights are fetched/cached on first use). + private extractor: Promise< + (texts: string[], opts: object) => Promise<{ tolist(): number[][] }> + > | null = null; + + constructor(model: string, dim: number) { + this.model = model; + this.dim = dim; + } + + private pipeline() { + if (!this.extractor) { + this.extractor = import('@huggingface/transformers') + .then(({ pipeline }) => pipeline('feature-extraction', this.model) as never) + .catch((err: unknown) => { + this.extractor = null; + const cause = err instanceof Error ? err.message : String(err); + throw new Error( + "The local embedder needs the optional '@huggingface/transformers' package. " + + 'Install it (`npm i @huggingface/transformers`) or use a remote embedder ' + + `(DOCS_EMBEDDER_ENDPOINT/API_KEY/MODEL). Cause: ${cause}`, + ); + }); + } + return this.extractor; + } + + async embed(texts: string[]): Promise { + const extractor = await this.pipeline(); + // Mean-pool + L2-normalize → one unit vector per text (cosine-ready). + const output = await extractor(texts, { pooling: 'mean', normalize: true }); + return output.tolist(); + } +} + +/** OpenAI/Azure-compatible embedder. Thin fetch wrapper — no SDK dependency. */ +class OpenAIEmbedder implements Embedder { + readonly model: string; + readonly dim: number; + private readonly endpoint: string; + private readonly apiKey: string; + + constructor(opts: { endpoint: string; apiKey: string; model: string; dim: number }) { + this.endpoint = opts.endpoint; + this.apiKey = opts.apiKey; + this.model = opts.model; + this.dim = opts.dim; + } + + async embed(texts: string[]): Promise { + const res = await fetch(this.endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json', 'api-key': this.apiKey }, + body: JSON.stringify({ input: texts, model: this.model }), + }); + if (!res.ok) { + throw new Error(`embedding request failed: ${res.status} ${await res.text()}`); + } + const json = (await res.json()) as { data: { embedding: number[] }[] }; + return json.data.map((d) => d.embedding); + } +} + +/** OpenAI-compatible chat LLM for answerQuestion synthesis. */ +class OpenAILlm implements Llm { + private readonly endpoint: string; + private readonly apiKey: string; + private readonly model: string; + + constructor(opts: { endpoint: string; apiKey: string; model: string }) { + this.endpoint = opts.endpoint; + this.apiKey = opts.apiKey; + this.model = opts.model; + } + + async answer(input: { question: string; context: string }): Promise<{ answer: string }> { + const res = await fetch(this.endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json', 'api-key': this.apiKey }, + body: JSON.stringify({ + model: this.model, + messages: [ + { + role: 'system', + content: + 'You answer questions using ONLY the provided Keboola documentation excerpts. ' + + 'Be concise. If the excerpts do not contain the answer, say so.', + }, + { role: 'user', content: `Question: ${input.question}\n\nDocs:\n${input.context}` }, + ], + }), + }); + if (!res.ok) { + throw new Error(`llm request failed: ${res.status} ${await res.text()}`); + } + const json = (await res.json()) as { choices: { message: { content: string } }[] }; + return { answer: json.choices[0]?.message.content ?? '' }; + } +} + +// --------------------------------------------------------------------------- +// Provider construction + assembly +// --------------------------------------------------------------------------- + +const buildDocsSearch = (pool: Pool, embedder: Embedder, llm: Llm | null): DocsSearch => ({ + search: (query, opts) => search(pool, embedder, query, opts), + recommendComponents: (query, opts) => + search(pool, embedder, query, { ...opts, componentOnly: true }), + answerQuestion: async (question, opts) => { + // Without an LLM we return bounded extracts (not full pages), so fetch fewer parents. + const k = opts?.k ?? (llm ? DEFAULT_K : EXTRACT_DOC_COUNT); + const docs = await search(pool, embedder, question, { ...opts, k }); + const sourceUrls = [...new Set(docs.map((d) => d.sourceUrl))]; + if (!llm) { + // No LLM configured: return the top matches as short, titled extracts (RFC trade-off). + // Cap per-doc length + doc count so a real (long) page can't blow up the response — + // full pages are the LLM's job, not the raw tool payload. + if (docs.length === 0) return { text: 'No relevant documentation was found.', sourceUrls }; + const text = docs + .slice(0, EXTRACT_DOC_COUNT) + .map((d) => { + const body = + d.content.length > EXTRACT_SNIPPET_CHARS + ? `${d.content.slice(0, EXTRACT_SNIPPET_CHARS).trimEnd()}…` + : d.content; + return d.title ? `## ${d.title}\n${body}` : body; + }) + .join('\n\n---\n\n'); + return { text, sourceUrls }; + } + const { answer } = await llm.answer({ question, context: formatContext(docs) }); + return { text: answer, sourceUrls }; + }, + isReady: async () => { + try { + const { rows } = await pool.query<{ last_success_at: Date | null; doc_count: number | null }>( + 'SELECT last_success_at, doc_count FROM index_meta LIMIT 1', + ); + const meta = rows[0]; + return Boolean(meta?.last_success_at) && (meta?.doc_count ?? 0) > 0; + } catch (err) { + logger.warn({ err }, 'docs-search index probe failed'); + return false; + } + }, + close: () => pool.end(), +}); + +/** + * Builds the query-time embedder from env, by `DOCS_EMBEDDER_MODEL`: + * - `stub` → deterministic offline {@link StubEmbedder} (CI/tests), dim = DOCS_EMBEDDER_DIM ?? 3072 + * - `local` → in-process {@link LocalEmbedder} (HuggingFace/ONNX), model = DOCS_EMBEDDER_LOCAL_MODEL + * ?? all-MiniLM-L6-v2, dim = DOCS_EMBEDDER_DIM ?? 384 + * - else (endpoint+key+model) → remote {@link OpenAIEmbedder}, dim = DOCS_EMBEDDER_DIM ?? 3072 + * Returns `null` when the config is incomplete (docs tools then gate off). Shared with the + * `docs:build` seeder so the index is built and queried with the same model + dim. + */ +export const createEmbedderFromEnv = (env: Env): Embedder | null => { + const kind = env.DOCS_EMBEDDER_MODEL; + if (kind === STUB_EMBEDDER_MODEL) { + return new StubEmbedder(env.DOCS_EMBEDDER_DIM ?? DEFAULT_EMBEDDER_DIM); + } + if (kind === LOCAL_EMBEDDER_MODEL) { + return new LocalEmbedder( + env.DOCS_EMBEDDER_LOCAL_MODEL ?? DEFAULT_LOCAL_MODEL, + env.DOCS_EMBEDDER_DIM ?? DEFAULT_LOCAL_DIM, + ); + } + if (!env.DOCS_EMBEDDER_ENDPOINT || !env.DOCS_EMBEDDER_API_KEY || !kind) { + return null; + } + return new OpenAIEmbedder({ + endpoint: env.DOCS_EMBEDDER_ENDPOINT, + apiKey: env.DOCS_EMBEDDER_API_KEY, + model: kind, + dim: env.DOCS_EMBEDDER_DIM ?? DEFAULT_EMBEDDER_DIM, + }); +}; + +/** + * Builds the docs-search provider from deployment env, or returns `null` when the index + * is not configured (no `DATABASE_URL`, or no embedder credentials). A `null` provider + * gates the docs tools off — the rest of the server is unaffected. + */ +export const createDocsSearchFromEnv = (env: Env): DocsSearch | null => { + if (!env.DATABASE_URL) return null; + const embedder = createEmbedderFromEnv(env); + if (!embedder) { + logger.warn('DATABASE_URL is set but DOCS_EMBEDDER_* is not; docs tools disabled.'); + return null; + } + const pool = new Pool({ connectionString: env.DATABASE_URL, max: 4 }); + const llm = + env.DOCS_LLM_ENDPOINT && env.DOCS_LLM_API_KEY && env.DOCS_LLM_MODEL + ? new OpenAILlm({ + endpoint: env.DOCS_LLM_ENDPOINT, + apiKey: env.DOCS_LLM_API_KEY, + model: env.DOCS_LLM_MODEL, + }) + : null; + return buildDocsSearch(pool, embedder, llm); +}; + +// --------------------------------------------------------------------------- +// Process-scoped accessor (the pool must outlive a single request) +// --------------------------------------------------------------------------- + +let cached: DocsSearch | null | undefined; +let override: DocsSearch | null | undefined; + +/** Returns the process-scoped docs-search provider (memoized), or `null` if unconfigured. */ +export const getDocsSearch = (env?: Env): DocsSearch | null => { + if (override !== undefined) return override; + if (cached === undefined) { + cached = createDocsSearchFromEnv(env ?? parseEnv()); + } + return cached; +}; + +/** Test seam: force the provider (or `null`) and bypass env. Pass `undefined` to reset. */ +export const setDocsSearchForTests = (provider: DocsSearch | null | undefined): void => { + override = provider; +}; diff --git a/src/clients/encryption.ts b/src/clients/encryption.ts new file mode 100644 index 000000000..18751cf74 --- /dev/null +++ b/src/clients/encryption.ts @@ -0,0 +1,30 @@ +// Secret redaction, ported from clients/encryption.py. + +const SECRET_KEY_PREFIX = '#'; +const ENCRYPTED_VALUE_PREFIX = 'KBC::'; +export const REDACTED_SECRET_VALUE = '[REDACTED]'; + +const isEncryptedValue = (value: unknown): boolean => + typeof value === 'string' && value.startsWith(ENCRYPTED_VALUE_PREFIX); + +/** + * Deep-copies a value, replacing plaintext `#`-prefixed secret values with `[REDACTED]`. + * Values already encrypted by the encryption service (`KBC::` ciphers) are kept (opaque). + * Plaintext secrets must never reach the model context. + */ +export const redactSecrets = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map(redactSecrets); + } + if (value !== null && typeof value === 'object') { + const out: Record = {}; + for (const [key, item] of Object.entries(value)) { + out[key] = + key.startsWith(SECRET_KEY_PREFIX) && !isEncryptedValue(item) + ? REDACTED_SECRET_VALUE + : redactSecrets(item); + } + return out; + } + return value; +}; diff --git a/src/clients/keboola.ts b/src/clients/keboola.ts new file mode 100644 index 000000000..95d428220 --- /dev/null +++ b/src/clients/keboola.ts @@ -0,0 +1,113 @@ +import { createDataScienceClient } from '@keboola/api-client/dataScience'; +import { createMetastoreClient } from '@keboola/api-client/metastore'; +import { createQueueClient } from '@keboola/api-client/queue'; +import { createStorageClient } from '@keboola/api-client/storage'; +import { createSyncActionsClient } from '@keboola/api-client/syncActions'; + +import type { Config } from '@/config'; +import { ProjectLinksManager } from '@/links'; +import { createRawClient, type RawClient } from './raw'; +import { createRetryMiddleware } from './retry'; +import { deriveServiceUrls } from './urls'; + +/** + * The set of Keboola service clients a tool handler operates with, built per + * request from the resolved Config. Mirrors the Python `KeboolaClient`, but reuses + * the published `@keboola/api-client` service clients instead of bespoke HTTP code. + * + * Clients are added here as the tools that need them are ported (Plan §4). The + * scheduler and AI-docs surfaces arrive once keboola/ui#6862 is published. + */ +export type KeboolaClients = { + storage: ReturnType; + queue: ReturnType; + metastore: ReturnType; + /** Typed Sync Actions client (sendSyncAction, gitRepository.*). */ + syncActions: ReturnType; + /** Typed Data Science client (apps CRUD, runs, logs tail, runtimes). */ + dataScience: ReturnType; + /** + * Raw Storage API client rooted at `/v2/storage`, for endpoints where + * api-client's typed methods diverge from the exact SAPI calls (e.g. table+column + * metadata). Mirrors the Python `KeboolaClient.storage_client` raw access. + */ + rawStorage: RawClient; + /** Raw Queue API client (for endpoints not in api-client, e.g. job creation). */ + rawQueue: RawClient; + /** + * Raw AI catalog client for `docs/components/{id}` — the component metadata + * (config schemas + examples) used by `get_components` / `get_config_examples` / + * config validation. Documentation Q&A and component recommendation moved to the + * pgvector docs-search index (see clients/docsSearch.ts); this catalog endpoint has + * no docs-search equivalent (the index holds markdown docs, not config schemas). + */ + rawAi: RawClient; + /** Raw Sync Actions service client (POST actions). */ + rawSyncActions: RawClient; + /** + * Effective branch id for branch-scoped endpoints. `'default'` is the Storage + * API's alias for the production branch (matches Python's `branch_id or 'default'`), + * so no default-branch lookup is needed. + */ + branchId: string; +}; + +export const createKeboolaClients = (config: Config): KeboolaClients => { + if (!config.storageApiUrl) { + throw new Error('Storage API URL is not configured.'); + } + if (!config.storageToken) { + throw new Error('Storage API token is not configured.'); + } + + const urls = deriveServiceUrls(config.storageApiUrl); + const token = config.storageToken; + // Storage endpoints accept the OAuth bearer token in preference to the SAPI token + // (matches Python's `bearer_or_sapi_token`). + const storageToken = config.bearerToken ? `Bearer ${config.bearerToken}` : token; + + // Retry transient failures (network errors, 5xx, 429) with exponential backoff so a flaky + // or briefly-unavailable Keboola service doesn't fail a tool call. Mirrors the raw client's + // own retry on the same status set. A fresh middleware instance per request is cheap. + const retry = createRetryMiddleware(3); + + // ponytail: SAPI token via X-StorageApi-Token (the common path). OAuth bearer + // token handling is layered in with the OAuth provider (Plan §5). + return { + storage: createStorageClient({ baseUrl: urls.storage, token, middlewares: [retry] }), + queue: createQueueClient({ baseUrl: urls.queue, token, middlewares: [retry] }), + metastore: createMetastoreClient({ baseUrl: urls.metastore, token, middlewares: [retry] }), + syncActions: createSyncActionsClient({ + baseUrl: urls.syncActions, + token, + middlewares: [retry], + }), + dataScience: createDataScienceClient({ + baseUrl: urls.dataScience, + token, + middlewares: [retry], + }), + rawStorage: createRawClient({ baseUrl: `${urls.storage}/v2/storage`, token: storageToken }), + rawQueue: createRawClient({ baseUrl: urls.queue, token }), + rawAi: createRawClient({ baseUrl: urls.ai, token }), + rawSyncActions: createRawClient({ baseUrl: urls.syncActions, token }), + branchId: config.branchId ?? 'default', + }; +}; + +/** + * Builds a ProjectLinksManager for the current project. Mirrors the Python + * `ProjectLinksManager.from_client`: resolves the project id from the verified token. + */ +export const createLinksManager = async ( + config: Config, + clients: KeboolaClients, +): Promise => { + const token = await clients.storage.tokens.verify(); + const projectId = String((token.owner as { id: string | number }).id); + return new ProjectLinksManager({ + baseUrl: config.storageApiUrl ?? '', + projectId, + branchId: config.branchId, + }); +}; diff --git a/src/clients/raw.ts b/src/clients/raw.ts new file mode 100644 index 000000000..fbd19cb1c --- /dev/null +++ b/src/clients/raw.ts @@ -0,0 +1,160 @@ +/** + * Raw Keboola HTTP client — a faithful port of the Python `RawKeboolaClient`. + * + * Used for endpoints where `@keboola/api-client`'s typed methods diverge from the + * exact Storage/Queue API calls the Python server made (e.g. table+column metadata, + * job creation). For endpoints api-client covers cleanly, prefer the typed client. + */ +const RETRYABLE_STATUS = new Set([408, 409, 425, 429, 500, 502, 503, 504]); +const MAX_RETRIES = 3; +const BACKOFF_BASE_MS = 1000; +const MAX_BACKOFF_MS = 10_000; + +export type RawRequestOptions = { + params?: Record; + body?: unknown; + headers?: Record; +}; + +export type RawClientOptions = { + baseUrl: string; + /** SAPI token, or an `Authorization` value prefixed with `Bearer `. */ + token?: string; + readonly?: boolean; + fetchFn?: typeof fetch; +}; + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const buildUrl = ( + baseUrl: string, + endpoint: string, + params?: RawRequestOptions['params'], +): string => { + const url = new URL(`${baseUrl}/${endpoint}`); + if (params) { + for (const [key, value] of Object.entries(params)) { + if (value === undefined) continue; + if (Array.isArray(value)) { + for (const item of value) url.searchParams.append(key, item); + } else { + url.searchParams.append(key, String(value)); + } + } + } + return url.toString(); +}; + +/** Error carrying the HTTP status, so callers can branch on it (e.g. 404 fallbacks). */ +export class RawHttpError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message); + this.name = 'RawHttpError'; + } +} + +/** Builds a detailed error from a failed response (port of `_raise_for_status`). */ +const errorFromResponse = async (response: Response): Promise => { + const parts = [`${response.status} ${response.statusText}`.trim()]; + const text = await response.text(); + try { + const data = JSON.parse(text) as Record; + const apiError = (data.exception as string) ?? (data.error as string); + if (apiError) parts.push(`API error: ${apiError}`); + if (data.exceptionId) { + parts.push(`Exception ID: ${String(data.exceptionId)}`); + parts.push('When contacting Keboola support please provide the exception ID.'); + } + } catch { + if (text) parts.push(`API error: ${text}`); + } + return new RawHttpError(parts.join('\n'), response.status); +}; + +export type RawClient = { + get: (endpoint: string, options?: RawRequestOptions) => Promise; + getText: (endpoint: string, options?: RawRequestOptions) => Promise; + post: (endpoint: string, options?: RawRequestOptions) => Promise; + put: (endpoint: string, options?: RawRequestOptions) => Promise; + patch: (endpoint: string, options?: RawRequestOptions) => Promise; + delete: (endpoint: string, options?: RawRequestOptions) => Promise; +}; + +export const createRawClient = (options: RawClientOptions): RawClient => { + const doFetch = options.fetchFn ?? fetch; + const baseHeaders: Record = { + 'Content-Type': 'application/json', + 'Accept-Encoding': 'gzip', + }; + if (options.token) { + if (options.token.startsWith('Bearer ')) { + baseHeaders['Authorization'] = options.token; + } else { + baseHeaders['X-StorageAPI-Token'] = options.token; + } + } + + const request = async ( + method: string, + endpoint: string, + opts: RawRequestOptions = {}, + ): Promise => { + if (options.readonly && method !== 'GET') { + throw new Error(`Forbidden ${method} operation on a readonly client: ${options.baseUrl}`); + } + const url = buildUrl(options.baseUrl, endpoint, opts.params); + const headers = { ...baseHeaders, ...opts.headers }; + const init: RequestInit = { method, headers }; + if (method !== 'GET' && method !== 'HEAD') { + init.body = JSON.stringify(opts.body ?? {}); + } + + let lastError: unknown; + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + const response = await doFetch(url, init); + if (response.ok || !RETRYABLE_STATUS.has(response.status) || attempt === MAX_RETRIES) { + return response; + } + } catch (error) { + lastError = error; + if (attempt === MAX_RETRIES) throw error; + } + await wait(Math.min(BACKOFF_BASE_MS * 2 ** attempt, MAX_BACKOFF_MS)); + } + throw lastError instanceof Error ? lastError : new Error('Request failed'); + }; + + const json = async ( + method: string, + endpoint: string, + opts?: RawRequestOptions, + ): Promise => { + const response = await request(method, endpoint, opts); + if (!response.ok) throw await errorFromResponse(response); + return response.json() as Promise; + }; + + return { + get: (endpoint, opts) => json('GET', endpoint, opts), + getText: async (endpoint, opts) => { + const response = await request('GET', endpoint, opts); + if (!response.ok) throw await errorFromResponse(response); + return response.text(); + }, + post: (endpoint, opts) => json('POST', endpoint, opts), + put: (endpoint, opts) => json('PUT', endpoint, opts), + patch: (endpoint, opts) => json('PATCH', endpoint, opts), + // DELETE commonly returns 204 No Content / empty body; tolerate that instead of + // throwing a JSON parse error (port of Python's `if response.content` guard). + delete: async (endpoint: string, opts?: RawRequestOptions): Promise => { + const response = await request('DELETE', endpoint, opts); + if (!response.ok) throw await errorFromResponse(response); + const text = await response.text(); + return (text ? JSON.parse(text) : null) as T; + }, + }; +}; diff --git a/src/clients/retry.ts b/src/clients/retry.ts new file mode 100644 index 000000000..466c6df3e --- /dev/null +++ b/src/clients/retry.ts @@ -0,0 +1,57 @@ +import type { MiddlewareFn } from '@keboola/api-client'; + +/** + * Retry middleware for the `@keboola/api-client` fetch clients. + * + * Retries transient failures — network errors and the same retryable HTTP status set the raw + * client uses (408/409/425/429/5xx) — with exponential backoff, so a flaky or briefly + * unavailable Keboola service doesn't fail a tool call. + * + * We implement it here (rather than importing the api-client's own `createRetryMiddleware`) + * because that symbol lives on the package's root barrel, which transitively pulls + * `dayjs/plugin/utc` and fails to resolve under the test runner. `MiddlewareFn` is a + * type-only import, fully erased at build/runtime, so this file never loads the barrel. + */ +const RETRYABLE_STATUS = new Set([408, 409, 425, 429, 500, 502, 503, 504]); +const DEFAULT_MAX_RETRIES = 3; +const BACKOFF_BASE_MS = 1000; +const MAX_BACKOFF_MS = 10_000; + +const wait = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); +const backoff = (attempt: number): number => + Math.min(BACKOFF_BASE_MS * 2 ** attempt, MAX_BACKOFF_MS); + +/** Reads an HTTP status off a thrown ApiError (`{ response: Response }`) without importing it. */ +const statusOf = (value: unknown): number | undefined => { + if (value && typeof value === 'object' && 'response' in value) { + const response = (value as { response?: { status?: number } }).response; + if (response && typeof response.status === 'number') return response.status; + } + return undefined; +}; + +export const createRetryMiddleware = (maxRetries = DEFAULT_MAX_RETRIES): MiddlewareFn => { + return (next) => async (request) => { + for (let attempt = 0; ; attempt++) { + try { + const result = await next(request); + // Some clients return a response (with status) instead of throwing on non-2xx. + const status = (result as { response?: { status?: number } })?.response?.status; + if (status !== undefined && RETRYABLE_STATUS.has(status) && attempt < maxRetries) { + await wait(backoff(attempt)); + continue; + } + return result; + } catch (error) { + // Retry only on an explicit retryable HTTP status (5xx/429/...). A statusless error + // (a genuine network failure, but also e.g. an unmocked request in a unit test) is + // NOT retried here — network-level retries are the raw client's job, and retrying + // every statusless throw would turn fast failures into slow ones. + const status = statusOf(error); + const retryable = status !== undefined && RETRYABLE_STATUS.has(status); + if (!retryable || attempt >= maxRetries) throw error; + await wait(backoff(attempt)); + } + } + }; +}; diff --git a/src/clients/urls.ts b/src/clients/urls.ts new file mode 100644 index 000000000..a9bb26483 --- /dev/null +++ b/src/clients/urls.ts @@ -0,0 +1,45 @@ +/** + * Per-service base URLs, derived from the Storage API URL's hostname suffix. + * Faithful port of the URL derivation in the Python `KeboolaClient` constructor: + * every Keboola service lives at `https://.`, where `` is + * whatever follows `connection.` in the Storage API hostname. + */ +export type ServiceUrls = { + storage: string; + metastore: string; + queue: string; + ai: string; + dataScience: string; + encryption: string; + scheduler: string; + syncActions: string; + queryService: string; +}; + +export const deriveServiceUrls = (storageApiUrl: string): ServiceUrls => { + let hostname: string; + try { + hostname = new URL(storageApiUrl).hostname; + } catch { + hostname = ''; + } + + if (!hostname.startsWith('connection.')) { + throw new Error(`Invalid Keboola Storage API URL: ${storageApiUrl}`); + } + + const suffix = hostname.slice('connection.'.length); + const at = (service: string): string => `https://${service}.${suffix}`; + + return { + storage: at('connection'), + metastore: at('metastore'), + queue: at('queue'), + ai: at('ai'), + dataScience: at('data-science'), + encryption: at('encryption'), + scheduler: at('scheduler'), + syncActions: at('sync-actions'), + queryService: at('query'), + }; +}; diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 000000000..b3f552830 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,209 @@ +/** + * Server configuration. + * + * Port of the Python `keboola_mcp_server.config.Config`. Values are resolved from + * a string map (CLI args, `KBC_*` env vars, or `X-*` HTTP headers) by normalizing + * keys: lowercased with `_`/`-` removed, then matched against the field name, the + * `KBC_`-prefixed name, and the `X-`-prefixed name (in that order). + */ + +export type ConfigFields = { + /** URL to the Storage API. */ + storageApiUrl?: string; + /** Token to access the Storage API. */ + storageToken?: string; + /** Branch ID to access the Storage API. */ + branchId?: string; + /** Workspace schema for buckets/tables and SQL queries. */ + workspaceSchema?: string; + /** OAuth client ID registered in the Keboola OAuth server. */ + oauthClientId?: string; + /** OAuth client secret registered in the Keboola OAuth server. */ + oauthClientSecret?: string; + /** URL of the OAuth server to authenticate with. */ + oauthServerUrl?: string; + /** OAuth scope to request. */ + oauthScope?: string; + /** URL where the MCP server is reachable. */ + mcpServerUrl?: string; + /** Secret key for encoding/decoding JWT tokens. */ + jwtSecret?: string; + /** Access token sent in the `Authorization: Bearer ` header. */ + bearerToken?: string; + /** ID of the ongoing conversation (supplied via HTTP header only). */ + conversationId?: string; + /** Comma-separated allow list of tool names (`X-Allowed-Tools` header). */ + allowedTools?: string; + /** Comma-separated deny list of tool names (`X-Disallowed-Tools` header). */ + disallowedTools?: string; + /** Read-only mode flag (`X-Read-Only-Mode` header). */ + readOnlyMode?: string; +}; + +type FieldName = keyof ConfigFields; + +// Aliases accepted in addition to the canonical field name. +const FIELD_ALIASES: Partial> = { + storageToken: ['storageApiToken'], +}; + +const FIELD_NAMES: FieldName[] = [ + 'storageApiUrl', + 'storageToken', + 'branchId', + 'workspaceSchema', + 'oauthClientId', + 'oauthClientSecret', + 'oauthServerUrl', + 'oauthScope', + 'mcpServerUrl', + 'jwtSecret', + 'bearerToken', + 'conversationId', + 'allowedTools', + 'disallowedTools', + 'readOnlyMode', +]; + +const SECRET_HINTS = ['token', 'password', 'secret']; +const BRANCH_PRODUCTION_ALIASES = new Set(['', 'none', 'null', 'default', 'production']); + +/** Lowercases and strips `_`/`-` so `KBC_STORAGE_TOKEN`, `storage-token`, `storageToken` all collide. */ +const normalize = (name: string): string => + name.toLowerCase().replaceAll('_', '').replaceAll('-', ''); + +const isUrlField = (name: string): boolean => name.toLowerCase().includes('url'); + +const isSecretField = (name: string): boolean => + SECRET_HINTS.some((hint) => name.toLowerCase().includes(hint)); + +/** + * Reduces a URL to scheme + host. Mirrors the Python `__post_init__` amendment: + * a bare `host/path` becomes `https://host`; localhost defaults to `http`. + */ +const amendUrl = (value: string): string => { + let url: URL | undefined; + try { + url = new URL(value); + } catch { + url = undefined; + } + + if (url?.host) { + const scheme = + url.protocol === 'http:' || url.protocol === 'https:' + ? url.protocol.replace(':', '') + : url.hostname.startsWith('localhost') + ? 'http' + : 'https'; + return `${scheme}://${url.host}`; + } + + // No scheme: treat the first path segment as the host. + const host = value.split('/', 1)[0]; + if (!host) { + throw new Error(`Invalid URL: ${value}`); + } + const scheme = host.startsWith('localhost') ? 'http' : 'https'; + return `${scheme}://${host}`; +}; + +const buildLookup = (map: Record): Map => { + const lookup = new Map(); + for (const [key, value] of Object.entries(map)) { + if (value !== undefined) { + lookup.set(normalize(key), value); + } + } + return lookup; +}; + +const readOptions = (map: Record): ConfigFields => { + const lookup = buildLookup(map); + const options: ConfigFields = {}; + + for (const field of FIELD_NAMES) { + const candidates = [field, ...(FIELD_ALIASES[field] ?? [])]; + for (const candidate of candidates) { + const variants = [candidate, `KBC_${candidate}`, `X-${candidate}`]; + const hit = variants.map(normalize).find((v) => lookup.has(v)); + if (hit !== undefined) { + options[field] = lookup.get(hit); + break; + } + } + } + + return amendFields(options); +}; + +const amendFields = (fields: ConfigFields): ConfigFields => { + const amended: ConfigFields = { ...fields }; + + for (const field of FIELD_NAMES) { + const value = amended[field]; + if (value && isUrlField(field)) { + amended[field] = amendUrl(value); + } + } + + if ( + amended.branchId !== undefined && + BRANCH_PRODUCTION_ALIASES.has(amended.branchId.toLowerCase()) + ) { + amended.branchId = undefined; + } + + return amended; +}; + +export class Config { + readonly storageApiUrl?: string; + readonly storageToken?: string; + readonly branchId?: string; + readonly workspaceSchema?: string; + readonly oauthClientId?: string; + readonly oauthClientSecret?: string; + readonly oauthServerUrl?: string; + readonly oauthScope?: string; + readonly mcpServerUrl?: string; + readonly jwtSecret?: string; + readonly bearerToken?: string; + readonly conversationId?: string; + readonly allowedTools?: string; + readonly disallowedTools?: string; + readonly readOnlyMode?: string; + + constructor(fields: ConfigFields = {}) { + Object.assign(this, amendFields(fields)); + } + + /** Builds a Config from a string map (env vars, headers, or CLI args). */ + static fromMap(map: Record): Config { + return new Config(readOptions(map)); + } + + /** Returns a new Config with values from the map layered over this one. */ + replaceBy(map: Record): Config { + return new Config({ ...this.toFields(), ...readOptions(map) }); + } + + toFields(): ConfigFields { + const fields: ConfigFields = {}; + for (const field of FIELD_NAMES) { + fields[field] = this[field]; + } + return fields; + } + + /** String form with secret fields redacted. */ + toString(): string { + const params = FIELD_NAMES.map((field) => { + const value = this[field]; + if (!value) return `${field}=None`; + if (isSecretField(field)) return `${field}='****'`; + return `${field}='${value}'`; + }); + return `Config(${params.join(', ')})`; + } +} diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 000000000..2d9ce4dbd --- /dev/null +++ b/src/constants.ts @@ -0,0 +1,28 @@ +// Well-known Keboola component IDs, ported from clients/client.py. + +export const ORCHESTRATOR_COMPONENT_ID = 'keboola.orchestrator'; +export const CONDITIONAL_FLOW_COMPONENT_ID = 'keboola.flow'; +export const DATA_APP_COMPONENT_ID = 'keboola.data-apps'; + +export type FlowType = typeof CONDITIONAL_FLOW_COMPONENT_ID | typeof ORCHESTRATOR_COMPONENT_ID; + +export const FLOW_TYPES: readonly FlowType[] = [ + CONDITIONAL_FLOW_COMPONENT_ID, + ORCHESTRATOR_COMPONENT_ID, +]; + +/** Keboola metadata field keys (subset; ported from config.py MetadataField). */ +export const MetadataField = { + DESCRIPTION: 'KBC.description', + PROJECT_DESCRIPTION: 'KBC.projectDescription', + CONFIGURATION_FOLDER_NAME: 'KBC.configuration.folderName', +} as const; + +/** All component types, used to expand an empty `component_types` filter. */ +export const ALL_COMPONENT_TYPES = [ + 'application', + 'extractor', + 'transformation', + 'writer', +] as const; +export type ComponentType = (typeof ALL_COMPONENT_TYPES)[number]; diff --git a/src/env.ts b/src/env.ts new file mode 100644 index 000000000..441f60798 --- /dev/null +++ b/src/env.ts @@ -0,0 +1,168 @@ +/** + * Validated, process-level deployment environment — segregated from the + * per-request runtime Config. + * + * Two distinct concepts, deliberately kept apart: + * + * - **This module (`env`)**: process/infra variables resolved once at boot — + * listen host/port, log level, app env/version, dd-trace, OAuth client + * credentials, and `HOSTNAME_SUFFIX` (the deploy convention that derives the + * Storage/OAuth/MCP URLs). These are set at `docker run` / k8s. + * - **`Config` (config.ts)**: the per-request tenant context (Storage token, + * branch, workspace) that in HTTP mode arrives via `X-*` headers / bearer + * token on each request. It is NOT validated here, because the server must be + * able to boot in multi-tenant HTTP mode with no Storage token present. + * + * Build vs. run: validation is skipped during the image build (`npm run build` + * runs with `SKIP_ENV_VALIDATION=1`), so a build never needs runtime secrets. + * Mirrors the kai-agent `env.ts` (createEnv) pattern, using zod directly since + * it is already a dependency. + */ +import { z } from 'zod'; + +import type { Config } from '@/config'; + +const boolFromString = z.enum(['true', 'false']).transform((v) => v === 'true'); + +/** Schema for process-level deployment env. Everything optional/defaulted so the + * server can boot in HTTP multi-tenant mode without per-request secrets. */ +const envSchema = z.object({ + // Listener (HTTP transport). + HOST: z.string().default('localhost'), + PORT: z.coerce.number().int().positive().default(8000), + + // Logging + app identity (port of ServerRuntimeInfo app_env/app_version). + LOG_LEVEL: z.string().default('INFO'), + APP_ENV: z.string().default('local'), + APP_VERSION: z.string().default('DEV'), + + /** + * Deploy convention: when the explicit Storage/OAuth/MCP URLs are absent, they + * are derived from this suffix (e.g. `keboola.com` → + * `https://connection.keboola.com`). Port of server.py's HOSTNAME_SUFFIX use. + */ + HOSTNAME_SUFFIX: z.string().optional(), + + // OAuth provider (process-level; tenant tokens are per-request). Presence of + // both client id + secret enables the provider (HTTP only). + KBC_OAUTH_CLIENT_ID: z.string().optional(), + KBC_OAUTH_CLIENT_SECRET: z.string().optional(), + KBC_OAUTH_SERVER_URL: z.string().optional(), + KBC_OAUTH_SCOPE: z.string().optional(), + KBC_MCP_SERVER_URL: z.string().optional(), + KBC_JWT_SECRET: z.string().optional(), + + // Docs-search index (pgvector). All optional: when DATABASE_URL (or the embedder + // credentials) is absent, the docs_query / find_component_id tools gate off and the + // rest of the server is unaffected. The index is read-only from the MCP's side — + // it is built out-of-band by a cron job. See feature_spec/docs-search-pgvector/. + DATABASE_URL: z.string().optional(), + // DOCS_EMBEDDER_MODEL selects the embedder: 'stub' (offline CI), 'local' (in-process + // HuggingFace/ONNX — no service/key), or a remote model name (needs ENDPOINT+API_KEY). + // DOCS_EMBEDDER_DIM must match the model output AND the pgvector column dim (defaults: + // 3072 for stub/remote, 384 for local). DOCS_EMBEDDER_LOCAL_MODEL overrides the local HF id. + DOCS_EMBEDDER_ENDPOINT: z.string().optional(), + DOCS_EMBEDDER_API_KEY: z.string().optional(), + DOCS_EMBEDDER_MODEL: z.string().optional(), + DOCS_EMBEDDER_LOCAL_MODEL: z.string().optional(), + DOCS_EMBEDDER_DIM: z.coerce.number().int().positive().optional(), + // LLM for answerQuestion synthesis. Optional: without it, docs_query falls back to + // returning the retrieved documentation snippets rather than a synthesized answer. + DOCS_LLM_ENDPOINT: z.string().optional(), + DOCS_LLM_API_KEY: z.string().optional(), + DOCS_LLM_MODEL: z.string().optional(), + + // Datadog APM (consumed by dd-trace via NODE_OPTIONS in the image; listed so + // the contract is explicit and validated). + DD_SERVICE: z.string().optional(), + DD_ENV: z.string().optional(), + DD_VERSION: z.string().optional(), + DD_AGENT_HOST: z.string().optional(), + DD_LOGS_INJECTION: boolFromString.optional(), +}); + +export type Env = z.infer; + +const shouldSkip = (raw: NodeJS.ProcessEnv): boolean => + raw.SKIP_ENV_VALIDATION === '1' || raw.SKIP_ENV_VALIDATION === 'true'; + +/** Parses + validates process env (treating empty strings as unset). On a build + * (`SKIP_ENV_VALIDATION`), returns parsed-with-defaults without throwing. */ +export const parseEnv = (raw: NodeJS.ProcessEnv = process.env): Env => { + const cleaned: Record = {}; + for (const [key, value] of Object.entries(raw)) { + cleaned[key] = value === '' ? undefined : value; + } + if (shouldSkip(raw)) { + // Build phase: never fail on runtime vars. Use valid values if they parse, + // otherwise fall back to all-defaults — the build doesn't consume them. + const skipped = envSchema.safeParse(cleaned); + return skipped.success ? skipped.data : envSchema.parse({}); + } + const result = envSchema.safeParse(cleaned); + if (!result.success) { + const issues = result.error.issues.map((i) => ` ${i.path.join('.')}: ${i.message}`).join('\n'); + throw new Error(`Invalid deployment environment:\n${issues}`); + } + return result.data; +}; + +/** + * Applies the HOSTNAME_SUFFIX-based deployment defaults to a base Config — a + * faithful port of server.py's create_server() derivation: + * - no storageApiUrl + suffix → https://connection. + * - oauth configured, no oauthServerUrl → https://connection. + * - oauth configured, no mcpServerUrl → https://mcp. + * - oauth configured, no oauthScope → "email" + */ +export const applyDeploymentDefaults = (config: Config, env: Env): Config => { + const patch: Record = {}; + const suffix = env.HOSTNAME_SUFFIX; + + if (!config.storageApiUrl && suffix) { + patch.storageApiUrl = `https://connection.${suffix}`; + } + + const oauthConfigured = Boolean(config.oauthClientId && config.oauthClientSecret); + if (oauthConfigured) { + if (!config.oauthServerUrl && suffix) patch.oauthServerUrl = `https://connection.${suffix}`; + if (!config.mcpServerUrl && suffix) patch.mcpServerUrl = `https://mcp.${suffix}`; + if (!config.oauthScope) patch.oauthScope = 'email'; + } + + return Object.keys(patch).length > 0 ? config.replaceBy(patch) : config; +}; + +// Keys whose values must never be logged (matched case-insensitively on the env name). +const SECRET_ENV_KEY = /token|secret|password|api[_-]?key|\bkey\b|jwt/i; + +/** Masks the credentials in a `postgres://user:pass@host/db` URL, keeping host/db visible. */ +const maskDbUrl = (url: string): string => { + try { + const u = new URL(url); + if (u.username) u.username = '***'; + if (u.password) u.password = '***'; + return u.toString(); + } catch { + return '***'; + } +}; + +/** + * A redacted, log-safe view of the deployment env for the startup dump: secret values + * masked (`***`), `DATABASE_URL` credentials stripped, unset values shown as `null` so the + * dump is complete. Never logs a raw token/key/password. + */ +export const redactedEnv = (env: Env): Record => { + const out: Record = {}; + // Iterate every schema key (not just the ones present) so the dump is complete: an unset + // optional shows as null rather than silently vanishing. + for (const key of Object.keys(envSchema.shape)) { + const value = (env as Record)[key]; + if (value === undefined || value === null) out[key] = null; + else if (key === 'DATABASE_URL' && typeof value === 'string') out[key] = maskDbUrl(value); + else if (SECRET_ENV_KEY.test(key)) out[key] = '***'; + else out[key] = value as string | number | boolean; + } + return out; +}; diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 000000000..722a5f1ad --- /dev/null +++ b/src/index.ts @@ -0,0 +1,94 @@ +import { parseArgs } from 'node:util'; + +import { Config } from '@/config'; +import { applyDeploymentDefaults, type Env, parseEnv, redactedEnv } from '@/env'; +import { logger } from '@/logger'; +import { createServer, SERVER_VERSION } from '@/server'; +import { startHttp } from '@/transports/http'; +import { startStdio } from '@/transports/stdio'; + +// 'http-compat' is an alias for 'streamable-http' kept for backwards compatibility. +type Transport = 'stdio' | 'streamable-http' | 'http-compat'; + +type ParsedCli = { transport: Transport; config: Config; env: Env; host: string; port: number }; + +const parseCliConfig = (): ParsedCli => { + const { values } = parseArgs({ + options: { + transport: { type: 'string', default: 'stdio' }, + 'log-level': { type: 'string' }, + 'api-url': { type: 'string' }, + 'storage-token': { type: 'string' }, + 'workspace-schema': { type: 'string' }, + host: { type: 'string' }, + port: { type: 'string' }, + }, + allowPositionals: false, + }); + + const transport = (values.transport ?? 'stdio') as Transport; + + // Process-level deployment env (validated, build/run-segregated). + const env = parseEnv(); + + // Per-request base config: KBC_*/X-* env, then CLI flags layered on top. + let config = Config.fromMap(process.env).replaceBy({ + storageApiUrl: values['api-url'], + storageToken: values['storage-token'], + workspaceSchema: values['workspace-schema'], + }); + // Derive Storage/OAuth/MCP URLs from HOSTNAME_SUFFIX when not set explicitly. + config = applyDeploymentDefaults(config, env); + + return { + transport, + config, + env, + // Precedence: explicit CLI flag > deployment env > schema default. + host: values.host ?? env.HOST, + port: values.port ? Number(values.port) : env.PORT, + }; +}; + +/** One-time startup dump of the resolved config (secrets redacted) — parity with the + * Python server, and the first thing to check when a tool misbehaves. */ +const logStartupConfig = (parsed: ParsedCli): void => { + const { env, config, transport, host, port } = parsed; + const docsIndex = env.DATABASE_URL + ? `configured (model=${env.DOCS_EMBEDDER_MODEL ?? 'unset'}, dim=${env.DOCS_EMBEDDER_DIM ?? 'default'})` + : 'not configured'; + logger.info( + { + transport, + host, + port, + version: SERVER_VERSION, + docsIndex, + config: config.toString(), + env: redactedEnv(env), + }, + 'Keboola MCP server starting', + ); +}; + +const main = async (): Promise => { + const parsed = parseCliConfig(); + const { transport, config, host, port } = parsed; + logStartupConfig(parsed); + + if (transport === 'stdio') { + if (config.oauthClientId || config.oauthClientSecret) { + throw new Error('OAuth authorization can only be used with HTTP-based transports.'); + } + await startStdio(createServer(config)); + return; + } + + // 'streamable-http' and its 'http-compat' alias both serve the Hono app. + startHttp(config, host, port); +}; + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/src/keboola_mcp_server/__init__.py b/src/keboola_mcp_server/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/keboola_mcp_server/__main__.py b/src/keboola_mcp_server/__main__.py deleted file mode 100644 index 0f8f24a2c..000000000 --- a/src/keboola_mcp_server/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Main entry point for the Keboola MCP server.""" - -from keboola_mcp_server.cli import main - -if __name__ == '__main__': - main() diff --git a/src/keboola_mcp_server/authorization.py b/src/keboola_mcp_server/authorization.py deleted file mode 100644 index 2a716834d..000000000 --- a/src/keboola_mcp_server/authorization.py +++ /dev/null @@ -1,163 +0,0 @@ -""" -Tool authorization middleware for granular access control. - -This module provides middleware to filter tools based on client-specific permissions, -allowing administrators to restrict which tools specific clients (like Devin) can access. - -Authorization is configured via HTTP headers: -- X-Allowed-Tools: Comma-separated list of allowed tool names -- X-Disallowed-Tools: Comma-separated list of tools to exclude (removed from allowed set) -- X-Read-Only-Mode: Set to "true" for read-only access (only tools with readOnlyHint=True) - -Note: These headers are intended to be injected by infrastructure/proxy layers (e.g., API gateways, -reverse proxies) rather than set directly by end clients. For direct client access control, -use Storage API token permissions which provide the security layer. -""" - -import logging - -from fastmcp.exceptions import ToolError -from fastmcp.server import middleware as fmw -from fastmcp.server.middleware import CallNext, MiddlewareContext -from fastmcp.tools import Tool -from mcp import types as mt -from starlette.requests import Request - -from keboola_mcp_server.mcp import get_http_request_or_none, is_read_only_tool - -LOG = logging.getLogger(__name__) - - -class ToolAuthorizationMiddleware(fmw.Middleware): - """ - Middleware that filters tools based on client-specific authorization. - - Authorization is configured via HTTP headers: - - X-Allowed-Tools: Comma-separated list of allowed tool names - - X-Disallowed-Tools: Comma-separated list of tools to exclude (removed from allowed set) - - X-Read-Only-Mode: Set to "true" for read-only access (filters to tools with readOnlyHint=True) - - The middleware: - - Filters the tools list in on_list_tools() to hide unauthorized tools - - Blocks unauthorized tool calls in on_call_tool() with a ToolError - """ - - @staticmethod - def _get_authorization_config( - http_rq: Request | None = None, - ) -> tuple[set[str] | None, set[str] | None, bool]: - """ - Determines the authorization configuration for the current request based on HTTP headers. - - Returns a tuple of (allowed_tools, disallowed_tools, read_only_mode): - - allowed_tools: Set of allowed tool names, or None if all tools are allowed - - disallowed_tools: Set of tool names to exclude, or None if no tools are explicitly disallowed - - read_only_mode: Whether X-Read-Only-Mode header is enabled - - :param http_rq: Explicit request to read headers from. Falls back to the FastMCP request - context when omitted. Raw Starlette routes (e.g. /preview/configuration) must pass it - explicitly because the FastMCP request contextvar is not populated for them. - """ - if http_rq is None: - http_rq = get_http_request_or_none() - if not http_rq: - # No HTTP request means no authorization headers are present, so we do not apply any filters. - return None, None, False - - allowed_tools: set[str] | None = None - disallowed_tools: set[str] | None = None - read_only_mode = False - - # Check X-Allowed-Tools header for explicit tool list - if header_tools := http_rq.headers.get('X-Allowed-Tools'): - parsed_tools = set(t.strip() for t in header_tools.split(',') if t.strip()) - if parsed_tools: - allowed_tools = parsed_tools - LOG.info(f'Tool authorization: X-Allowed-Tools={sorted(allowed_tools)}') - - # Check X-Read-Only-Mode header - if http_rq.headers.get('X-Read-Only-Mode', '').lower() in ('true', '1', 'yes'): - read_only_mode = True - LOG.info('Tool authorization: X-Read-Only-Mode=true') - - # Check X-Disallowed-Tools header for tools to exclude - if header_disallowed := http_rq.headers.get('X-Disallowed-Tools'): - parsed_tools = set(t.strip() for t in header_disallowed.split(',') if t.strip()) - if parsed_tools: - disallowed_tools = parsed_tools - LOG.info(f'Tool authorization: X-Disallowed-Tools={sorted(disallowed_tools)}') - - return allowed_tools, disallowed_tools, read_only_mode - - @staticmethod - def _is_tool_name_authorized( - tool_name: str, - is_read_only: bool, - allowed_tools: set[str] | None, - disallowed_tools: set[str] | None, - read_only_mode: bool, - ) -> bool: - """ - Header-based (X-Allowed-Tools / X-Disallowed-Tools / X-Read-Only-Mode) authorization decision - for a single tool identified by name. - - This is the single source of truth for the header-based gating. :meth:`_is_tool_authorized` - uses it for the MCP middleware path; the raw ``/preview/configuration`` Starlette route reuses - it (see ``preview.py``) so the preview path enforces exactly the same rules. - """ - # First check if tool is in disallowed list (if any disallow filter is configured) - if disallowed_tools and tool_name in disallowed_tools: - return False - # Check read-only mode - only allow tools with readOnlyHint=True - if read_only_mode and not is_read_only: - return False - # Then check if tool is in allowed list (if specified) - if allowed_tools is not None and tool_name not in allowed_tools: - return False - return True - - @staticmethod - def _is_tool_authorized( - tool: Tool, allowed_tools: set[str] | None, disallowed_tools: set[str] | None, read_only_mode: bool - ) -> bool: - """Check if a tool is authorized based on allowed/disallowed sets and read-only mode.""" - return ToolAuthorizationMiddleware._is_tool_name_authorized( - tool.name, is_read_only_tool(tool), allowed_tools, disallowed_tools, read_only_mode - ) - - async def on_list_tools( - self, context: MiddlewareContext[mt.ListToolsRequest], call_next: CallNext[mt.ListToolsRequest, list[Tool]] - ) -> list[Tool]: - """Filters the tools list to only include authorized tools.""" - tools = await call_next(context) - - allowed_tools, disallowed_tools, read_only_mode = self._get_authorization_config() - if allowed_tools is None and not disallowed_tools and not read_only_mode: - return tools - - filtered_tools = [ - t for t in tools if self._is_tool_authorized(t, allowed_tools, disallowed_tools, read_only_mode) - ] - LOG.debug(f'Tool authorization: filtered {len(tools)} tools to {len(filtered_tools)} allowed tools') - return filtered_tools - - async def on_call_tool( - self, - context: MiddlewareContext[mt.CallToolRequestParams], - call_next: CallNext[mt.CallToolRequestParams, mt.CallToolResult], - ) -> mt.CallToolResult: - """Blocks calls to unauthorized tools.""" - tool_name = context.message.name - allowed_tools, disallowed_tools, read_only_mode = self._get_authorization_config() - - # For on_call_tool, we need to get the tool to check its annotations - tool = await context.fastmcp_context.fastmcp.get_tool(tool_name) - - if not self._is_tool_authorized(tool, allowed_tools, disallowed_tools, read_only_mode): - LOG.info(f'Tool authorization denied: {tool_name} not authorized') - raise ToolError( - f'Access denied: The tool "{tool_name}" is not authorized for this client. ' - f'Contact your administrator to request access.' - ) - - return await call_next(context) diff --git a/src/keboola_mcp_server/cli.py b/src/keboola_mcp_server/cli.py deleted file mode 100644 index 667457331..000000000 --- a/src/keboola_mcp_server/cli.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Command-line interface for the Keboola MCP server.""" - -import argparse -import asyncio -import contextlib -import json -import logging.config -import os -import pathlib -import sys -import traceback -from typing import Optional - -import pydantic -from fastmcp import FastMCP -from starlette.exceptions import HTTPException -from starlette.middleware import Middleware -from starlette.requests import Request -from starlette.responses import JSONResponse - -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.mcp import ForwardSlashMiddleware, is_read_only_tool, is_semantic_tool -from keboola_mcp_server.server import CustomRoutes, create_server - -LOG = logging.getLogger(__name__) - - -def parse_args(args: Optional[list[str]] = None) -> argparse.Namespace: - """Parses command line arguments.""" - parser = argparse.ArgumentParser( - prog='python -m keboola-mcp-server', - description='Keboola MCP Server', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - '--transport', - choices=['stdio', 'streamable-http', 'http-compat'], - default='stdio', - help='Transport to use for MCP communication', - ) - parser.add_argument( - '--log-level', - choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], - default='INFO', - help='Logging level', - ) - parser.add_argument( - '--api-url', - metavar='URL', - help=( - 'Keboola Storage API URL using format of https://connection..keboola.com. Example: For AWS region ' - '"eu-central-1", use: https://connection.eu-central-1.keboola.com' - ), - ) - parser.add_argument('--storage-token', metavar='STR', help='Keboola Storage API token.') - parser.add_argument('--workspace-schema', metavar='STR', help='Keboola Storage API workspace schema.') - parser.add_argument('--host', default='localhost', metavar='STR', help='The host to listen on.') - parser.add_argument('--port', type=int, default=8000, metavar='INT', help='The port to listen on.') - parser.add_argument('--log-config', type=pathlib.Path, metavar='PATH', help='Logging config file.') - - return parser.parse_args(args) - - -def _create_exception_handler(status_code: int = 500, log_exception: bool = False): - """ - Returns a JSON message response for all unhandled errors from request handlers. The response JSON body - will show exception message and traceback (if the app runs in the debug mode). - - :param status_code: the HTTP status code to return; if not specified 500 (Server Error) status code is used - """ - - async def _exception_handler(request: Request, exc): - exc_str = f'{type(exc).__name__}: {exc}' - if log_exception: - LOG.exception(f'Unhandled error: {exc_str}') - - if request.app.debug: - exc_type, exc_value, exc_traceback = sys.exc_info() - exc_text = ''.join(traceback.format_exception(exc_type, exc_value, exc_traceback)) - return JSONResponse({'message': exc_str, 'exception': exc_text}, status_code) - - else: - return JSONResponse({'message': exc_str}, status_code) - - return _exception_handler - - -async def _http_exception_handler(request: Request, exc: HTTPException): - return JSONResponse({'message': exc.detail}, status_code=exc.status_code) - - -_bad_request_handler = _create_exception_handler(status_code=400) -_exception_handlers = { - HTTPException: _http_exception_handler, - json.JSONDecodeError: _bad_request_handler, - pydantic.ValidationError: _bad_request_handler, - ValueError: _bad_request_handler, - Exception: _create_exception_handler(status_code=500, log_exception=True), -} - - -async def run_server(args: Optional[list[str]] = None) -> None: - """Runs the MCP server in async mode.""" - parsed_args = parse_args(args) - - log_config: pathlib.Path | None = parsed_args.log_config - if not log_config and os.environ.get('LOG_CONFIG'): - log_config = pathlib.Path(os.environ.get('LOG_CONFIG')) - if log_config and not log_config.is_file(): - LOG.warning(f'Invalid log config file: {log_config}. Using default logging configuration.') - log_config = None - - if log_config: - # remove fastmcp's rich handler, which is aggressively set up during "import fastmcp" - fastmcp_logger = logging.getLogger('fastmcp') - for hdlr in fastmcp_logger.handlers[:]: - fastmcp_logger.removeHandler(hdlr) - fastmcp_logger.propagate = True - fastmcp_logger.setLevel(logging.NOTSET) - logging.config.fileConfig(log_config, disable_existing_loggers=False) - else: - logging.basicConfig( - format='%(asctime)s %(name)s %(levelname)s: %(message)s', - level=parsed_args.log_level, - stream=sys.stderr, - ) - - # Create config from the CLI arguments - config = Config( - storage_api_url=parsed_args.api_url, - storage_token=parsed_args.storage_token, - workspace_schema=parsed_args.workspace_schema, - ) - - try: - # Create and run the server - if parsed_args.transport == 'stdio': - runtime_config = ServerRuntimeInfo(transport=parsed_args.transport) - keboola_mcp_server: FastMCP = create_server(config, runtime_info=runtime_config) - if config.oauth_client_id or config.oauth_client_secret: - raise RuntimeError('OAuth authorization can only be used with HTTP-based transports.') - await keboola_mcp_server.run_async(transport=parsed_args.transport) - else: - # 'http-compat' is an alias for 'streamable-http' kept for backwards compatibility. - # We use local imports here due to the temporary nature of this code. - - from contextlib import asynccontextmanager - - import uvicorn - from fastmcp.server.http import StarletteWithLifespan - from starlette.applications import Starlette - - mount_paths: dict[str, StarletteWithLifespan] = {} - custom_routes: CustomRoutes | None = None - transports: list[str] = [] - mcp_server: FastMCP | None = None - - if parsed_args.transport in ['http-compat', 'streamable-http']: - http_runtime_config = ServerRuntimeInfo('http-compat/streamable-http') - mcp_server, custom_routes = create_server( - config, runtime_info=http_runtime_config, custom_routes_handling='return' - ) - http_app: StarletteWithLifespan = mcp_server.http_app( - path='/', - transport='streamable-http', - stateless_http=True, - ) - mount_paths['/mcp'] = http_app - transports.append('Streamable-HTTP') - - @asynccontextmanager - async def lifespan(_app: Starlette): - async with contextlib.AsyncExitStack() as stack: - for _inner_app in mount_paths.values(): - await stack.enter_async_context(_inner_app.lifespan(_app)) - yield - - app = Starlette( - middleware=[Middleware(ForwardSlashMiddleware)], - lifespan=lifespan, - exception_handlers=_exception_handlers, - ) - for path, inner_app in mount_paths.items(): - app.mount(path, inner_app) - - custom_routes.add_to_starlette(app) - - assert isinstance(mcp_server, FastMCP) - _tools = await mcp_server.list_tools(run_middleware=False) - app.state.mcp_tools_input_schema = {tool.name: tool.parameters for tool in _tools} - # Used by the /preview/configuration authorization check to enforce X-Read-Only-Mode - # and the ToolsFilteringMiddleware-parity gating (read-only role, semantic feature). - app.state.mcp_read_only_tools = {tool.name for tool in _tools if is_read_only_tool(tool)} - app.state.mcp_semantic_tools = {tool.name for tool in _tools if is_semantic_tool(tool)} - - config = uvicorn.Config( - app, - host=parsed_args.host, - port=parsed_args.port, - log_config=log_config, - timeout_graceful_shutdown=0, - lifespan='on', - ) - server = uvicorn.Server(config) - LOG.info( - f'Starting MCP server with {", ".join(transports)} transport{"s" if len(transports) > 1 else ""}' - f' on http://{parsed_args.host}:{parsed_args.port}/' - ) - - await server.serve() - - except Exception as e: - LOG.exception(f'Server failed: {e}') - sys.exit(1) - - -def main(args: Optional[list[str]] = None) -> None: - asyncio.run(run_server(args)) - - -if __name__ == '__main__': - main() diff --git a/src/keboola_mcp_server/clients/__init__.py b/src/keboola_mcp_server/clients/__init__.py deleted file mode 100644 index 5a7962c31..000000000 --- a/src/keboola_mcp_server/clients/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -from keboola_mcp_server.clients.ai_service import AIServiceClient -from keboola_mcp_server.clients.base import KeboolaServiceClient, RawKeboolaClient -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.encryption import EncryptionClient -from keboola_mcp_server.clients.jobs_queue import JobsQueueClient -from keboola_mcp_server.clients.metastore import MetastoreClient -from keboola_mcp_server.clients.scheduler import SchedulerClient -from keboola_mcp_server.clients.storage import AsyncStorageClient - -__all__ = [ - 'KeboolaClient', - 'EncryptionClient', - 'AsyncStorageClient', - 'AIServiceClient', - 'JobsQueueClient', - 'MetastoreClient', - 'SchedulerClient', - 'RawKeboolaClient', - 'KeboolaServiceClient', -] diff --git a/src/keboola_mcp_server/clients/ai_service.py b/src/keboola_mcp_server/clients/ai_service.py deleted file mode 100644 index a08559cd1..000000000 --- a/src/keboola_mcp_server/clients/ai_service.py +++ /dev/null @@ -1,96 +0,0 @@ -from typing import Any, Optional, cast - -from pydantic import AliasChoices, BaseModel, Field - -from keboola_mcp_server.clients.base import JsonDict, KeboolaServiceClient, RawKeboolaClient - - -class DocsQuestionResponse(BaseModel): - """ - The AI service response to a request to `/docs/question` endpoint. - """ - - text: str = Field(description='Text of the answer to a documentation query.') - source_urls: list[str] = Field( - description='List of URLs to the sources of the answer.', - default_factory=list, - alias='sourceUrls', - ) - - -class SuggestedComponent(BaseModel): - """The AI service response to a /docs/suggest-component request.""" - - component_id: str = Field( - description='The component ID.', validation_alias=AliasChoices('componentId', 'component_id') - ) - score: float = Field(description='Score of the component suggestion.') - source: str = Field(description='Source of the component suggestion.') - - -class ComponentSuggestionResponse(BaseModel): - """The AI service response to a /suggest/component request.""" - - components: list[SuggestedComponent] = Field(description='List of suggested components.', default_factory=list) - - -class AIServiceClient(KeboolaServiceClient): - """Async client for Keboola AI Service.""" - - @classmethod - def create( - cls, - root_url: str, - token: Optional[str], - headers: dict[str, Any] | None = None, - readonly: bool | None = None, - ) -> 'AIServiceClient': - """ - Creates an AIServiceClient from a Keboola Storage API token. - - :param root_url: The root URL of the AI service API. - :param token: The Keboola Storage API token. If None, the client will not send any authorization header. - :param headers: Additional headers for the requests. - :param readonly: If True, the client will only use HTTP GET, HEAD operations. - :return: A new instance of AIServiceClient. - """ - return cls( - raw_client=RawKeboolaClient(base_api_url=root_url, api_token=token, headers=headers, readonly=readonly) - ) - - async def get_component_detail(self, component_id: str) -> JsonDict: - """ - Retrieves information about a given component. - - :param component_id: The id of the component. - :return: Component details as dictionary. - """ - return cast(JsonDict, await self.get(endpoint=f'docs/components/{component_id}')) - - async def docs_question(self, query: str) -> DocsQuestionResponse: - """ - Answers a question using the Keboola documentation as a source. - :param query: The query to answer. - :return: Response containing the answer and source URLs. - """ - response = await self.raw_client.post( - endpoint='docs/question', - data={'query': query}, - headers={'Accept': 'application/json'}, - ) - - return DocsQuestionResponse.model_validate(response) - - async def suggest_component(self, query: str) -> ComponentSuggestionResponse: - """ - Provides list of component suggestions based on natural language query. - :param query: The query to answer. - :return: Response containing the list of suggested component IDs, their score and source. - """ - response = await self.raw_client.post( - endpoint='suggest/component', - data={'prompt': query}, - headers={'Accept': 'application/json'}, - ) - - return ComponentSuggestionResponse.model_validate(response) diff --git a/src/keboola_mcp_server/clients/base.py b/src/keboola_mcp_server/clients/base.py deleted file mode 100644 index 110689d25..000000000 --- a/src/keboola_mcp_server/clients/base.py +++ /dev/null @@ -1,374 +0,0 @@ -import json -import logging -from http import HTTPStatus -from typing import Any, Optional, Union, cast - -import httpx -from httpx_retries import Retry, RetryTransport - -JsonPrimitive = Union[int, float, str, bool, None] -JsonDict = dict[str, Union[JsonPrimitive, 'JsonStruct']] -JsonList = list[Union[JsonPrimitive, 'JsonStruct']] -JsonStruct = Union[JsonDict, JsonList] - -LOG = logging.getLogger(__name__) - - -class RawKeboolaClient: - """ - Raw async client for Keboola services. - - Implements the basic HTTP methods (GET, POST, PUT, DELETE) - and can be used to implement high-level functions in clients for individual services. - """ - - def __init__( - self, - base_api_url: str, - api_token: Optional[str], - headers: dict[str, Any] | None = None, - timeout: httpx.Timeout | None = None, - readonly: bool | None = None, - ) -> None: - self.base_api_url = base_api_url - self.headers = { - 'Content-Type': 'application/json', - 'Accept-Encoding': 'gzip', - } - if api_token: - if api_token.startswith('Bearer '): - self.headers['Authorization'] = api_token - else: - self.headers['X-StorageAPI-Token'] = api_token - self.timeout = timeout or httpx.Timeout(connect=5.0, read=60.0, write=10.0, pool=5.0) - # Store retry config, not the transport - transports cannot be shared across concurrent AsyncClient instances - self._retry = Retry( - total=3, - backoff_factor=1.0, - max_backoff_wait=10, - status_forcelist=frozenset(Retry.RETRYABLE_STATUS_CODES | {HTTPStatus.CONFLICT}), - ) - if headers: - self.headers.update(headers) - self.readonly = readonly - - def _create_transport(self) -> RetryTransport: - """ - Creates a new RetryTransport instance. Each AsyncClient instance needs its own transport. - The transports cannot be shared among the AsyncClient instances. - """ - return RetryTransport(retry=self._retry) - - @staticmethod - def _raise_for_status(response: httpx.Response) -> None: - """ - Checks the HTTP response status code and raises an exception with a detailed message. The message will - include "error" and "exceptionId" fields if they are present in the response. - """ - try: - response.raise_for_status() - except httpx.HTTPStatusError as e: - message_parts = [str(e)] - - try: - error_data = response.json() - LOG.error(f'API error data: {error_data}') - - if error_msg := error_data.get('exception'): - # Query Service error message - message_parts.append(f'API error: {error_msg}') - - elif error_msg := error_data.get('error'): - # SAPI error message - message_parts.append(f'API error: {error_msg}') - - if exception_id := error_data.get('exceptionId'): - message_parts.append(f'Exception ID: {exception_id}') - message_parts.append('When contacting Keboola support please provide the exception ID.') - - except ValueError: - try: - if response.text: - message_parts.append(f'API error: {response.text}') - except Exception: - pass # should never get here - - raise httpx.HTTPStatusError('\n'.join(message_parts), request=response.request, response=response) from e - - async def get( - self, - endpoint: str, - params: dict[str, Any] | None = None, - headers: dict[str, Any] | None = None, - ) -> JsonStruct: - """ - Makes a GET request to the service API. - - :param endpoint: API endpoint to call - :param params: Query parameters for the request - :param headers: Additional headers for the request - :return: API response as dictionary - """ - headers = self.headers | (headers or {}) - async with httpx.AsyncClient(timeout=self.timeout, transport=self._create_transport()) as client: - response = await client.get( - f'{self.base_api_url}/{endpoint}', - params=params, - headers=headers, - ) - self._raise_for_status(response) - return cast(JsonStruct, response.json()) - - async def get_text( - self, - endpoint: str, - params: dict[str, Any] | None = None, - headers: dict[str, Any] | None = None, - ) -> str: - """ - Makes a GET request to the service API and returns the response as text. - - :param endpoint: API endpoint to call - :param params: Query parameters for the request - :param headers: Additional headers for the request - :return: API response as text - """ - headers = self.headers | (headers or {}) - async with httpx.AsyncClient(timeout=self.timeout, transport=self._create_transport()) as client: - response = await client.get( - f'{self.base_api_url}/{endpoint}', - params=params, - headers=headers, - ) - self._raise_for_status(response) - return cast(str, response.text) - - async def post( - self, - endpoint: str, - data: dict[str, Any] | None = None, - params: dict[str, Any] | None = None, - headers: dict[str, Any] | None = None, - timeout: httpx.Timeout | None = None, - ) -> JsonStruct: - """ - Makes a POST request to the service API. - - :param endpoint: API endpoint to call - :param data: Request payload - :param params: Query parameters for the request - :param headers: Additional headers for the request - :param timeout: Optional per-call timeout override; falls back to the client default when None - :return: API response as dictionary - """ - if self.readonly: - raise RuntimeError(f'Forbidden POST operation on a readonly client: {self.base_api_url}') - - headers = self.headers | (headers or {}) - async with httpx.AsyncClient(timeout=timeout or self.timeout, transport=self._create_transport()) as client: - response = await client.post( - f'{self.base_api_url}/{endpoint}', - params=params, - headers=headers, - content=json.dumps(data or {}, ensure_ascii=False).encode('utf-8'), - ) - self._raise_for_status(response) - return cast(JsonStruct, response.json()) - - async def put( - self, - endpoint: str, - data: dict[str, Any] | None = None, - params: dict[str, Any] | None = None, - headers: dict[str, Any] | None = None, - ) -> JsonStruct: - """ - Makes a PUT request to the service API. - - :param endpoint: API endpoint to call - :param data: Request payload - :param params: Query parameters for the request - :param headers: Additional headers for the request - :return: API response as dictionary - """ - if self.readonly: - raise RuntimeError(f'Forbidden PUT operation on a readonly client: {self.base_api_url}') - - headers = self.headers | (headers or {}) - async with httpx.AsyncClient(timeout=self.timeout, transport=self._create_transport()) as client: - response = await client.put( - f'{self.base_api_url}/{endpoint}', - params=params, - headers=headers, - content=json.dumps(data or {}, ensure_ascii=False).encode('utf-8'), - ) - self._raise_for_status(response) - return cast(JsonStruct, response.json()) - - async def delete( - self, - endpoint: str, - headers: dict[str, Any] | None = None, - ) -> JsonStruct | None: - """ - Makes a DELETE request to the service API. - - :param endpoint: API endpoint to call - :param headers: Additional headers for the request - :return: API response as dictionary - """ - if self.readonly: - raise RuntimeError(f'Forbidden DELETE operation on a readonly client: {self.base_api_url}') - - headers = self.headers | (headers or {}) - async with httpx.AsyncClient(timeout=self.timeout, transport=self._create_transport()) as client: - response = await client.delete( - f'{self.base_api_url}/{endpoint}', - headers=headers, - ) - self._raise_for_status(response) - - if response.content: - return cast(JsonStruct, response.json()) - - return None - - async def patch( - self, - endpoint: str, - data: Optional[dict[str, Any]] = None, - params: Optional[dict[str, Any]] = None, - headers: Optional[dict[str, Any]] = None, - ) -> JsonStruct: - """ - Makes a PATCH request to the service API. - - :param endpoint: API endpoint to call - :param data: Request payload - :param params: Query parameters for the request - :param headers: Additional headers for the request - :return: API response as dictionary - """ - if self.readonly: - raise RuntimeError(f'Forbidden PATCH operation on a readonly client: {self.base_api_url}') - - headers = self.headers | (headers or {}) - async with httpx.AsyncClient(timeout=self.timeout, transport=self._create_transport()) as client: - response = await client.patch( - f'{self.base_api_url}/{endpoint}', - params=params, - headers=headers, - content=json.dumps(data or {}, ensure_ascii=False).encode('utf-8'), - ) - self._raise_for_status(response) - return cast(JsonStruct, response.json()) - - -class KeboolaServiceClient: - """ - Base class for Keboola service clients. - - Implements the basic HTTP methods (GET, POST, PUT, DELETE) - and is used as a base class for clients for individual services. - """ - - def __init__(self, raw_client: RawKeboolaClient) -> None: - """ - Creates a client instance. - - The inherited classes should implement the `create` method - rather than overriding this constructor. - - :param raw_client: The raw client to use - """ - self.raw_client = raw_client - - async def get( - self, - endpoint: str, - params: Optional[dict[str, Any]] = None, - ) -> JsonStruct: - """ - Makes a GET request to the service API. - - :param endpoint: API endpoint to call - :param params: Query parameters for the request - :return: API response as dictionary - """ - return await self.raw_client.get(endpoint=endpoint, params=params) - - async def get_text( - self, - endpoint: str, - params: Optional[dict[str, Any]] = None, - ) -> str: - """ - Makes a GET request to the service API. - - :param endpoint: API endpoint to call - :param params: Query parameters for the request - :return: API response as text - """ - return await self.raw_client.get_text(endpoint=endpoint, params=params) - - async def post( - self, - endpoint: str, - data: Optional[dict[str, Any]] = None, - params: Optional[dict[str, Any]] = None, - timeout: Optional[httpx.Timeout] = None, - ) -> JsonStruct: - """ - Makes a POST request to the service API. - - :param endpoint: API endpoint to call - :param data: Request payload - :param params: Query parameters for the request - :param timeout: Optional per-call timeout override; falls back to the client default when None - :return: API response as dictionary - """ - return await self.raw_client.post(endpoint=endpoint, data=data, params=params, timeout=timeout) - - async def put( - self, - endpoint: str, - data: Optional[dict[str, Any]] = None, - params: Optional[dict[str, Any]] = None, - ) -> JsonStruct: - """ - Makes a PUT request to the service API. - - :param endpoint: API endpoint to call - :param data: Request payload - :param params: Query parameters for the request - :return: API response as dictionary - """ - return await self.raw_client.put(endpoint=endpoint, data=data, params=params) - - async def delete( - self, - endpoint: str, - ) -> JsonStruct | None: - """ - Makes a DELETE request to the service API. - - :param endpoint: API endpoint to call - :return: API response as dictionary - """ - return await self.raw_client.delete(endpoint=endpoint) - - async def patch( - self, - endpoint: str, - data: Optional[dict[str, Any]] = None, - params: Optional[dict[str, Any]] = None, - ) -> JsonStruct: - """ - Makes a PATCH request to the service API. - - :param endpoint: API endpoint to call - :param data: Request payload - :param params: Query parameters for the request - :return: API response as dictionary - """ - return await self.raw_client.patch(endpoint=endpoint, data=data, params=params) diff --git a/src/keboola_mcp_server/clients/client.py b/src/keboola_mcp_server/clients/client.py deleted file mode 100644 index a8187ce1e..000000000 --- a/src/keboola_mcp_server/clients/client.py +++ /dev/null @@ -1,296 +0,0 @@ -"""Keboola Storage API client wrapper.""" - -import logging -from typing import Any, Literal, Mapping, Sequence, TypeVar -from urllib.parse import urlparse, urlunparse - -import httpx - -from keboola_mcp_server.clients.ai_service import AIServiceClient -from keboola_mcp_server.clients.data_science import DataScienceClient -from keboola_mcp_server.clients.encryption import EncryptionClient -from keboola_mcp_server.clients.jobs_queue import JobsQueueClient -from keboola_mcp_server.clients.metastore import MetastoreClient -from keboola_mcp_server.clients.scheduler import SchedulerClient -from keboola_mcp_server.clients.storage import AsyncStorageClient, JsonDict -from keboola_mcp_server.clients.sync_actions import SyncActionsClient - -LOG = logging.getLogger(__name__) - -T = TypeVar('T') - -# Input types for the global search endpoint parameters -BranchType = Literal['production', 'development'] - - -ORCHESTRATOR_COMPONENT_ID = 'keboola.orchestrator' -CONDITIONAL_FLOW_COMPONENT_ID = 'keboola.flow' -DATA_APP_COMPONENT_ID = 'keboola.data-apps' -FlowType = Literal['keboola.flow', 'keboola.orchestrator'] -FLOW_TYPES: Sequence[FlowType] = (CONDITIONAL_FLOW_COMPONENT_ID, ORCHESTRATOR_COMPONENT_ID) - - -def get_metadata_property( - metadata: list[Mapping[str, Any]], - key: str, - *, - provider: str | None = None, - preferred_providers: list[str] | None = None, - default: T | None = None, -) -> T | None: - """ - Gets the value of a metadata property based on the provided key and optional provider. If multiple metadata entries - exist with the same key, the most recent one is returned. - - :param metadata: A list of metadata entries. - :param key: The metadata property key to search for. - :param provider: Specifies the metadata provider name to filter by. - :param preferred_providers: Specifies a list of preferred metadata providers to order the metadata items by. - :param default: The default value to return if the metadata property is not found. - - :return: The value of the most recent matching metadata entry if found, or None otherwise. - """ - if provider and preferred_providers: - raise ValueError('Specifying both provider and preferred_providers makes no sense.') - - def _sort_key(m: Mapping[str, Any]) -> tuple[Any, ...]: - # TODO: ideally we should first convert the timestamps to UTC - if preferred_providers: - if (_p := m.get('provider')) and _p in preferred_providers: - _pidx = preferred_providers.index(_p) - else: - _pidx = len(preferred_providers) - return -1 * _pidx, m.get('timestamp') or '' - else: - return (m.get('timestamp') or '',) - - filtered = [ - m for m in metadata if m['key'] == key and (not provider or ('provider' in m and m['provider'] == provider)) - ] - item = max(filtered, key=_sort_key, default=None) - value = item.get('value') if item else None - return value if value is not None else default - - -class KeboolaClient: - """Class holding clients for Keboola APIs: Storage API, Job Queue API, and AI Service.""" - - STATE_KEY = 'sapi_client' - - @classmethod - def from_state(cls, state: Mapping[str, Any]) -> 'KeboolaClient': - instance = state[cls.STATE_KEY] - assert isinstance(instance, KeboolaClient), f'Expected KeboolaClient, got: {instance}' - return instance - - async def with_branch_id(self, branch_id: str | None) -> 'KeboolaClient': - """ - Gets a KeboolaClient configured for the given branch. It verifies that the branch exists - and normalizes the default-branch ID to None. - """ - if branch_id == self.branch_id: - return self - elif not branch_id: - return KeboolaClient( - storage_api_url=self.storage_api_url, - storage_api_token=self.token, - bearer_token=self._bearer_token, - branch_id=None, - headers=self._headers, - ) - else: - try: - detail = await self.storage_client.dev_branch_detail(branch_id) - is_default = detail.get('isDefault') is True - - except httpx.HTTPStatusError as exc: - if exc.response.status_code == 404: - message = f'Branch "{branch_id}" not found' - LOG.error(f'{message}: {exc.response.text}') - raise httpx.HTTPStatusError(message, request=exc.request, response=exc.response) from exc - else: - LOG.error(f'Failed to get details of "{branch_id}" branch: {exc.response.text}') - raise exc - - # Converts the branch id referring to the main/production branch to None as we expect - normalized_branch_id = None if is_default else branch_id - return KeboolaClient( - storage_api_url=self.storage_api_url, - storage_api_token=self.token, - bearer_token=self._bearer_token, - branch_id=normalized_branch_id, - headers=self._headers, - ) - - def __init__( - self, - *, - storage_api_url: str, - storage_api_token: str, - bearer_token: str | None = None, - branch_id: str | None = None, - headers: Mapping[str, Any] | None = None, - readonly: bool | None = None, - ) -> None: - """ - Initialize the client. - - :param storage_api_token: Keboola Storage API token - :param storage_api_url: Keboola Storage API URL - :param bearer_token: The access token issued by Keboola OAuth server - :param branch_id: Keboola branch ID - :param headers: Additional headers for the requests sent by all clients - :param readonly: If True, the client will only use HTTP GET, HEAD operations. - """ - self._token = storage_api_token - self._bearer_token = bearer_token - self._branch_id = branch_id - self._headers = dict(headers) if headers else None - self._features_cache: set[str] | None = None - # Session-scoped cache of flow configuration schemas keyed by flow type (component id). - # Mirrors _features_cache: fetched once per session so it is never stale across runs. - self._flow_schema_cache: dict[str, JsonDict] = {} - - sapi_url_parsed = urlparse(storage_api_url) - if not sapi_url_parsed.hostname or not sapi_url_parsed.hostname.startswith('connection.'): - raise ValueError(f'Invalid Keboola Storage API URL: {storage_api_url}') - - self._hostname_suffix = sapi_url_parsed.hostname.split('connection.')[1] - self._storage_api_url = urlunparse(('https', f'connection.{self._hostname_suffix}', '', '', '', '')) - metastore_api_url = urlunparse(('https', f'metastore.{self._hostname_suffix}', '', '', '', '')) - queue_api_url = urlunparse(('https', f'queue.{self._hostname_suffix}', '', '', '', '')) - ai_service_api_url = urlunparse(('https', f'ai.{self._hostname_suffix}', '', '', '', '')) - data_science_api_url = urlunparse(('https', f'data-science.{self._hostname_suffix}', '', '', '', '')) - encryption_api_url = urlunparse(('https', f'encryption.{self._hostname_suffix}', '', '', '', '')) - scheduler_api_url = urlunparse(('https', f'scheduler.{self._hostname_suffix}', '', '', '', '')) - sync_actions_api_url = urlunparse(('https', f'sync-actions.{self._hostname_suffix}', '', '', '', '')) - - # Initialize clients for individual services - bearer_or_sapi_token = f'Bearer {bearer_token}' if bearer_token else self._token - # The encryption service does not require an authorization header, so we pass None as the token - self._encryption_client = EncryptionClient.create( - root_url=encryption_api_url, token=None, headers=self._headers - ) - self._storage_client = AsyncStorageClient.create( - root_url=self._storage_api_url, - token=bearer_or_sapi_token, - branch_id=branch_id, - headers=self._headers, - readonly=readonly, - encryption_client=self._encryption_client, - ) - self._jobs_queue_client = JobsQueueClient.create( - root_url=queue_api_url, token=self._token, branch_id=branch_id, headers=self._headers, readonly=readonly - ) - self._ai_service_client = AIServiceClient.create( - root_url=ai_service_api_url, token=self._token, headers=self._headers, readonly=readonly - ) - # Data-science (sandboxes-service) git-repo credential endpoints require an admin-context - # token (CanManageAppRepoCredentials -> StorageApiToken::isAdminToken()). The OAuth bearer - # token carries admin context; the SAPI token minted for OAuth sessions does not. Pass the - # bearer token when available so credential minting works for OAuth clients (falls back to - # the SAPI token otherwise). See AI-3398. - self._data_science_client = DataScienceClient.create( - root_url=data_science_api_url, - token=bearer_or_sapi_token, - branch_id=branch_id, - headers=self._headers, - readonly=readonly, - ) - self._scheduler_client = SchedulerClient.create( - root_url=scheduler_api_url, token=bearer_or_sapi_token, headers=self._headers, readonly=readonly - ) - self._sync_actions_client = SyncActionsClient.create( - root_url=sync_actions_api_url, - token=self._token, - branch_id=branch_id, - headers=self._headers, - readonly=readonly, - ) - self._metastore_client = MetastoreClient.create( - root_url=metastore_api_url, - token=bearer_or_sapi_token, - branch_id=branch_id, - headers=self._headers, - readonly=readonly, - ) - - @property - def hostname_suffix(self) -> str: - return self._hostname_suffix - - @property - def storage_api_url(self) -> str: - return self._storage_api_url - - @property - def token(self) -> str: - return self._token - - @property - def bearer_token(self) -> str | None: - """ - Gets the OAuth bearer token issued by Keboola OAuth server, if available. - Returns None if only storage token authentication is used. - """ - return self._bearer_token - - @property - def branch_id(self) -> str | None: - """ - Gets ID of the Keboola branch that the MCP server is bound to or None if it's bound - to the main/production branch. - """ - return self._branch_id - - async def has_feature(self, feature: str) -> bool: - """Checks if the project has a specific feature enabled. Results are cached.""" - if self._features_cache is None: - token_info = await self._storage_client.verify_token() - owner = token_info.get('owner', {}) - self._features_cache = set(owner.get('features', []) if isinstance(owner, dict) else []) - return feature in self._features_cache - - def get_cached_flow_schema(self, flow_type: str) -> JsonDict | None: - """Return the cached configuration schema for the given flow type, or None if not cached.""" - return self._flow_schema_cache.get(flow_type) - - def cache_flow_schema(self, flow_type: str, schema: JsonDict) -> None: - """Cache the configuration schema for the given flow type for the rest of the session.""" - self._flow_schema_cache[flow_type] = schema - - @property - def headers(self) -> dict[str, Any] | None: - return dict(self._headers) if self._headers else None - - @property - def storage_client(self) -> 'AsyncStorageClient': - return self._storage_client - - @property - def jobs_queue_client(self) -> 'JobsQueueClient': - return self._jobs_queue_client - - @property - def ai_service_client(self) -> 'AIServiceClient': - return self._ai_service_client - - @property - def data_science_client(self) -> 'DataScienceClient': - return self._data_science_client - - @property - def encryption_client(self) -> 'EncryptionClient': - return self._encryption_client - - @property - def scheduler_client(self) -> 'SchedulerClient': - return self._scheduler_client - - @property - def sync_actions_client(self) -> 'SyncActionsClient': - return self._sync_actions_client - - @property - def metastore_client(self) -> 'MetastoreClient': - return self._metastore_client diff --git a/src/keboola_mcp_server/clients/data_science.py b/src/keboola_mcp_server/clients/data_science.py deleted file mode 100644 index e51a29be0..000000000 --- a/src/keboola_mcp_server/clients/data_science.py +++ /dev/null @@ -1,600 +0,0 @@ -import logging -from datetime import datetime -from typing import Any, Union, cast - -from pydantic import AliasChoices, BaseModel, ConfigDict, Field - -from keboola_mcp_server.clients.base import KeboolaServiceClient, RawKeboolaClient - -LOG = logging.getLogger(__name__) - - -class DataAppResponse(BaseModel): - id: str = Field(validation_alias=AliasChoices('id', 'data_app_id'), description='The data app ID') - project_id: str = Field(validation_alias=AliasChoices('projectId', 'project_id'), description='The project ID') - component_id: str = Field( - validation_alias=AliasChoices('componentId', 'component_id'), description='The component ID' - ) - branch_id: str | None = Field(validation_alias=AliasChoices('branchId', 'branch_id'), description='The branch ID') - config_id: str = Field( - validation_alias=AliasChoices('configId', 'config_id'), description='The component config ID' - ) - config_version: str = Field( - validation_alias=AliasChoices('configVersion', 'config_version'), description='The config version' - ) - type: str = Field(description='The type of the data app') - state: str = Field(description='The state of the data app') - desired_state: str = Field( - validation_alias=AliasChoices('desiredState', 'desired_state'), description='The desired state' - ) - last_request_timestamp: str | None = Field( - validation_alias=AliasChoices('lastRequestTimestamp', 'last_request_timestamp'), - default=None, - description='The last request timestamp', - ) - last_start_timestamp: str | None = Field( - validation_alias=AliasChoices('lastStartTimestamp', 'last_start_timestamp'), - default=None, - description='The last start timestamp', - ) - url: str | None = Field( - validation_alias=AliasChoices('url', 'url'), description='The URL of the running data app', default=None - ) - auto_suspend_after_seconds: int | None = Field( - validation_alias=AliasChoices('autoSuspendAfterSeconds', 'auto_suspend_after_seconds'), - description='The auto suspend after seconds', - default=None, - ) - size: str | None = Field( - validation_alias=AliasChoices('size', 'size'), description='The size of the data app', default=None - ) - - -class DataAppConfig(BaseModel): - """ - The simplified data app config model, which is used for creating a data app within the mcp server. - """ - - class Parameters(BaseModel): - class DataApp(BaseModel): - slug: str = Field(description='The slug of the data app') - streamlit: dict[str, str] = Field( - description=( - 'The streamlit configuration, expected to have a key with TOML file name and the value with the ' - 'file content' - ) - ) - secrets: dict[str, str] | None = Field(description='The secrets of the data app', default=None) - - size: str = Field(description='The size of the data app') - auto_suspend_after_seconds: int = Field( - validation_alias=AliasChoices('autoSuspendAfterSeconds', 'auto_suspend_after_seconds'), - serialization_alias='autoSuspendAfterSeconds', - description='The auto suspend after seconds', - ) - data_app: DataApp = Field( - description='The data app sub config', - serialization_alias='dataApp', - validation_alias=AliasChoices('dataApp', 'data_app'), - ) - id: str | None = Field(description='The id of the data app', default=None) - script: list[str] | None = Field(description='The script of the data app', default=None) - packages: list[str] | None = Field( - description='The python packages needed to be installed in the data app', default=None - ) - - class Authorization(BaseModel): - class AppProxy(BaseModel): - auth_providers: list[dict[str, Any]] = Field(description='The auth providers') - auth_rules: list[dict[str, Any]] = Field(description='The auth rules') - - app_proxy: AppProxy = Field(description='The app proxy') - - parameters: Parameters = Field(description='The parameters of the data app') - authorization: Authorization = Field(description='The authorization of the data app') - # Optional with a None default so an app without storage mappings omits the key entirely. An - # empty object would be serialized as `[]` by the backend and break the Writable Tables editor - # (AI-3135); see `_prune_empty_storage_objects` in tools/data_apps.py. - storage: dict[str, Any] | None = Field(description='The storage of the data app', default=None) - - -class CodeDataAppConfig(BaseModel): - """ - Config model for python-js (code) data apps backed by a managed git repository. - - Unlike `DataAppConfig` (Streamlit), python-js apps don't embed source code in the config. - Code lives in the managed git repo; the config only carries deployment metadata - (slug, auto-suspend, optional runtime overrides). - """ - - model_config = ConfigDict(populate_by_name=True) - - class Parameters(BaseModel): - class DataApp(BaseModel): - class Git(BaseModel): - """External-git binding for a python-js draft data app. - - When set, the data-science API treats the app as externally configured: it does NOT - provision a managed repo for it, and the data-app runtime clones the configured - `repository` (at `branch`) using `username`/`#password` as HTTPS basic auth on every - deploy. - - Use case: a draft points at its parent prod app's managed repo, with credentials - minted on the prod app via `create_app_git_credential`. - """ - - model_config = ConfigDict(populate_by_name=True) - - repository: str = Field(description='HTTPS clone URL of the upstream managed git repo.') - username: str = Field( - description=( - 'Username for HTTPS basic auth. The git-service ignores this and only validates ' 'the token.' - ), - ) - password: str = Field( - validation_alias=AliasChoices('#password', 'password'), - serialization_alias='#password', - description=( - 'Encrypted HTTPS token (KBC::ConfigSecureGKMS::...). Must be passed through ' - 'EncryptionClient.encrypt before writing to Storage so the platform can ' - 'decrypt it at runtime.' - ), - ) - branch: str | None = Field( - default=None, - description='Branch to deploy from. None defaults to the platform default ("main").', - ) - - slug: str = Field(description='The slug of the data app (used as URL subdomain).') - secrets: dict[str, str] | None = Field( - description=( - 'Runtime secrets exposed to the data app as environment variables. ' - 'KBC_TOKEN/KBC_URL/BRANCH_ID are always injected by the platform and must not be ' - 'set here. WORKSPACE_ID is set by the platform only when ' - '`runtime.workspace.enabled = true`; on projects without the ' - '`data-apps-storage-workspace` feature, WORKSPACE_ID must be passed here instead.' - ), - default=None, - ) - git: 'CodeDataAppConfig.Parameters.DataApp.Git | None' = Field( - default=None, - description=( - "External-git binding. Set on drafts to point at the parent prod app's " - 'managed repo + a fresh prod-issued credential + the draft branch. Leave ' - 'unset on prod apps (which own their own managed repo via `useManagedGitRepo`).' - ), - ) - is_draft: bool | None = Field( - validation_alias=AliasChoices('isDraft', 'is_draft'), - serialization_alias='isDraft', - default=None, - description=( - 'When true, the UI hides this app from the main data-apps list and lists it ' - 'under its parent prod app instead. Set automatically on draft creation.' - ), - ) - parent_configuration_id: str | None = Field( - validation_alias=AliasChoices('parentConfigurationId', 'parent_configuration_id'), - serialization_alias='parentConfigurationId', - default=None, - description=( - 'Storage configuration ID of the prod python-js data app this draft iterates against. ' - "Set automatically on draft creation. Used by get_data_apps to list a prod app's drafts." - ), - ) - - auto_suspend_after_seconds: int = Field( - validation_alias=AliasChoices('autoSuspendAfterSeconds', 'auto_suspend_after_seconds'), - serialization_alias='autoSuspendAfterSeconds', - description='The number of seconds after which the running data app is automatically suspended.', - ) - data_app: 'CodeDataAppConfig.Parameters.DataApp' = Field( - validation_alias=AliasChoices('dataApp', 'data_app'), - serialization_alias='dataApp', - description='The data app sub config.', - ) - - class Runtime(BaseModel): - class Image(BaseModel): - version: str = Field(description='The runtime image version tag.') - - class Workspace(BaseModel): - enabled: bool = Field( - description=( - 'When true, the platform auto-provisions a workspace per data app and injects ' - 'its WORKSPACE_ID into the runtime env.' - ), - ) - - image: 'CodeDataAppConfig.Runtime.Image | None' = Field( - default=None, - description=( - 'Optional pin of the runtime image version. Omit to let the data-science platform ' - 'apply its default for python-js apps.' - ), - ) - workspace: 'CodeDataAppConfig.Runtime.Workspace | None' = Field( - default=None, - description=( - 'Optional workspace runtime config. Provide `{enabled: true}` to opt into ' - 'platform-managed per-app workspaces.' - ), - ) - - parameters: 'CodeDataAppConfig.Parameters' = Field(description='The parameters of the data app.') - runtime: 'CodeDataAppConfig.Runtime | None' = Field( - default=None, - description=( - 'Optional runtime configuration block (image pin, per-app workspace, etc.). Omit ' - 'entirely when no runtime overrides are needed — the platform picks defaults.' - ), - ) - authorization: DataAppConfig.Authorization | None = Field( - default=None, - description=( - 'Optional authorization block. Same shape as for Streamlit data apps. Omit (None) to let the ' - 'DSAPI apply its default behavior for python-js apps.' - ), - ) - storage: dict[str, Any] | None = Field( - default=None, - description=( - 'Optional Storage input/output mappings (validated against the storage JSON schema). ' - 'Omit when the app does not need Storage I/O.' - ), - ) - - -class CreatedGitCredentialResponse(BaseModel): - """Response model for credential creation on a managed-git-repo data app. - - Matches the `CreatedCredential` schema from sandboxes-service. For `http_token` - credentials the response includes a one-time `secret` that cannot be retrieved later. - """ - - model_config = ConfigDict(populate_by_name=True) - - id: str = Field(description='The ID of the created credential.') - type: str = Field(description='The credential type, e.g. "http_token" or "ssh_key".') - name: str = Field(default='', description='Caller-supplied display label (may be empty).') - permissions: str = Field(description='The permissions of the credential, e.g. "readWrite" or "readOnly".') - owner_admin_id: str | None = Field( - validation_alias=AliasChoices('ownerAdminId', 'owner_admin_id'), - default=None, - description='The admin ID that owns the credential.', - ) - created_at: str | None = Field( - validation_alias=AliasChoices('createdAt', 'created_at'), - default=None, - description='The timestamp when the credential was created.', - ) - secret: str | None = Field( - default=None, - description=( - 'One-time secret returned only at creation for `http_token` credentials. ' - 'It cannot be retrieved by subsequent reads.' - ), - ) - - -class AppGitRepoResponse(BaseModel): - """Response model for the managed git repo info of a data app.""" - - model_config = ConfigDict(populate_by_name=True) - - ssh_url: str | None = Field( - validation_alias=AliasChoices('sshUrl', 'ssh_url'), - default=None, - description='SSH clone URL. `null` for externally configured HTTP(S) repositories.', - ) - https_url: str | None = Field( - validation_alias=AliasChoices('httpsUrl', 'https_url'), - default=None, - description=( - 'HTTPS clone URL (without embedded credentials). `null` for externally configured SSH repositories.' - ), - ) - is_managed_git_repo: bool = Field( - validation_alias=AliasChoices('isManagedGitRepo', 'is_managed_git_repo'), - default=False, - description='Whether the repository is a managed git repository provisioned by the service.', - ) - - -class AppRunFailureReason(BaseModel): - """Machine-readable failure info attached by the platform to an unsuccessful AppRun.""" - - model_config = ConfigDict(populate_by_name=True) - - reason: str | None = Field( - default=None, - description=( - 'Machine-readable code identifying the kind of failure, ' - 'e.g. "ConfigDecryptionFailed" or "StartupProbeFailed".' - ), - ) - message: str | None = Field(default=None, description='Human-readable explanation of the failure.') - - -class AppRunResponse(BaseModel): - """Response model for a single AppRun — one deployment attempt of a data app.""" - - model_config = ConfigDict(populate_by_name=True) - - id: str = Field(description='The ID of the app run.') - app_id: str | None = Field( - validation_alias=AliasChoices('appId', 'app_id'), - default=None, - description='The ID of the data app this run belongs to.', - ) - state: str = Field(description='The state of the run: "starting", "running", "finished" or "failed".') - created_at: str | None = Field( - validation_alias=AliasChoices('createdAt', 'created_at'), - default=None, - description='The timestamp when the run was created.', - ) - started_at: str | None = Field( - validation_alias=AliasChoices('startedAt', 'started_at'), - default=None, - description='The timestamp when the app became ready, or `null` if it never started.', - ) - stopped_at: str | None = Field( - validation_alias=AliasChoices('stoppedAt', 'stopped_at'), - default=None, - description='The timestamp when the run stopped, or `null` while it is still active.', - ) - startup_logs: str | None = Field( - validation_alias=AliasChoices('startupLogs', 'startup_logs'), - default=None, - description='Output of the startup phase (entrypoint log), when available.', - ) - failure_reason: AppRunFailureReason | None = Field( - validation_alias=AliasChoices('failureReason', 'failure_reason'), - default=None, - description=( - 'Why the run was not successful. Populated by the platform for failed runs, including ' - 'setup-phase failures (e.g. invalid secrets) that produce no container logs.' - ), - ) - mode: str | None = Field(default=None, description='The mode of the run, e.g. "prod" or "dev".') - - -class DataScienceClient(KeboolaServiceClient): - - def __init__(self, raw_client: RawKeboolaClient, branch_id: str | None = None) -> None: - """ - Creates a DataScienceClient from a RawKeboolaClient and a branch id. - - :param raw_client: The raw client to use - :param branch_id: The id of the branch - """ - super().__init__(raw_client=raw_client) - self._branch_id = branch_id - - @classmethod - def create( - cls, - root_url: str, - token: str | None, - branch_id: str | None = None, - headers: dict[str, Any] | None = None, - readonly: bool | None = None, - ) -> 'DataScienceClient': - """ - Creates a DataScienceClient from a Keboola Storage API token. - - :param root_url: The root URL of the service API - :param token: The Keboola Storage API token. If None, the client will not send any authorization header. - :param branch_id: The id of the Keboola project branch to work on - :param headers: Additional headers for the requests - :param readonly: If True, the client will only use HTTP GET, HEAD operations. - :return: A new instance of DataScienceClient - """ - return cls( - raw_client=RawKeboolaClient( - base_api_url=root_url, - api_token=token, - headers=headers, - readonly=readonly, - ), - branch_id=branch_id, - ) - - async def get_data_app(self, data_app_id: str) -> DataAppResponse: - """ - Get a data app by its ID. - - :param data_app_id: The ID of the data app - :return: The data app - """ - response = await self.get(endpoint=f'apps/{data_app_id}') - return DataAppResponse.model_validate(response) - - async def deploy_data_app( - self, - data_app_id: str, - config_version: str | None = None, - *, - mode: str | None = None, - restart_if_running: bool = True, - update_dependencies: bool = False, - ) -> DataAppResponse: - """ - Deploy a data app by its ID. - - :param data_app_id: The ID of the data app - :param config_version: The version of the config to deploy. Required for Streamlit apps; omit for python-js - apps backed by a managed git repo (they have no Storage configVersion). - :param mode: Deployment mode. Set to 'dev' to deploy a python-js draft as a dev version - (hot reload + auto-auth for iframe preview). Leave None for Streamlit apps and - for prod deploys. - :param restart_if_running: Whether to restart the data app if it is already running - :param update_dependencies: If set to `true`, latest package versions are installed during app startup, - instead of using frozen versions. - :return: The data app - """ - data: dict[str, Any] = { - 'desiredState': 'running', - 'restartIfRunning': restart_if_running, - 'updateDependencies': update_dependencies, - } - if config_version is not None: - data['configVersion'] = config_version - if mode is not None: - data['mode'] = mode - response = await self.patch(endpoint=f'apps/{data_app_id}', data=data) - return DataAppResponse.model_validate(response) - - async def suspend_data_app(self, data_app_id: str) -> DataAppResponse: - """ - Suspend a data app by setting its desired state to 'stopped'. - :param data_app_id: Data app ID to suspend - :return: Updated data app response with the new state - """ - data = {'desiredState': 'stopped'} - response = await self.patch(endpoint=f'apps/{data_app_id}', data=data) - return DataAppResponse.model_validate(response) - - async def get_data_app_password(self, data_app_id: str) -> str: - """ - Get the password for a data app by its ID. - """ - response = await self.get(endpoint=f'apps/{data_app_id}/password') - assert isinstance(response, dict) - return cast(str, response['password']) - - async def create_data_app( - self, - name: str, - description: str, - configuration: Union['DataAppConfig', 'CodeDataAppConfig'], - *, - app_type: str = 'streamlit', - use_managed_git_repo: bool = False, - ) -> DataAppResponse: - """ - Create a data app from a simplified config used in the MCP server. - - :param name: The name of the data app - :param description: The description of the data app - :param configuration: The simplified configuration of the data app - :param app_type: The data app type, e.g. 'streamlit' or 'python-js'. Defaults to 'streamlit'. - :param use_managed_git_repo: When True, the data-science API provisions a managed git repo for the app. - Only meaningful for python-js apps. Pass False on drafts that bring their own - external-git binding via `configuration.parameters.dataApp.git`. - :return: The data app - """ - data: dict[str, Any] = { - 'branchId': self._branch_id, - 'name': name, - 'type': app_type, - 'description': description, - 'config': configuration.model_dump(exclude_none=True, by_alias=True), - } - if use_managed_git_repo: - data['useManagedGitRepo'] = True - response = await self.post(endpoint='apps', data=data) - return DataAppResponse.model_validate(response) - - async def create_app_git_credential( - self, - data_app_id: str, - *, - permissions: str = 'readWrite', - ) -> CreatedGitCredentialResponse: - """ - Create an HTTP-token credential on a managed-git-repo data app so the caller can clone, - pull, and push to the app's repo over HTTPS. The response includes a one-time `secret` - that is not returned by any subsequent read. - - :param data_app_id: The ID of the data app - :param permissions: 'readWrite' (default) or 'readOnly'. - :return: The created credential, including the one-time `secret`. - """ - data = {'type': 'http_token', 'permissions': permissions} - response = await self.post(endpoint=f'apps/{data_app_id}/git-repo/credentials', data=data) - return CreatedGitCredentialResponse.model_validate(response) - - async def get_app_git_repo(self, data_app_id: str) -> AppGitRepoResponse: - """ - Get the managed git repo info (clone URL) for a data app. - - Only meaningful for python-js apps created with `use_managed_git_repo=True`. - - :param data_app_id: The ID of the data app - :return: The git repo info, including the clone URL. - """ - response = await self.get(endpoint=f'apps/{data_app_id}/git-repo') - return AppGitRepoResponse.model_validate(response) - - async def list_app_runs(self, data_app_id: str, *, limit: int = 5, offset: int = 0) -> list[AppRunResponse]: - """ - List runs (deployment attempts) of a data app, newest first. - - :param data_app_id: The ID of the data app - :param limit: Maximum number of runs to return - :param offset: Number of runs to skip - :return: The app runs, including `failure_reason` for unsuccessful ones. - """ - response = await self.get(endpoint=f'apps/{data_app_id}/runs', params={'limit': limit, 'offset': offset}) - assert isinstance(response, list) - return [AppRunResponse.model_validate(run) for run in response] - - async def delete_data_app(self, data_app_id: str) -> None: - """ - Delete a data app by its ID. - - The DSAPI delete endpoint removes the data app only if its desired and current states match. - - If they do not match, it returns a 400 Bad Request. - - Desired state is the state where the app is supposed to be after the action is completed. While current - state reflects the actual state of the app. E.g. If we deploy the app, the desired state is 'running' and the - current state is 'started' until the app is deployed. - - When successful, DSAPI deletes both the app configuration from storage and the data app itself. - If the configuration was already deleted, DSAPI does not delete the data app and returns 500 error. - :param data_app_id: ID of the data app to delete - """ - await self.delete(endpoint=f'apps/{data_app_id}') - - async def list_data_apps(self, limit: int = 100, offset: int = 0) -> list[DataAppResponse]: - """ - List all data apps. - """ - response = await self.get(endpoint='apps', params={'limit': limit, 'offset': offset}) - return [DataAppResponse.model_validate(app) for app in response] - - async def tail_app_logs( - self, - app_id: str, - *, - since: datetime | None, - lines: int | None, - ) -> str: - """ - Tail application logs. Either `since` or `lines` must be provided but not both at the same time. - In case when none of the parameters are provided, it uses the `lines` parameter with - the last 100 lines. - :param app_id: ID of the app. - :param since: ISO-8601 timestamp with nanoseconds as a datetime object - Providing microseconds is enough, nanoseconds are not supported via datetime - E.g: since = datetime.now(timezone.utc) - timedelta(days=1) - :param lines: Number of log lines from the end. Defaults to 100. - :return: Logs as plain text. - :raise ValueError: If both "since" and "lines" are provided. - :raise ValueError: If neither "since" nor "lines" are provided. - :raise httpx.HTTPStatusError: For non-200 status codes. - """ - if since and lines: - raise ValueError('You cannot use both "since" and "lines" query parameters together.') - elif since is None and lines is None: - raise ValueError('Either "since" or "lines" must be provided.') - - if lines is not None: - lines = max(lines, 1) # Ensure lines is at least 1 - params = {'lines': lines} - elif since is not None: - iso_since = since.isoformat(timespec='microseconds') - params = {'since': iso_since} - else: - raise ValueError('Either "since" or "lines" must be provided.') - - response = await self.get_text(endpoint=f'apps/{app_id}/logs/tail', params=params) - return cast(str, response) diff --git a/src/keboola_mcp_server/clients/encryption.py b/src/keboola_mcp_server/clients/encryption.py deleted file mode 100644 index 1237a5171..000000000 --- a/src/keboola_mcp_server/clients/encryption.py +++ /dev/null @@ -1,119 +0,0 @@ -from typing import Any, Iterator, cast - -from keboola_mcp_server.clients.base import JsonDict, KeboolaServiceClient, RawKeboolaClient - -EncValue = str | JsonDict - -# Keys starting with this prefix hold secret values that must never be stored in plaintext. -SECRET_KEY_PREFIX = '#' -# Values already encrypted by the encryption service start with this prefix. -ENCRYPTED_VALUE_PREFIX = 'KBC::' -# Placeholder that the MCP server returns instead of plaintext secret values on config reads. -REDACTED_SECRET_VALUE = '[REDACTED]' - - -def is_encrypted_value(value: Any) -> bool: - """Checks if the value is a cipher created by the encryption service.""" - return isinstance(value, str) and value.startswith(ENCRYPTED_VALUE_PREFIX) - - -def iter_secret_items(value: Any) -> Iterator[tuple[str, Any]]: - """ - Recursively walks dicts and lists and yields (key, value) pairs for every key - starting with the '#' secret prefix. - """ - if isinstance(value, dict): - for key, item in value.items(): - if isinstance(key, str) and key.startswith(SECRET_KEY_PREFIX): - yield key, item - else: - yield from iter_secret_items(item) - elif isinstance(value, list): - for item in value: - yield from iter_secret_items(item) - - -def contains_plaintext_secrets(value: Any) -> bool: - """Checks if the value contains any '#'-prefixed key whose value is not yet encrypted.""" - return any(not is_encrypted_value(item) for _, item in iter_secret_items(value)) - - -def redact_secrets(value: Any, *, mask: str = REDACTED_SECRET_VALUE) -> Any: - """ - Returns a deep copy of the value with plaintext '#'-prefixed secret values replaced by the mask. - Values already encrypted by the encryption service ('KBC::' ciphers) are kept as they are opaque. - """ - if isinstance(value, dict): - return { - key: ( - mask - if isinstance(key, str) and key.startswith(SECRET_KEY_PREFIX) and not is_encrypted_value(item) - else redact_secrets(item, mask=mask) - ) - for key, item in value.items() - } - elif isinstance(value, list): - return [redact_secrets(item, mask=mask) for item in value] - else: - return value - - -class EncryptionClient(KeboolaServiceClient): - - @classmethod - def create( - cls, - root_url: str, - token: str | None = None, - headers: dict[str, Any] | None = None, - ) -> 'EncryptionClient': - """ - Creates an EncryptionClient from a Keboola Storage API token. - - :param root_url: The root URL of the service API. - :param token: The Keboola Storage API token. If None, the client will not send any authorization header. - :param headers: Additional headers for the requests. - :return: A new instance of EncryptionClient - """ - return cls(raw_client=RawKeboolaClient(base_api_url=root_url, api_token=token, headers=headers)) - - async def encrypt( - self, - value: EncValue, - *, - project_id: str | None = None, - component_id: str | None = None, - config_id: str | None = None, - ) -> EncValue: - """ - Encrypt a value using the encryption service, returns encrypted value. Parameters are optional and the ciphers - created by the service are dependent on those parameters when decrypting. Decryption is done automatically - when using encrypted values in a request to Storage API (for components) - See: https://developers.keboola.com/overview/encryption/ - If value is a dict, values whose keys start with '#' are encrypted. - If value is a str, it is encrypted. - If value contains already encrypted values, they are returned as is. - - :param value: The value to encrypt - :param project_id: The project ID - :param component_id: The component ID (optional) - :param config_id: The config ID (optional) - :return: The encrypted value, same type as input - """ - if component_id and project_id is None: - raise ValueError('project_id is required if component_id is provided') - if config_id and component_id is None: - raise ValueError('component_id is required if config_id is provided') - - params = { - 'componentId': component_id, - 'projectId': project_id, - 'configId': config_id, - } - params = {k: v for k, v in params.items() if v is not None} - response = await self.raw_client.post( - endpoint='encrypt', - params=params, - data=cast(dict[str, Any], value), - ) - return cast(EncValue, response) diff --git a/src/keboola_mcp_server/clients/jobs_queue.py b/src/keboola_mcp_server/clients/jobs_queue.py deleted file mode 100644 index 71435f956..000000000 --- a/src/keboola_mcp_server/clients/jobs_queue.py +++ /dev/null @@ -1,155 +0,0 @@ -from collections.abc import Sequence -from typing import Any, Optional, cast - -from keboola_mcp_server.clients.base import JsonDict, JsonList, KeboolaServiceClient, RawKeboolaClient - - -class JobsQueueClient(KeboolaServiceClient): - """ - Async client for Keboola Job Queue API. - """ - - def __init__(self, raw_client: RawKeboolaClient, branch_id: str | None = None) -> None: - """ - Creates a JobsQueueClient from a RawKeboolaClient and a branch id. - - :param raw_client: The raw client to use - :param branch_id: The id of the branch - """ - super().__init__(raw_client=raw_client) - self._branch_id = branch_id - - @classmethod - def create( - cls, - root_url: str, - token: str, - branch_id: str | None = None, - headers: dict[str, Any] | None = None, - readonly: bool | None = None, - ) -> 'JobsQueueClient': - """ - Creates a JobsQueue client. - - :param root_url: Root url of API. e.g. "https://queue.keboola.com/". - :param token: The Keboola Storage API token - :param branch_id: The id of the Keboola project branch to work on - :param headers: Additional headers for the requests. - :param readonly: If True, the client will only use HTTP GET, HEAD operations. - :return: A new instance of JobsQueueClient. - """ - return cls( - raw_client=RawKeboolaClient(base_api_url=root_url, api_token=token, headers=headers, readonly=readonly), - branch_id=branch_id, - ) - - async def get_job_detail(self, job_id: str) -> JsonDict: - """ - Retrieves information about a given job. - - :param job_id: The id of the job. - :return: Job details as dictionary. - """ - - return cast(JsonDict, await self.get(endpoint=f'jobs/{job_id}')) - - async def search_jobs_by( - self, - component_id: Optional[str] = None, - config_id: Optional[str] = None, - status: Optional[list[str]] = None, - limit: int = 100, - offset: int = 0, - sort_by: Optional[str] = 'startTime', - sort_order: Optional[str] = 'desc', - ) -> JsonList: - """ - Searches for jobs based on the provided parameters. - - :param component_id: The id of the component. - :param config_id: The id of the configuration. - :param status: The status of the jobs to filter by. - :param limit: The number of jobs to return. - :param offset: The offset of the jobs to return. - :param sort_by: The field to sort the jobs by. - :param sort_order: The order to sort the jobs by. - :return: Dictionary containing matching jobs. - """ - params = { - 'branchId': self._branch_id, - 'componentId': component_id, - 'configId': config_id, - 'status': status, - 'limit': limit, - 'offset': offset, - 'sortBy': sort_by, - 'sortOrder': sort_order, - } - params = {k: v for k, v in params.items() if v is not None} - return await self._search(params=params) - - async def create_job( - self, - component_id: str, - configuration_id: str, - configuration_row_ids: Sequence[str] | None = None, - ) -> JsonDict: - """ - Creates a new job. - - :param component_id: The id of the component. - :param configuration_id: The id of the configuration. - :param configuration_row_ids: Optional list of configuration row IDs to run. - :return: The response from the API call - created job or raise an error. - """ - payload = { - 'component': component_id, - 'config': configuration_id, - 'mode': 'run', - } - if self._branch_id: - payload['branchId'] = self._branch_id - if configuration_row_ids: - payload['configRowIds'] = list(configuration_row_ids) - return cast(JsonDict, await self.post(endpoint='jobs', data=payload)) - - async def _search(self, params: dict[str, Any]) -> JsonList: - """ - Searches for jobs based on the provided parameters. - - :param params: The parameters to search for. - :return: Dictionary containing matching jobs. - - Parameters (copied from the API docs): - - id str/list[str]: Search jobs by id - - runId str/list[str]: Search jobs by runId - - branchId str/list[str]: Search jobs by branchId - - tokenId str/list[str]: Search jobs by tokenId - - tokenDescription str/list[str]: Search jobs by tokenDescription - - componentId str/list[str]: Search jobs by componentId - - component str/list[str]: Search jobs by componentId, alias for componentId - - configId str/list[str]: Search jobs by configId - - config str/list[str]: Search jobs by configId, alias for configId - - configRowIds str/list[str]: Search jobs by configRowIds - - status str/list[str]: Search jobs by status - - createdTimeFrom str: The jobs that were created after the given date - e.g. "2021-01-01, -8 hours, -1 week,..." - - createdTimeTo str: The jobs that were created before the given date - e.g. "2021-01-01, today, last monday,..." - - startTimeFrom str: The jobs that were started after the given date - e.g. "2021-01-01, -8 hours, -1 week,..." - - startTimeTo str: The jobs that were started before the given date - e.g. "2021-01-01, today, last monday,..." - - endTimeTo str: The jobs that were finished before the given date - e.g. "2021-01-01, today, last monday,..." - - endTimeFrom str: The jobs that were finished after the given date - e.g. "2021-01-01, -8 hours, -1 week,..." - - limit int: The number of jobs returned, default 100 - - offset int: The jobs page offset, default 0 - - sortBy str: The jobs sorting field, default "id" - values: id, runId, projectId, branchId, componentId, configId, tokenDescription, status, createdTime, - updatedTime, startTime, endTime, durationSeconds - - sortOrder str: The jobs sorting order, default "desc" - values: asc, desc - """ - return cast(JsonList, await self.get(endpoint='search/jobs', params=params)) diff --git a/src/keboola_mcp_server/clients/metastore.py b/src/keboola_mcp_server/clients/metastore.py deleted file mode 100644 index 1b0c9ab69..000000000 --- a/src/keboola_mcp_server/clients/metastore.py +++ /dev/null @@ -1,246 +0,0 @@ -"""Keboola Metastore API client.""" - -from typing import Any - -from pydantic import AliasChoices, BaseModel, Field, TypeAdapter - -from keboola_mcp_server.clients.base import JsonDict, JsonStruct, KeboolaServiceClient, RawKeboolaClient - - -class MetaObjectMeta(BaseModel): - """Metadata from the JSON:API 'meta' field — same structure for all object types.""" - - branch: str | None = Field(default=None) - name: str | None = Field(default=None) - revision: int | None = Field(default=None) - schema_version: str | None = Field( - validation_alias=AliasChoices('schemaVersion', 'schema_version'), - serialization_alias='schemaVersion', - default=None, - ) - project_id: int | None = Field( - validation_alias=AliasChoices('projectId', 'project_id'), - serialization_alias='projectId', - default=None, - ) - organization_id: str | None = Field( - validation_alias=AliasChoices('organizationId', 'organization_id'), - serialization_alias='organizationId', - default=None, - ) - created_at: str | None = Field( - validation_alias=AliasChoices('createdAt', 'created_at'), - serialization_alias='createdAt', - default=None, - ) - last_updated: str | None = Field( - validation_alias=AliasChoices('lastUpdated', 'last_updated'), - serialization_alias='lastUpdated', - default=None, - ) - deleted_at: str | None = Field( - validation_alias=AliasChoices('deletedAt', 'deleted_at'), - serialization_alias='deletedAt', - default=None, - ) - revision_created_at: str | None = Field( - validation_alias=AliasChoices('revisionCreatedAt', 'revision_created_at'), - serialization_alias='revisionCreatedAt', - default=None, - ) - - -class MetastoreObject(BaseModel): - """Single object from the Metastore JSON:API response.""" - - type: str | None = Field(default=None) - id: str | None = Field(default=None) - attributes: dict[str, Any] | None = Field(default=None) - relationships: dict[str, Any] | None = Field(default=None) - meta: MetaObjectMeta | None = Field(default=None) - - -LIST_ADAPTER: TypeAdapter[list[MetastoreObject]] = TypeAdapter(list[MetastoreObject]) - - -class MetastoreClient(KeboolaServiceClient): - """Client for interacting with the Metastore API.""" - - def __init__(self, raw_client: RawKeboolaClient, branch_id: str | None = None) -> None: - super().__init__(raw_client=raw_client) - self._branch_id: str | None = branch_id - - @classmethod - def create( - cls, - root_url: str, - token: str | None, - *, - branch_id: str | None = None, - headers: dict[str, Any] | None = None, - readonly: bool | None = None, - ) -> 'MetastoreClient': - client = cls( - raw_client=RawKeboolaClient( - base_api_url=root_url, - api_token=token, - headers=headers, - readonly=readonly, - ), - branch_id=branch_id, - ) - return client - - async def get_schema(self, object_type: str, version: str | None = None) -> JsonDict: - endpoint = f'api/v1/schema/{object_type}/{version}' if version else f'api/v1/schema/{object_type}' - response = await self.get(endpoint=endpoint) - if not isinstance(response, dict): - raise ValueError('Unexpected metastore schema response format.') - return response - - async def list_objects( - self, - object_type: str, - *, - filter_by: str | None = None, - limit: int | None = None, - offset: int | None = None, - organization_scope: bool = False, - ) -> list[MetastoreObject]: - endpoint = ( - f'api/v1/repository/{object_type}/organization' - if organization_scope - else f'api/v1/repository/{object_type}' - ) - params: dict[str, Any] = {} - if filter_by is not None: - params['filter'] = filter_by - if limit is not None: - params['limit'] = limit - if offset is not None: - params['offset'] = offset - - response = await self.get( - endpoint=endpoint, - params=params or None, - ) - return self._parse_list(response) - - async def get_object( - self, - object_type: str, - uuid: str, - ) -> MetastoreObject: - response = await self.get( - endpoint=f'api/v1/repository/{object_type}/{uuid}', - ) - return self._parse_object(response) - - async def create_object( - self, - object_type: str, - *, - name: str | None = None, - data: dict[str, Any], - schema_version: str | None = None, - scope: str | None = None, - ) -> MetastoreObject: - payload: dict[str, Any] = {'data': data} - if name is not None: - payload['name'] = name - if schema_version is not None: - payload['schemaVersion'] = schema_version - if scope is not None: - payload['scope'] = scope - if self._branch_id is not None: - payload['branch'] = self._branch_id - - response = await self.post(endpoint=f'api/v1/repository/{object_type}', data=payload) - return self._parse_object(response) - - async def patch_object( - self, - object_type: str, - uuid: str, - *, - name: str | None = None, - data: dict[str, Any] | None = None, - ) -> MetastoreObject: - payload: dict[str, Any] = {} - if name is not None: - payload['name'] = name - if data is not None: - payload['data'] = data - - response = await self.patch(endpoint=f'api/v1/repository/{object_type}/{uuid}', data=payload) - return self._parse_object(response) - - async def put_object( - self, - object_type: str, - uuid: str, - *, - name: str, - data: dict[str, Any], - ) -> MetastoreObject: - response = await self.put( - endpoint=f'api/v1/repository/{object_type}/{uuid}', - data={'name': name, 'data': data}, - ) - return self._parse_object(response) - - async def delete_object(self, object_type: str, uuid: str) -> JsonStruct | None: - return await self.delete(endpoint=f'api/v1/repository/{object_type}/{uuid}') - - async def list_revisions( - self, - object_type: str, - *, - filter_by: str | None = None, - limit: int | None = None, - offset: int | None = None, - ) -> list[MetastoreObject]: - params: dict[str, Any] = {} - if filter_by is not None: - params['filter'] = filter_by - if limit is not None: - params['limit'] = limit - if offset is not None: - params['offset'] = offset - response = await self.get( - endpoint=f'api/v1/repository/{object_type}/revisions', - params=params or None, - ) - return self._parse_list(response) - - async def get_revision( - self, - object_type: str, - uuid: str, - revision: int, - ) -> MetastoreObject: - response = await self.get( - endpoint=f'api/v1/repository/{object_type}/{uuid}/revisions/{revision}', - ) - return self._parse_object(response) - - async def delete_revision(self, object_type: str, uuid: str, revision: int) -> JsonStruct | None: - return await self.delete(endpoint=f'api/v1/repository/{object_type}/{uuid}/revisions/{revision}') - - @staticmethod - def _parse_list(response: JsonStruct) -> list[MetastoreObject]: - if not isinstance(response, dict): - raise ValueError('Unexpected metastore response format: expected JSON object with "data" key.') - data = response.get('data') - if not isinstance(data, list): - raise ValueError('Unexpected metastore response format: "data" is not an array.') - return LIST_ADAPTER.validate_python(data) - - @staticmethod - def _parse_object(response: JsonStruct) -> MetastoreObject: - if not isinstance(response, dict): - raise ValueError('Unexpected metastore response format: expected JSON object.') - data = response.get('data', response) - if not isinstance(data, dict): - raise ValueError('Unexpected metastore response format: "data" is not an object.') - return MetastoreObject.model_validate(data) diff --git a/src/keboola_mcp_server/clients/query.py b/src/keboola_mcp_server/clients/query.py deleted file mode 100644 index 80beef1c0..000000000 --- a/src/keboola_mcp_server/clients/query.py +++ /dev/null @@ -1,129 +0,0 @@ -from typing import Any, cast - -from keboola_mcp_server.clients import KeboolaServiceClient, RawKeboolaClient -from keboola_mcp_server.clients.base import JsonDict - - -class QueryServiceClient(KeboolaServiceClient): - - def __init__(self, raw_client: RawKeboolaClient, branch_id: str) -> None: - """ - Creates a QueryServiceClient from a RawKeboolaClient and a branch id. - - :param raw_client: The raw client to use - :param branch_id: The id of the Keboola project branch to work on - """ - super().__init__(raw_client=raw_client) - self._branch_id: str = branch_id - if not self._branch_id: - raise ValueError('Branch id is required') - if self._branch_id in ['default', 'main']: - raise ValueError(f'The real branch id is required, got: "{self._branch_id}"') - - @property - def branch_id(self) -> str: - """Returns the real branch ID (no symbolic names such as 'default' or 'main').""" - return self._branch_id - - @classmethod - def create( - cls, - *, - root_url: str, - version: str = 'v1', - branch_id: str, - token: str | None, - headers: JsonDict | None = None, - ) -> 'QueryServiceClient': - """ - Creates a QueryServiceClient from a Keboola Storage API token. - - :param root_url: The root URL of the service API. - :param version: The version of the API to use (default: 'v1'). - :param branch_id: The id of the Keboola project branch to work on. - :param token: The Keboola Storage API token, If None, the client will not send any authorization header. - :param headers: Additional headers for the requests. - :return: A new instance of QueryServiceClient. - """ - return cls( - raw_client=RawKeboolaClient( - base_api_url=f'{root_url}/api/{version}', - api_token=token, - headers=headers, - ), - branch_id=branch_id, - ) - - async def submit_job( - self, statements: list[str], workspace_id: str, actor_type: str | None = None, transactional: bool | None = None - ) -> str: - """ - Creates a new query job with SQL statements in the specified branch and workspace. - - :param statements: The SQL statements to be executed. - :param workspace_id: The id of the Keboola project workspace to work on. - :param actor_type: The type of actor to use -- 'user' or 'system'. - :param transactional: Whether the job should be executed in a transaction. - :return: The unique identifier of the submitted job. - """ - payload: JsonDict = {'statements': statements} - if actor_type: - payload['actorType'] = actor_type - if transactional is not None: - payload['transactional'] = transactional - resp = cast( - JsonDict, - await self.post(endpoint=f'branches/{self._branch_id}/workspaces/{workspace_id}/queries', data=payload), - ) - return resp['queryJobId'] - - async def get_job_status(self, job_id: str) -> JsonDict: - """ - Gets the status of a job by its job ID. - - :param job_id: The unique identifier for the job whose status is being retrieved. - :return: A dictionary containing the status details of the specified job and its SQL statements. - """ - return cast(JsonDict, await self.get(endpoint=f'queries/{job_id}')) - - async def cancel_job(self, job_id: str, reason: str) -> JsonDict: - """ - Cancels a running query job. - - :param job_id: The unique identifier for the query job to cancel. - :param reason: The reason for cancellation (for audit trail). - :return: The response from the API call. - """ - payload: JsonDict = {'reason': reason} - return cast(JsonDict, await self.post(endpoint=f'queries/{job_id}/cancel', data=payload)) - - def build_cancel_url(self, job_id: str) -> str: - """ - Returns the absolute URL clients should POST to in order to cancel the given query job. - - Used to surface the cancellation handle to MCP clients via a progress notification so - that they can cancel the query directly against Query Service without routing through - the originating MCP server replica. The endpoint accepts the same auth header the client - already uses to talk to the MCP server (`X-StorageAPI-Token` or `Authorization: Bearer`). - """ - return f'{self.raw_client.base_api_url}/queries/{job_id}/cancel' - - async def get_job_results( - self, job_id: str, statement_id: str, *, offset: int | None = None, limit: int | None = None - ) -> JsonDict: - """ - Gets the results of a specific statement within a query job and returns data, rows affected count, - and status information with pagination support. - - :param job_id: A unique identifier for the query job. - :param statement_id: A unique identifier for the specific query statement within the job. - :param offset: The offset of the first row to return. - :param limit: The maximum number of rows to return. - :return: The query statement results. - """ - params: dict[str, Any] = {} - if offset is not None: - params['offset'] = offset - if limit is not None: - params['pageSize'] = limit - return cast(JsonDict, await self.get(endpoint=f'queries/{job_id}/{statement_id}/results', params=params)) diff --git a/src/keboola_mcp_server/clients/scheduler.py b/src/keboola_mcp_server/clients/scheduler.py deleted file mode 100644 index 854798e6b..000000000 --- a/src/keboola_mcp_server/clients/scheduler.py +++ /dev/null @@ -1,181 +0,0 @@ -""" -Keboola Scheduler API client. - -This client handles communication with the Scheduler API (scheduler.keboola.com) -for managing scheduled flow executions. -""" - -import logging -from datetime import datetime -from typing import Any - -from pydantic import AliasChoices, BaseModel, Field - -from keboola_mcp_server.clients.base import KeboolaServiceClient, RawKeboolaClient - -LOG = logging.getLogger(__name__) - - -class Schedule(BaseModel): - - cron_tab: str = Field( - validation_alias=AliasChoices('cronTab', 'cron_tab', 'cron-tab'), - serialization_alias='cronTab', - description='Cron expression for scheduling', - ) - timezone: str = Field(description='Timezone for the schedule') - state: str = Field(description='Schedule state (enabled/disabled)') - - -class TargetConfiguration(BaseModel): - - component_id: str = Field( - validation_alias=AliasChoices('componentId', 'component_id', 'component-id'), - serialization_alias='componentId', - description='Component ID to execute', - ) - configuration_id: str = Field( - validation_alias=AliasChoices('configurationId', 'configuration_id', 'configuration-id'), - serialization_alias='configurationId', - description='Configuration ID to execute', - ) - mode: str = Field(description='Execution mode (run)') - tag: str | None = Field(default=None, description='Optional tag version') - - -class TargetExecution(BaseModel): - """Target execution model having information about the execution of the target component configuration.""" - - job_id: str | None = Field( - default=None, - validation_alias=AliasChoices('jobId', 'job_id', 'job-id'), - serialization_alias='jobId', - description='Job ID of the execution', - ) - execution_time: datetime | None = Field( - default=None, - validation_alias=AliasChoices('executionTime', 'execution_time', 'execution-time'), - serialization_alias='executionTime', - description='Execution time', - ) - - -class ScheduleApiResponse(BaseModel): - """Schedule API response model.""" - - id: str = Field(description='Schedule ID (numeric string)') - token_id: str = Field( - validation_alias=AliasChoices('tokenId', 'token_id', 'token-id'), - serialization_alias='tokenId', - description='Token ID used for authentication', - ) - configuration_id: str = Field( - validation_alias=AliasChoices('configurationId', 'configuration_id', 'configuration-id'), - serialization_alias='configurationId', - description='Configuration ID from Storage API', - ) - configuration_version_id: str = Field( - validation_alias=AliasChoices('configurationVersionId', 'configuration_version_id', 'configuration-version-id'), - serialization_alias='configurationVersionId', - description='Configuration version ID', - ) - schedule: Schedule = Field(description='Schedule configuration') - target: TargetConfiguration = Field(description='Target configuration') - executions: list[TargetExecution] = Field(default_factory=list, description='List of recent executions') - - -class SchedulerClient(KeboolaServiceClient): - """Client for interacting with the Keboola Scheduler API.""" - - def __init__(self, raw_client: RawKeboolaClient) -> None: - """ - Creates a SchedulerClient from a RawKeboolaClient. - - :param raw_client: The raw client to use - :param branch_id: The id of the branch - """ - super().__init__(raw_client=raw_client) - - @classmethod - def create( - cls, - root_url: str, - token: str | None, - headers: dict[str, Any] | None = None, - readonly: bool | None = None, - ) -> 'SchedulerClient': - """ - Creates a SchedulerClient from a Keboola Storage API token. - - :param root_url: The root URL of the Scheduler API - :param token: The Keboola Storage API token. If None, the client will not send any authorization header. - :param headers: Additional headers for the requests - :param readonly: If True, the client will only use HTTP GET, HEAD operations. - :return: A new instance of SchedulerClient - """ - return cls( - raw_client=RawKeboolaClient( - base_api_url=root_url, - api_token=token, - headers=headers, - readonly=readonly, - ) - ) - - async def activate_schedule(self, schedule_config_id: str) -> ScheduleApiResponse: - """ - Activate a schedule in the Scheduler API by its Storage API configuration ID. - - This is the second step in schedule creation, after the schedule configuration - has been created in Storage API. - - :param schedule_config_id: The schedule configuration ID in Storage API - :return: The schedule response with id, schedule, target, etc. - """ - payload = {'configurationId': schedule_config_id} - response = await self.post(endpoint='schedules', data=payload) - return ScheduleApiResponse.model_validate(response) - - async def get_schedule(self, schedule_id: str) -> ScheduleApiResponse: - """ - Get schedule details by schedule ID from Scheduler API. - - :param schedule_id: The schedule ID (numeric string) - :return: The schedule details - """ - response = await self.get(endpoint=f'schedules/{schedule_id}') - return ScheduleApiResponse.model_validate(response) - - async def list_schedules_by_config_id(self, component_id: str, configuration_id: str) -> list[ScheduleApiResponse]: - """ - Get schedules details by Storage API component and configuration ID. - - :param component_id: The Storage API component ID - :param configuration_id: The Storage API configuration ID - :return: The list of schedules details - """ - params = { - 'componentId': component_id, - 'configurationId': configuration_id, - } - response = await self.get(endpoint='schedules', params=params) - return [ScheduleApiResponse.model_validate(schedule) for schedule in response] - - async def list_schedules(self) -> list[ScheduleApiResponse]: - """ - List all schedules for the current project/token. - - :return: The list of schedules details - """ - response = await self.get(endpoint='schedules') - if isinstance(response, list): - return [ScheduleApiResponse.model_validate(schedule) for schedule in response] - return [ScheduleApiResponse.model_validate(response)] - - async def delete_schedule(self, schedule_config_id: str) -> None: - """ - Delete a schedule by its Storage API configuration ID. - - :param schedule_config_id: The schedule configuration ID in Storage API - """ - await self.delete(endpoint=f'configurations/{schedule_config_id}') diff --git a/src/keboola_mcp_server/clients/storage.py b/src/keboola_mcp_server/clients/storage.py deleted file mode 100644 index 72fd2d444..000000000 --- a/src/keboola_mcp_server/clients/storage.py +++ /dev/null @@ -1,1122 +0,0 @@ -import logging -import math -from datetime import datetime -from typing import Any, Iterable, Literal, Mapping, Optional, Sequence, cast - -from pydantic import AliasChoices, BaseModel, Field, field_validator - -from keboola_mcp_server.clients.base import JsonDict, KeboolaServiceClient, RawKeboolaClient -from keboola_mcp_server.clients.encryption import ( - REDACTED_SECRET_VALUE, - EncryptionClient, - is_encrypted_value, - iter_secret_items, -) - -LOG = logging.getLogger(__name__) - - -ComponentResource = Literal['configuration', 'rows', 'state'] -StorageEventType = Literal['info', 'success', 'warn', 'error'] - -# Project features that can be checked with the is_enabled method -ProjectFeature = Literal['global-search', 'storage-branches'] - -ItemType = Literal[ - 'flow', - 'bucket', - 'table', - 'transformation', - 'configuration', - 'configuration-row', - 'workspace', - 'shared-code', - 'rows', - 'state', -] - -ComponentType = Literal['application', 'extractor', 'transformation', 'writer'] - - -class GlobalSearchResponse(BaseModel): - """The SAPI global search response.""" - - class Item(BaseModel): - id: str = Field(description='The id of the item.') - name: str = Field(description='The name of the item.') - type: ItemType = Field(description='The type of the item.') - full_path: dict[str, Any] = Field( - description=( - 'The full path of the item containing project, branch and other information depending on the ' - 'type of the item.' - ), - alias='fullPath', - ) - component_id: Optional[str] = Field( - default=None, description='The id of the component the item belongs to.', alias='componentId' - ) - organization_id: int = Field( - description='The id of the organization the item belongs to.', alias='organizationId' - ) - project_id: int = Field(description='The id of the project the item belongs to.', alias='projectId') - project_name: str = Field(description='The name of the project the item belongs to.', alias='projectName') - created: datetime = Field(description='The date and time the item was created in ISO format.') - - @property - def branch_id(self) -> str | None: - """The id of the branch the item belongs to, extracted from the full path.""" - branch = self.full_path.get('branch') - if isinstance(branch, dict) and branch.get('id') is not None: - return str(branch['id']) - return None - - @property - def branch_name(self) -> str | None: - """The name of the branch the item belongs to, extracted from the full path.""" - branch = self.full_path.get('branch') - if isinstance(branch, dict) and branch.get('name'): - return str(branch['name']) - return None - - all: int = Field(description='Total number of found results.') - items: list[Item] = Field(description='List of search results of the GlobalSearchType.') - by_type: dict[str, int] = Field( - description='Mapping of found types to the number of corresponding results.', alias='byType' - ) - by_project: dict[str, str] = Field(description='Mapping of project id to project name.', alias='byProject') - - @field_validator('by_type', 'by_project', mode='before') - @classmethod - def validate_dict_fields(cls, current_value: Any) -> Any: - # If the value is empty-list/None, return an empty dictionary, otherwise return the value - if not current_value: - return dict() - return current_value - - -class APIFlowResponse(BaseModel): - """ - Raw API response for configuration endpoints. - - Note: will be removed soon due to removal of flow specific client methods. - """ - - # Core identification fields - configuration_id: str = Field( - description='The ID of the flow configuration', - validation_alias=AliasChoices('id', 'configuration_id', 'configurationId', 'configuration-id'), - serialization_alias='id', - ) - name: str = Field(description='The name of the flow configuration') - description: Optional[str] = Field(default=None, description='The description of the flow configuration') - - # Versioning and state - version: int = Field(description='The version of the flow configuration') - is_disabled: bool = Field( - default=False, - description='Whether the flow configuration is disabled', - validation_alias=AliasChoices('isDisabled', 'is_disabled', 'is-disabled'), - serialization_alias='isDisabled', - ) - is_deleted: bool = Field( - default=False, - description='Whether the flow configuration is deleted', - validation_alias=AliasChoices('isDeleted', 'is_deleted', 'is-deleted'), - serialization_alias='isDeleted', - ) - - # Flow-specific configuration data (as returned by API) - configuration: dict[str, Any] = Field( - description='The nested flow configuration object containing phases and tasks' - ) - - # Change tracking - change_description: Optional[str] = Field( - default=None, - description='The description of the latest changes', - validation_alias=AliasChoices('changeDescription', 'change_description', 'change-description'), - serialization_alias='changeDescription', - ) - - # Metadata - metadata: list[dict[str, Any]] = Field( - default_factory=list, - description='Flow configuration metadata', - validation_alias=AliasChoices('metadata', 'configuration_metadata', 'configurationMetadata'), - ) - - # Timestamps - created: Optional[str] = Field(None, description='Creation timestamp') - updated: Optional[str] = Field(None, description='Last update timestamp') - - -class ComponentAPIResponse(BaseModel): - """ - Raw component response that can handle both Storage API and AI Service API responses. - - Storage API (/v2/storage/components/{id}) returns just the core fields. - AI Service API (/docs/components/{id}) returns core fields + optional documentation metadata. - - The optional fields will be None when parsing Storage API responses. - """ - - # Core fields present in both APIs (SAPI and AI service) - component_id: str = Field( - description='The ID of the component', - validation_alias=AliasChoices('component_id', 'id', 'componentId', 'component-id'), - ) - component_name: str = Field( - description='The name of the component', - validation_alias=AliasChoices( - 'name', - 'component_name', - 'componentName', - 'component-name', - ), - ) - type: str = Field( - description='Component type (extractor, writer, application)', - validation_alias=AliasChoices('type', 'component_type', 'componentType', 'component-type'), - ) - flags: list[str] = Field( - default_factory=list, - description='Developer portal flags', - validation_alias=AliasChoices('flags', 'component_flags', 'componentFlags', 'component-flags'), - ) - categories: list[str] = Field( - default_factory=list, - description='Component categories', - validation_alias=AliasChoices( - 'categories', - 'component_categories', - 'componentCategories', - 'component-categories', - ), - ) - - # Optional metadata fields only present in AI Service API responses - documentation_url: str | None = Field( - default=None, - description='Documentation URL', - validation_alias=AliasChoices('documentationUrl', 'documentation_url', 'documentation-url'), - ) - documentation: str | None = Field( - default=None, - description='Component documentation', - validation_alias=AliasChoices('documentation'), - ) - configuration_schema: dict[str, Any] | None = Field( - default=None, - description='Configuration schema', - validation_alias=AliasChoices('configurationSchema', 'configuration_schema', 'configuration-schema'), - ) - configuration_row_schema: dict[str, Any] | None = Field( - default=None, - description='Configuration row schema', - validation_alias=AliasChoices('configurationRowSchema', 'configuration_row_schema', 'configuration-row-schema'), - ) - data: dict[str, Any] | None = Field( - default=None, - description='Additional component metadata', - validation_alias=AliasChoices('data'), - ) - - -class ConfigurationAPIResponse(BaseModel): - """ - Raw API response for configuration endpoints. - - Mirrors the actual JSON structure returned by Keboola Storage API for: - - configuration_detail() - - configuration_list() - - configuration_create() - - configuration_update() - """ - - component_id: str = Field( - description='The ID of the component', - validation_alias=AliasChoices('component_id', 'componentId', 'component-id'), - ) - configuration_id: str = Field( - description='The ID of the configuration', - validation_alias=AliasChoices('configuration_id', 'id', 'configurationId', 'configuration-id'), - ) - name: str = Field(description='The name of the configuration') - description: Optional[str] = Field(default=None, description='The description of the configuration') - version: int = Field(description='The version of the configuration') - is_disabled: bool = Field( - default=False, - description='Whether the configuration is disabled', - validation_alias=AliasChoices('isDisabled', 'is_disabled', 'is-disabled'), - ) - is_deleted: bool = Field( - default=False, - description='Whether the configuration is deleted', - validation_alias=AliasChoices('isDeleted', 'is_deleted', 'is-deleted'), - ) - configuration: dict[str, Any] = Field( - description='The nested configuration object containing parameters and storage' - ) - rows: Optional[list[dict[str, Any]]] = Field( - default=None, description='The row configurations within this configuration' - ) - change_description: Optional[str] = Field( - default=None, - description='The description of the latest changes', - validation_alias=AliasChoices('changeDescription', 'change_description', 'change-description'), - ) - metadata: list[dict[str, Any]] = Field( - default_factory=list, - description='Configuration metadata', - validation_alias=AliasChoices('metadata', 'configuration_metadata', 'configurationMetadata'), - ) - - -class CreateConfigurationAPIResponse(BaseModel): - id: str = Field(description='Unique identifier of the newly created configuration.') - name: str = Field(description='Human-readable name of the configuration.') - description: Optional[str] = Field(default='', description='Optional description of the configuration.') - created: datetime = Field(description='Timestamp when the configuration was created (ISO 8601).') - creator_token: dict[str, Any] = Field( - description='Metadata about the token that created the configuration.', alias='creatorToken' - ) - version: int = Field(description='Version number of the configuration.') - change_description: Optional[str] = Field( - description='Optional description of the change that introduced this configuration version.', - alias='changeDescription', - ) - is_disabled: bool = Field( - description='Indicates whether the configuration is currently disabled.', alias='isDisabled' - ) - is_deleted: bool = Field( - description='Indicates whether the configuration has been marked as deleted.', alias='isDeleted' - ) - configuration: Optional[dict[str, Any]] = Field( - description='User-defined configuration payload (key-value structure).' - ) - state: Optional[dict[str, Any]] = Field( - description='Internal runtime state data associated with the configuration.' - ) - current_version: Optional[dict[str, Any]] = Field( - description='Metadata about the currently deployed version of the configuration.', alias='currentVersion' - ) - - -class AsyncStorageClient(KeboolaServiceClient): - - def __init__( - self, - raw_client: RawKeboolaClient, - branch_id: str | None = None, - encryption_client: EncryptionClient | None = None, - ) -> None: - """ - Creates an AsyncStorageClient from a RawKeboolaClient and a branch id. - - :param raw_client: The raw client to use - :param branch_id: The id of the Keboola project branch to work on - :param encryption_client: The encryption service client used to encrypt '#'-prefixed secret values - before they are written to the Storage API. If None, writing a configuration that contains - plaintext secrets raises an error (fail-closed). - """ - super().__init__(raw_client=raw_client) - self._branch_id: str = branch_id or 'default' - self._encryption_client = encryption_client - - @classmethod - def create( - cls, - *, - root_url: str, - token: Optional[str], - version: str = 'v2', - branch_id: str | None = None, - headers: dict[str, Any] | None = None, - readonly: bool | None = None, - encryption_client: EncryptionClient | None = None, - ) -> 'AsyncStorageClient': - """ - Creates an AsyncStorageClient from a Keboola Storage API token. - - :param root_url: The root URL of the service API - :param token: The Keboola Storage API token, If None, the client will not send any authorization header. - :param version: The version of the API to use (default: 'v2') - :param branch_id: The id of the Keboola project branch to work on - :param headers: Additional headers for the requests - :param readonly: If True, the client will only use HTTP GET, HEAD operations. - :param encryption_client: The encryption service client used to encrypt '#'-prefixed secret values - before they are written to the Storage API. - :return: A new instance of AsyncStorageClient - """ - return cls( - raw_client=RawKeboolaClient( - base_api_url=f'{root_url}/{version}/storage', - api_token=token, - headers=headers, - readonly=readonly, - ), - branch_id=branch_id, - encryption_client=encryption_client, - ) - - async def _encrypt_secrets(self, component_id: str, configuration: dict[str, Any]) -> dict[str, Any]: - """ - Encrypts plaintext '#'-prefixed secret values in the configuration using the Encryption API - before the configuration is written to the Storage API. The Storage API does not encrypt - '#'-values server-side, so without this step the secrets would be stored in plaintext. - - Fail-closed: if the configuration contains plaintext secrets and they cannot be encrypted, - this raises an error rather than letting the plaintext be stored. - - :param component_id: The id of the component the configuration belongs to. - :param configuration: The configuration definition as a dictionary. - :return: The configuration with all '#'-prefixed values encrypted. - """ - plaintext_keys = [key for key, value in iter_secret_items(configuration) if not is_encrypted_value(value)] - if not plaintext_keys: - return configuration - - redacted_keys = [key for key, value in iter_secret_items(configuration) if value == REDACTED_SECRET_VALUE] - if redacted_keys: - raise ValueError( - f'The configuration contains redacted secret values for keys: {sorted(set(redacted_keys))}. ' - f'These are placeholders returned on configuration reads, not the actual secret values. ' - f'Either leave the existing secret values untouched or ask the user to provide new ones.' - ) - - if self._encryption_client is None: - raise ValueError( - f'The configuration contains plaintext secret values for keys: {sorted(set(plaintext_keys))}, ' - f'but no encryption client is available. Refusing to store secrets in plaintext.' - ) - - project_id = await self.project_id() - encrypted = await self._encryption_client.encrypt( - configuration, component_id=component_id, project_id=project_id - ) - return cast(dict[str, Any], encrypted) - - async def branches_list(self) -> list[JsonDict]: - """ - Gets the list of branches in a project. - """ - return cast(list[JsonDict], await self.get(endpoint='dev-branches')) - - async def dev_branch_detail(self, branch_id: str | int) -> JsonDict: - """ - Gets details for a development branch. - """ - return cast(JsonDict, await self.get(endpoint=f'dev-branches/{branch_id}')) - - async def branch_metadata_get(self) -> list[JsonDict]: - """ - Retrieves metadata for the current branch. - - :return: Branch metadata as a list of dictionaries. Each dictionary contains the 'key' and 'value' keys. - """ - return cast(list[JsonDict], await self.get(endpoint=f'branch/{self._branch_id}/metadata')) - - async def branch_metadata_update(self, metadata: dict[str, Any]) -> list[JsonDict]: - """ - Updates metadata for the current branch. - - :param metadata: The metadata to update. - :return: The SAPI call response - updated metadata or raise an error. - """ - payload = { - 'metadata': [{'key': key, 'value': value} for key, value in metadata.items()], - } - return cast(list[JsonDict], await self.post(endpoint=f'branch/{self._branch_id}/metadata', data=payload)) - - async def bucket_detail(self, bucket_id: str, branch_id: str | None = None) -> JsonDict: - """ - Retrieves information about a given bucket. - - :param bucket_id: The id of the bucket - :param branch_id: Optional branch ID override (uses client's branch_id if not specified) - :return: Bucket details as dictionary - """ - bid = branch_id or self._branch_id - return cast(JsonDict, await self.get(endpoint=f'branch/{bid}/buckets/{bucket_id}')) - - async def bucket_list(self, include: list[str] | None = None, branch_id: str | None = None) -> list[JsonDict]: - """ - Lists all buckets. - - :param include: List of fields to include in the response ('metadata' or 'linkedBuckets') - :param branch_id: Optional branch ID override (uses client's branch_id if not specified) - :return: List of buckets as dictionary - """ - bid = branch_id or self._branch_id - params = {} - if include is not None and isinstance(include, list): - params['include'] = ','.join(include) - return cast(list[JsonDict], await self.get(endpoint=f'branch/{bid}/buckets', params=params)) - - async def bucket_metadata_delete(self, bucket_id: str, metadata_id: str) -> None: - """ - Deletes metadata for a given bucket. - - :param bucket_id: The id of the bucket - :param metadata_id: The id of the metadata - """ - await self.delete(endpoint=f'buckets/{bucket_id}/metadata/{metadata_id}') - - async def bucket_metadata_get(self, bucket_id: str) -> list[JsonDict]: - """ - Retrieves metadata for a given bucket. - - :param bucket_id: The id of the bucket - :return: Bucket metadata as a list of dictionaries. Each dictionary contains the 'key' and 'value' keys. - """ - return cast(list[JsonDict], await self.get(endpoint=f'buckets/{bucket_id}/metadata')) - - async def bucket_metadata_update( - self, - bucket_id: str, - metadata: dict[str, Any], - provider: str = 'user', - ) -> list[JsonDict]: - """ - Updates metadata for a given bucket. - - :param bucket_id: The id of the bucket - :param metadata: The metadata to update. - :param provider: The provider of the metadata ('user' by default). - :return: Bucket metadata as a list of dictionaries. Each dictionary contains the 'key' and 'value' keys. - """ - payload = { - 'provider': provider, - 'metadata': [{'key': key, 'value': value} for key, value in metadata.items()], - } - return cast(list[JsonDict], await self.post(endpoint=f'buckets/{bucket_id}/metadata', data=payload)) - - async def bucket_table_list( - self, bucket_id: str, include: list[str] | None = None, branch_id: str | None = None - ) -> list[JsonDict]: - """ - Lists all tables in a given bucket. - - :param bucket_id: The id of the bucket - :param include: List of fields to include in the response - :param branch_id: Optional branch ID override (uses client's branch_id if not specified) - :return: List of tables as dictionary - """ - bid = branch_id or self._branch_id - params = {} - if include is not None and isinstance(include, list): - params['include'] = ','.join(include) - return cast(list[JsonDict], await self.get(endpoint=f'branch/{bid}/buckets/{bucket_id}/tables', params=params)) - - async def column_metadata_delete(self, column_id: str, metadata_id: str) -> None: - """ - Deletes metadata for a given column. - - :param column_id: The id of the column - :param metadata_id: The id of the metadata - """ - await self.delete(endpoint=f'columns/{column_id}/metadata/{metadata_id}') - - async def column_metadata_get(self, column_id: str) -> list[JsonDict]: - """ - Retrieves metadata for a given column. - - :param column_id: The id of the column - :return: Column metadata as a list of dictionaries. Each dictionary contains the 'key' and 'value' keys. - """ - return cast(list[JsonDict], await self.get(endpoint=f'columns/{column_id}/metadata')) - - async def component_detail(self, component_id: str) -> JsonDict: - """ - Retrieves information about a given component. - - :param component_id: The id of the component - :return: Component details as a dictionary - """ - return cast(JsonDict, await self.get(endpoint=f'branch/{self._branch_id}/components/{component_id}')) - - async def component_list( - self, component_type: str | None = None, include: list[ComponentResource] | None = None - ) -> list[JsonDict]: - """ - Lists all components of a given type. - - :param component_type: The type of the component (extractor, writer, application, etc.) - :param include: Comma separated list of resources to include. - Available resources: configuration, rows and state. - :return: List of components as dictionary - """ - endpoint = f'branch/{self._branch_id}/components' - params: dict[str, Any] = {} - if component_type: - params['componentType'] = component_type - if include is not None and isinstance(include, list): - params['include'] = ','.join(include) - - return cast(list[JsonDict], await self.get(endpoint=endpoint, params=params)) - - async def configuration_create( - self, - component_id: str, - name: str, - description: str, - configuration: dict[str, Any], - ) -> JsonDict: - """ - Creates a new configuration for a component. - - :param component_id: The id of the component for which to create the configuration. - :param name: The name of the configuration. - :param description: The description of the configuration. - :param configuration: The configuration definition as a dictionary. - - :return: The SAPI call response - created configuration or raise an error. - """ - endpoint = f'branch/{self._branch_id}/components/{component_id}/configs' - - payload = { - 'name': name, - 'description': description, - 'configuration': await self._encrypt_secrets(component_id, configuration), - } - return cast(JsonDict, await self.post(endpoint=endpoint, data=payload)) - - async def configuration_delete(self, component_id: str, configuration_id: str, skip_trash: bool = False) -> None: - """ - Deletes a configuration. - - :param component_id: The id of the component. - :param configuration_id: The id of the configuration. - :param skip_trash: If True, the configuration is deleted without moving to the trash. - (Technically it means the API endpoint is called twice.) - :raises httpx.HTTPStatusError: If the (component_id, configuration_id) is not found. - """ - endpoint = f'branch/{self._branch_id}/components/{component_id}/configs/{configuration_id}' - await self.delete(endpoint=endpoint) - if skip_trash: - await self.delete(endpoint=endpoint) - - async def configuration_detail(self, component_id: str, configuration_id: str) -> JsonDict: - """ - Retrieves information about a given configuration. - - :param component_id: The id of the component. - :param configuration_id: The id of the configuration. - :return: The parsed json from the HTTP response. - :raises ValueError: If the component_id or configuration_id is invalid. - """ - if not isinstance(component_id, str) or component_id == '': - raise ValueError(f"Invalid component_id '{component_id}'.") - if not isinstance(configuration_id, str) or configuration_id == '': - raise ValueError(f"Invalid configuration_id '{configuration_id}'.") - endpoint = f'branch/{self._branch_id}/components/{component_id}/configs/{configuration_id}' - - return cast(JsonDict, await self.get(endpoint=endpoint)) - - async def configuration_list(self, component_id: str) -> list[JsonDict]: - """ - Lists configurations of the given component. - - :param component_id: The id of the component. - :return: List of configurations. - :raises ValueError: If the component_id is invalid. - """ - if not isinstance(component_id, str) or component_id == '': - raise ValueError(f"Invalid component_id '{component_id}'.") - endpoint = f'branch/{self._branch_id}/components/{component_id}/configs' - - return cast(list[JsonDict], await self.get(endpoint=endpoint)) - - async def component_configurations_search( - self, - component_id: str | None = None, - metadata_keys: list[str] | None = None, - ) -> list[JsonDict]: - """ - Searches component configurations by component and metadata keys. - All filters are applied server-side by the SAPI search endpoint. - - :param component_id: Optional component ID to filter results. - :param metadata_keys: List of metadata keys to filter by — returns only configurations - that have at least one of the specified metadata keys set. - :return: List of matching configurations as dictionaries. - """ - if not (component_id or metadata_keys): - return [] - endpoint = f'branch/{self._branch_id}/search/component-configurations' - params: dict[str, Any] = {} - if component_id: - params['componentId'] = component_id - for i, key in enumerate(metadata_keys or []): - params[f'metadataKeys[{i}]'] = key - return cast(list[JsonDict], await self.get(endpoint=endpoint, params=params)) - - async def configuration_metadata_get(self, component_id: str, configuration_id: str) -> list[JsonDict]: - """ - Retrieves metadata for a given configuration. - - :param component_id: The id of the component. - :param configuration_id: The id of the configuration. - :return: Configuration metadata as a list of dictionaries. Each entry contains at minimum 'id', 'key', - 'value', and 'provider' fields, plus timestamp fields ('timestamp', 'created', etc.). - """ - endpoint = f'branch/{self._branch_id}/components/{component_id}/configs/{configuration_id}/metadata' - return cast(list[JsonDict], await self.get(endpoint=endpoint)) - - async def configuration_metadata_update( - self, - component_id: str, - configuration_id: str, - metadata: dict[str, Any], - ) -> list[JsonDict]: - """ - Updates metadata for the given configuration. - - :param component_id: The id of the component. - :param configuration_id: The id of the configuration. - :param metadata: The metadata to update. - :return: Updated configuration metadata as a list of dictionaries. Each entry contains at minimum 'id', - 'key', 'value', and 'provider' fields, plus timestamp fields ('timestamp', 'created', etc.). - """ - endpoint = f'branch/{self._branch_id}/components/{component_id}/configs/{configuration_id}/metadata' - payload = { - 'metadata': [{'key': key, 'value': value} for key, value in metadata.items()], - } - return cast(list[JsonDict], await self.post(endpoint=endpoint, data=payload)) - - async def configuration_metadata_delete(self, component_id: str, configuration_id: str, metadata_id: str) -> None: - """Deletes a single metadata entry for a configuration.""" - endpoint = ( - f'branch/{self._branch_id}/components/{component_id}' f'/configs/{configuration_id}/metadata/{metadata_id}' - ) - await self.delete(endpoint=endpoint) - - async def configuration_update( - self, - component_id: str, - configuration_id: str, - configuration: dict[str, Any], - change_description: str, - updated_name: Optional[str] = None, - updated_description: Optional[str] = None, - is_disabled: bool | None = None, - ) -> JsonDict: - """ - Updates a component configuration. - - :param component_id: The id of the component. - :param configuration_id: The id of the configuration. - :param configuration: The updated configuration dictionary. - :param change_description: The description of the modification to the configuration. - :param updated_name: The updated name of the configuration, if None, the original - name is preserved. - :param updated_description: The entire description of the updated configuration, if None, the original - description is preserved. - :param is_disabled: Whether the configuration should be disabled. - :return: The SAPI call response - updated configuration or raise an error. - """ - endpoint = f'branch/{self._branch_id}/components/{component_id}/configs/{configuration_id}' - - payload: dict[str, Any] = { - 'configuration': await self._encrypt_secrets(component_id, configuration), - 'changeDescription': change_description, - } - if updated_name: - payload['name'] = updated_name - - if updated_description: - payload['description'] = updated_description - - if is_disabled is not None: - payload['isDisabled'] = is_disabled - - return cast(JsonDict, await self.put(endpoint=endpoint, data=payload)) - - async def configuration_row_create( - self, - component_id: str, - config_id: str, - name: str, - description: str, - configuration: dict[str, Any], - ) -> JsonDict: - """ - Creates a new row configuration for a component configuration. - - :param component_id: The ID of the component. - :param config_id: The ID of the configuration. - :param name: The name of the row configuration. - :param description: The description of the row configuration. - :param configuration: The configuration data to create row configuration. - :return: The SAPI call response - created row configuration or raise an error. - """ - payload = { - 'name': name, - 'description': description, - 'configuration': await self._encrypt_secrets(component_id, configuration), - } - - return cast( - JsonDict, - await self.post( - endpoint=f'branch/{self._branch_id}/components/{component_id}/configs/{config_id}/rows', - data=payload, - ), - ) - - async def configuration_row_update( - self, - component_id: str, - config_id: str, - configuration_row_id: str, - configuration: dict[str, Any], - change_description: str, - updated_name: Optional[str] = None, - updated_description: Optional[str] = None, - is_disabled: bool | None = None, - ) -> JsonDict: - """ - Updates a row configuration for a component configuration. - - :param configuration: The configuration data to update row configuration. - :param component_id: The ID of the component. - :param config_id: The ID of the configuration. - :param configuration_row_id: The ID of the row. - :param change_description: The description of the changes made. - :param updated_name: The updated name of the configuration, if None, the original - name is preserved. - :param updated_description: The updated description of the configuration, if None, the original - description is preserved. - :param is_disabled: Whether the configuration row should be disabled. - :return: The SAPI call response - updated row configuration or raise an error. - """ - - payload: dict[str, Any] = { - 'configuration': await self._encrypt_secrets(component_id, configuration), - 'changeDescription': change_description, - } - if updated_name: - payload['name'] = updated_name - - if updated_description: - payload['description'] = updated_description - - if is_disabled is not None: - payload['isDisabled'] = is_disabled - - return cast( - JsonDict, - await self.put( - endpoint=f'branch/{self._branch_id}/components/{component_id}/configs/{config_id}' - f'/rows/{configuration_row_id}', - data=payload, - ), - ) - - async def configuration_row_detail(self, component_id: str, config_id: str, configuration_row_id: str) -> JsonDict: - """ - Retrieves details of a specific configuration row. - - :param component_id: The id of the component. - :param config_id: The id of the configuration. - :param configuration_row_id: The id of the configuration row. - :return: Configuration row details. - """ - endpoint = f'branch/{self._branch_id}/components/{component_id}/configs/{config_id}/rows/{configuration_row_id}' - return cast(JsonDict, await self.get(endpoint=endpoint)) - - async def configuration_versions(self, component_id: str, config_id: str) -> list[JsonDict]: - """ - Retrieves details of a specific configuration version. - """ - endpoint = f'branch/{self._branch_id}/components/{component_id}/configs/{config_id}/versions' - return cast(list[JsonDict], await self.get(endpoint=endpoint)) - - async def configuration_version_latest(self, component_id: str, config_id: str) -> int: - """ - Retrieves details of the last configuration version. - """ - versions = await self.configuration_versions(component_id, config_id) - latest_version = 0 - for data in versions: - assert isinstance(data, dict) - assert isinstance(data['version'], int) - if latest_version is None or data['version'] > latest_version: - latest_version = data['version'] - return latest_version - - async def job_detail(self, job_id: str | int) -> JsonDict: - """ - NOTE: To get info for regular jobs, use the Job Queue API. - Retrieves information about a given job. - - :param job_id: The id of the job - :return: Job details as dictionary - """ - return cast(JsonDict, await self.get(endpoint=f'jobs/{job_id}')) # TODO: no branch support - - async def global_search( - self, - query: str, - limit: int = 100, - offset: int = 0, - types: Sequence[ItemType] = tuple(), - branch_scope: Literal['current', 'all'] = 'current', - ) -> GlobalSearchResponse: - """ - Searches for items in the storage by name. The search is conducted only through entity names to ensure - confidentiality. The request is always scoped to the current project via `projectIds[]`. - - :param query: The query to search for. - :param limit: The maximum number of items to return. - :param offset: The offset to start from, pagination parameter. - :param types: The types of items to search for. - :param branch_scope: 'current' restricts the search to the branch this client operates on - (production branches on the default branch, the specific dev branch otherwise); - 'all' searches the whole project across all branches. - """ - params: dict[str, Any] = { - 'query': query, - 'projectIds[]': [await self.project_id()], - 'types[]': types, - 'limit': limit, - 'offset': offset, - } - if branch_scope == 'current': - if self._branch_id == 'default': - params['branchTypes[]'] = 'production' - else: - params['branchTypes[]'] = 'development' - params['branchIds[]'] = self._branch_id - params = {k: v for k, v in params.items() if v} - raw_resp = await self.get(endpoint='global-search', params=params) - return GlobalSearchResponse.model_validate(raw_resp) - - async def table_detail(self, table_id: str, branch_id: str | None = None) -> JsonDict: - """ - Retrieves information about a given table. - - :param table_id: The id of the table - :param branch_id: Optional branch ID override (uses client's branch_id if not specified) - :return: Table details as dictionary - """ - bid = branch_id or self._branch_id - return cast(JsonDict, await self.get(endpoint=f'branch/{bid}/tables/{table_id}')) - - async def table_metadata_delete(self, table_id: str, metadata_id: str) -> None: - """ - Deletes metadata for a given table. - - :param table_id: The id of the table - :param metadata_id: The id of the metadata - """ - await self.delete(endpoint=f'tables/{table_id}/metadata/{metadata_id}') - - async def table_metadata_get(self, table_id: str) -> list[JsonDict]: - """ - Retrieves metadata for a given table. - - :param table_id: The id of the table - :return: Table metadata as a list of dictionaries. Each dictionary contains the 'key' and 'value' keys. - """ - return cast(list[JsonDict], await self.get(endpoint=f'tables/{table_id}/metadata')) - - async def table_metadata_update( - self, - table_id: str, - metadata: dict[str, Any] | None = None, - columns_metadata: dict[str, list[dict[str, Any]]] | None = None, - provider: str = 'user', - ) -> JsonDict: - """ - Updates metadata for a given table. At least one of the `metadata` or `columns_metadata` arguments - must be provided. - - :param table_id: The id of the table - :param metadata: The metadata to update. - :param columns_metadata: The column metadata to update. Mapping of column names to a list of dictionaries. - Each dictionary contains the 'key' and 'value' keys. - :param provider: The provider of the metadata ('user' by default). - :return: Dictionary with 'metadata' key under which the table metadata is stored as a list of dictionaries. - Each dictionary contains the 'key' and 'value' keys. Under 'columnsMetadata' key, the column metadata - is stored as a mapping of column names to a list of dictionaries. - """ - if not metadata and not columns_metadata: - raise ValueError('At least one of the `metadata` or `columns_metadata` arguments must be provided.') - - payload: dict[str, Any] = {'provider': provider} - if metadata: - payload['metadata'] = [{'key': key, 'value': value} for key, value in metadata.items()] - if columns_metadata: - payload['columnsMetadata'] = columns_metadata - - return cast(JsonDict, await self.post(endpoint=f'tables/{table_id}/metadata', data=payload)) - - # TODO: no branch support - async def trigger_event( - self, - message: str, - component_id: str, - configuration_id: str | None = None, - event_type: StorageEventType | None = None, - params: Mapping[str, Any] | None = None, - results: Mapping[str, Any] | None = None, - duration: float | None = None, - run_id: str | None = None, - ) -> JsonDict: - """ - Sends a Storage API event. - - :param message: The event message. - :param component_id: The ID of the component triggering the event. - :param configuration_id: The ID of the component configuration triggering the event. - :param event_type: The type of event. - :param params: The component parameters. The structure of the params object must follow the JSON schema - registered for the component_id. - :param results: The component results. The structure of the results object must follow the JSON schema - registered for the component_id. - :param duration: The component processing duration in seconds. - :param run_id: The ID of the associated component job. - - :return: Dictionary with the new event ID. - """ - payload: dict[str, Any] = { - 'message': message, - 'component': component_id, - } - if configuration_id: - payload['configurationId'] = configuration_id - if event_type: - payload['type'] = event_type - if params: - payload['params'] = params - if results: - payload['results'] = results - if duration is not None: - # The events API ignores floats, so we round up to the nearest integer. - payload['duration'] = int(math.ceil(duration)) - if run_id: - payload['runId'] = run_id - - LOG.info(f'[trigger_event] payload={payload}') - - return cast(JsonDict, await self.post(endpoint='events', data=payload)) - - async def list_events( - self, - job_id: str, - limit: int | None = None, - offset: int | None = None, - ) -> list[JsonDict]: - """ - Lists Storage API events for a job. Used to retrieve job execution logs. - - Note: The Storage API ``runId`` query parameter matches the **job's id**, not the - job's hierarchical ``runId`` field (e.g. ``"parent.child"``). - - :param job_id: The job ID to fetch events for. - :param limit: Maximum number of events to return (default 50, API max 10000). - :param offset: Offset for pagination (default 0). - :return: List of event dictionaries. - """ - params: dict[str, Any] = { - 'runId': job_id, - 'limit': limit or 50, - 'offset': offset or 0, - 'forceUuid': 'true', - } - return cast(list[JsonDict], await self.get(endpoint='events', params=params)) - - async def workspace_create_for_config( - self, - component_id: str, - config_id: str, - login_type: str, - backend: str, - async_run: bool = True, - read_only_storage_access: bool = False, - ) -> JsonDict: - """Thin wrapper for POST /branch/{branch_id}/components/{component_id}/configs/{config_id}/workspaces.""" - data: dict[str, Any] = { - 'readOnlyStorageAccess': read_only_storage_access, - 'loginType': login_type, - 'backend': backend, - } - return cast( - JsonDict, - await self.post( - endpoint=f'branch/{self._branch_id}/components/{component_id}/configs/{config_id}/workspaces', - params={'async': async_run}, - data=data, - ), - ) - - async def workspace_detail(self, workspace_id: int) -> JsonDict: - """ - Retrieves information about a given workspace. - - :param workspace_id: The id of the workspace - :return: Workspace details as dictionary - """ - return cast(JsonDict, await self.get(endpoint=f'branch/{self._branch_id}/workspaces/{workspace_id}')) - - async def workspace_list(self) -> list[JsonDict]: - """ - Lists all workspaces in the project. - - :return: List of workspaces - """ - return cast(list[JsonDict], await self.get(endpoint=f'branch/{self._branch_id}/workspaces')) - - async def verify_token(self) -> JsonDict: - """ - Checks the token privileges and returns information about the project to which the token belongs. - - :return: Token and project information - """ - return cast(JsonDict, await self.get(endpoint='tokens/verify')) - - async def project_id(self) -> str: - """ - Retrieves the project id. - :return: Project id. - """ - raw_data = cast(JsonDict, await self.get(endpoint='tokens/verify')) - assert isinstance(raw_data['owner'], dict) - return str(raw_data['owner']['id']) - - async def is_enabled(self, features: ProjectFeature | Iterable[ProjectFeature]) -> bool: - """ - Checks if the features are enabled in the project - conjunction of features. - :param features: The features to check. - :return: True if the features are enabled, False otherwise. - """ - features = [features] if isinstance(features, str) else features - verified_info = await self.verify_token() - project_data = cast(JsonDict, verified_info['owner']) - project_features = cast(list[str], project_data.get('features', [])) - return all(feature in project_features for feature in features) - - async def token_create( - self, - description: str, - component_access: list[str] | None = None, - expires_in: int | None = None, - ) -> JsonDict: - """ - Creates a new Storage API token. - - :param description: Description of the token - :param component_access: List of component IDs the token should have access to - :param expires_in: Token expiration time in seconds - :return: Token creation response containing the token and its details - """ - token_data: dict[str, Any] = {'description': description} - - if component_access: - token_data['componentAccess'] = component_access - - if expires_in: - token_data['expiresIn'] = expires_in - - return cast(JsonDict, await self.post(endpoint='tokens', data=token_data)) diff --git a/src/keboola_mcp_server/clients/sync_actions.py b/src/keboola_mcp_server/clients/sync_actions.py deleted file mode 100644 index f5df4d08e..000000000 --- a/src/keboola_mcp_server/clients/sync_actions.py +++ /dev/null @@ -1,67 +0,0 @@ -from typing import Any - -from keboola_mcp_server.clients.base import JsonStruct, KeboolaServiceClient, RawKeboolaClient - - -class SyncActionsClient(KeboolaServiceClient): - """ - Async client for Keboola Sync Actions API. - """ - - def __init__(self, raw_client: RawKeboolaClient, branch_id: str | None = None) -> None: - """ - Creates a SyncActionsClient from a RawKeboolaClient and a branch id. - - :param raw_client: The raw client to use - :param branch_id: The id of the Keboola project branch to work on - """ - super().__init__(raw_client=raw_client) - self._branch_id: str = branch_id or 'default' - - @classmethod - def create( - cls, - *, - root_url: str, - token: str, - branch_id: str | None = None, - headers: dict[str, Any] | None = None, - readonly: bool | None = None, - ) -> 'SyncActionsClient': - """ - Creates a SyncActions client. - - :param root_url: Root url of API. e.g. "https://sync-actions.keboola.com/". - :param token: The Keboola Storage API token - :param branch_id: The id of the Keboola project branch to work on - :param headers: Additional headers for the requests. - :param readonly: If True, the client will only use HTTP GET, HEAD operations. - :return: A new instance of SyncActionsClient. - """ - return cls( - raw_client=RawKeboolaClient(base_api_url=root_url, api_token=token, headers=headers, readonly=readonly), - branch_id=branch_id, - ) - - async def execute_action( - self, - component_id: str, - action: str, - config_data: dict[str, Any], - ) -> JsonStruct: - """ - Executes a synchronous action for a component. - - :param component_id: The ID of the component. - :param action: The sync action to execute (e.g., "testConnection"). - :param config_data: The configuration data payload. - :return: The action result as a dict or list. - """ - payload: dict[str, Any] = { - 'configData': config_data, - 'componentId': component_id, - 'action': action, - } - if self._branch_id: - payload['branchId'] = self._branch_id - return await self.post(endpoint='actions', data=payload) diff --git a/src/keboola_mcp_server/config.py b/src/keboola_mcp_server/config.py deleted file mode 100644 index 8cb7e8e3e..000000000 --- a/src/keboola_mcp_server/config.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Configuration handling for the Keboola MCP server.""" - -import dataclasses -import importlib.metadata -import logging -import os -import uuid -from dataclasses import dataclass, field -from typing import Any, Literal, Mapping, Optional -from urllib.parse import urlparse, urlunparse - -LOG = logging.getLogger(__name__) -_NO_VALUE_MARKER = '__NO_VALUE_MARKER__' -Transport = Literal['stdio', 'streamable-http', 'http-compat/streamable-http'] - - -@dataclass(frozen=True) -class Config: - """Server configuration.""" - - storage_api_url: Optional[str] = None - """The URL to the Storage API.""" - storage_token: Optional[str] = field(default=None, metadata={'aliases': ['storage_api_token']}) - """The token to access the storage API using the MCP tools.""" - branch_id: Optional[str] = None - """The branch ID to access the storage API using the MCP tools.""" - workspace_schema: Optional[str] = None - """Workspace schema to access the buckets, tables and execute sql queries.""" - oauth_client_id: Optional[str] = None - """OAuth client ID registered in the Keboola OAuth Server.""" - oauth_client_secret: Optional[str] = None - """OAuth client secret registered in the Keboola OAuth Server.""" - oauth_server_url: Optional[str] = None - """The URL of the OAuth server to authenticate with.""" - oauth_scope: Optional[str] = None - """The OAuth scope to request from the OAuth server.""" - mcp_server_url: Optional[str] = None - """The URL where the MCP server si reachable.""" - jwt_secret: Optional[str] = None - """The secret key for encoding and decoding JWT tokens.""" - bearer_token: Optional[str] = None - """The access-token issued by Keboola OAuth server to be sent in 'Authorization: Bearer ' header.""" - conversation_id: Optional[str] = None - """The ID of the ongoing conversation with the MCP server. This is supplied only by the HTTP header.""" - - def __post_init__(self) -> None: - for f in dataclasses.fields(self): - if 'url' not in f.name: - continue - value = getattr(self, f.name) - if value: - orig_value = value - url_value = urlparse(value) - if url_value.netloc: - if (scheme := url_value.scheme) not in ['http', 'https']: - scheme = 'http' if url_value.netloc.startswith('localhost') else 'https' - value = urlunparse((scheme, url_value.netloc, '', '', '', '')) - elif url_value.path: - value = urlunparse(('https', url_value.path.split('/', maxsplit=1)[0], '', '', '', '')) - else: - raise ValueError(f'Invalid URL: {value}') - if value != orig_value: - LOG.warning(f'Amended "{f.name}" value from "{orig_value}" to "{value}".') - object.__setattr__(self, f.name, value) - - if self.branch_id is not None and self.branch_id.lower() in ['', 'none', 'null', 'default', 'production']: - object.__setattr__(self, 'branch_id', None) - - @staticmethod - def _normalize(name: str) -> str: - """Removes dashes and underscores from the input string and turns it into lowercase.""" - return name.lower().replace('_', '').replace('-', '') - - @classmethod - def _read_options(cls, d: Mapping[str, str]) -> Mapping[str, Any]: - data = {cls._normalize(k): v for k, v, in d.items()} - options: dict[str, Any] = {} - for f in dataclasses.fields(cls): - field_names = [f.name] + f.metadata.get('aliases', []) - - for name in field_names: - value: Optional[str] = _NO_VALUE_MARKER - - if (dict_name := cls._normalize(name)) in data: - value = data[dict_name] - - elif (dict_name := cls._normalize(f'KBC_{name}')) in data: - # environment variables start with KBC_ - value = data[dict_name] - - elif (dict_name := cls._normalize(f'X-{name}')) in data: - # HTTP headers start with X- - value = data[dict_name] - - if value is not _NO_VALUE_MARKER: - if f.type is Optional[bool]: - options[f.name] = value.lower() in ('true', 'yes', '1') - elif f.type is Optional[str]: - options[f.name] = value - else: - raise ValueError(f'Unsupported type {f.type} for field {f.name}') - break - - return options - - @classmethod - def from_dict(cls, d: Mapping[str, str]) -> 'Config': - """ - Creates new `Config` instance with values read from the input mapping. - The keys in the input mapping can either be the names of the fields in `Config` class - or their uppercase variant prefixed with 'KBC_'. - """ - return cls(**cls._read_options(d)) - - def replace_by(self, d: Mapping[str, str]) -> 'Config': - """ - Creates new `Config` instance from the existing one by replacing the values from the input mapping. - The keys in the input mapping can either be the names of the fields in `Config` class - or their uppercase variant prefixed with 'KBC_'. - """ - return dataclasses.replace(self, **self._read_options(d)) - - def __repr__(self) -> str: - params: list[str] = [] - for f in dataclasses.fields(self): - value = getattr(self, f.name) - if value: - if 'token' in f.name or 'password' in f.name or 'secret' in f.name: - params.append(f"{f.name}='****'") - else: - if isinstance(value, str): - params.append(f"{f.name}='{value}'") - else: - params.append(f'{f.name}={value}') - else: - params.append(f'{f.name}=None') - joined_params = ', '.join(params) - return f'Config({joined_params})' - - -@dataclass(frozen=True) -class ServerRuntimeInfo: - """Server runtime Information.""" - - transport: Transport - """Transport used by the MCP server (e.g., 'stdio', 'streamable-http').""" - server_id: str = field(default_factory=lambda: uuid.uuid4().hex) - """The ID of the MCP server.""" - app_env: str = field(default_factory=lambda: os.getenv('APP_ENV') or 'local') - """The environment of the MCP server application.""" - app_version: str = field(default_factory=lambda: os.getenv('APP_VERSION') or 'DEV') - """The version of the MCP server application.""" - server_version: str = importlib.metadata.version('keboola_mcp_server') - """The version of the Keboola MCP server library.""" - mcp_library_version: str = importlib.metadata.version('mcp') - """The version of the MCP library.""" - fastmcp_library_version: str = importlib.metadata.version('fastmcp') - """The version of the FastMCP library.""" - - -class MetadataField: - """ - Predefined names of Keboola metadata fields. - """ - - DESCRIPTION = 'KBC.description' - PROJECT_DESCRIPTION = 'KBC.projectDescription' - SHARED_DESCRIPTION = 'KBC.sharedDescription' # set when sharing a bucket via Data Catalog - - # set for configurations created by MCP tools; - # expected value: 'true' - CREATED_BY_MCP = 'KBC.MCP.createdBy' - - # set for configurations updated by MCP tools; - # the full key should end by a version number; - # expected value: 'true' - UPDATED_BY_MCP_PREFIX = 'KBC.MCP.updatedBy.version.' - - # Branch filtering works only for "fake development branches" - FAKE_DEVELOPMENT_BRANCH = 'KBC.createdBy.branch.id' - - # Component lineage metadata for created/updated configuration sources - CREATED_BY_COMPONENT_ID = 'KBC.createdBy.component.id' - CREATED_BY_CONFIGURATION_ID = 'KBC.createdBy.configuration.id' - CREATED_BY_CONFIGURATION_ROW_ID = 'KBC.createdBy.configurationRow.id' - UPDATED_BY_COMPONENT_ID = 'KBC.lastUpdatedBy.component.id' - UPDATED_BY_CONFIGURATION_ID = 'KBC.lastUpdatedBy.configuration.id' - UPDATED_BY_CONFIGURATION_ROW_ID = 'KBC.lastUpdatedBy.configurationRow.id' - - # Folder name for organizing configurations in the UI - CONFIGURATION_FOLDER_NAME = 'KBC.configuration.folderName' - - # Data type metadata fields - DATATYPE_TYPE = 'KBC.datatype.type' - DATATYPE_NULLABLE = 'KBC.datatype.nullable' - DATATYPE_BASETYPE = 'KBC.datatype.basetype' diff --git a/src/keboola_mcp_server/errors.py b/src/keboola_mcp_server/errors.py deleted file mode 100644 index 474ff1427..000000000 --- a/src/keboola_mcp_server/errors.py +++ /dev/null @@ -1,254 +0,0 @@ -import inspect -import json -import logging -import time -from functools import wraps -from typing import Any, Callable, Mapping, Optional, Type, TypeVar, cast - -import jsonschema -import yaml -from fastmcp import Context -from fastmcp.exceptions import ToolError -from fastmcp.server import middleware as fmw -from fastmcp.server.middleware import CallNext, MiddlewareContext -from fastmcp.utilities.types import find_kwarg_by_type -from mcp import types as mt -from pydantic import BaseModel, ValidationError -from pydantic_core import ErrorDetails - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.storage import StorageEventType -from keboola_mcp_server.mcp import CONVERSATION_ID, ServerState, get_http_request_or_none - -LOG = logging.getLogger(__name__) -F = TypeVar('F', bound=Callable[..., Any]) - -_USER_AGENT_TO_COMPONENT_ID: Mapping[str, str] = { - 'read-only-chat': 'keboola.ai-chat', - 'kai-assistant': 'keboola.kai-assistant', -} - - -MAX_ARG_VALUE_LEN = 10_000 # Maximum length (bytes) of a single tool argument value in the Storage Events payload. - - -class _JsonWrapper(BaseModel): - """ - Utility class for safely encoding arbitrary Python objects to JSON strings. - - Uses Pydantic's serialization to handle complex objects as well as simple types like int, float, bool, str, etc. - Primary use case is serializing tool function parameters for Keboola Storage API events. - """ - - data: Any # The arbitrary object to be JSON serialized - - @classmethod - def encode(cls, obj: Any) -> str: - return json.dumps(cls(data=obj).model_dump()['data'], ensure_ascii=False) - - @classmethod - def encode_truncated(cls, obj: Any) -> str: - """Encode obj to JSON, replacing the value with a truncation notice if it exceeds MAX_ARG_VALUE_LEN.""" - encoded = cls.encode(obj) - # Measure the size of the value as it will appear in the final JSON payload, - # i.e. once it is JSON-encoded again as a string value. - payload_encoded = json.dumps(encoded, ensure_ascii=False) - encoded_bytes = len(payload_encoded.encode('utf-8')) - if encoded_bytes <= MAX_ARG_VALUE_LEN: - return encoded - return json.dumps(f'[value truncated, original length: {encoded_bytes} bytes]') - - -async def _trigger_event( - func: Callable, args: tuple, kwargs: dict, exception: Exception | None, execution_time: float -) -> None: - # TODO: This is not always correct. In general tool functions can be registered - # in the MCP server under different names. - tool_name = func.__name__ - - sig = inspect.signature(func) - bound_args = sig.bind(*args, **kwargs) - bound_args.apply_defaults() - - ctx_param_name = find_kwarg_by_type(func, Context) - assert ctx_param_name, f'The tool function {tool_name} must have a "Context" parameter.' - - ctx = bound_args.arguments.get(ctx_param_name) - assert isinstance(ctx, Context), ( - f'The tool function {tool_name} has invalid "{ctx_param_name}" parameter. ' - f'Expecting instance of "Context", got {type(ctx)}.' - ) - - runtime_info = ServerState.from_context(ctx).runtime_info - - user_agent: str | None = None - if client_params := ctx.session.client_params: - user_agent = f'{client_params.clientInfo.name}/{client_params.clientInfo.version}' - if not user_agent: - user_agent = ctx.client_id - if not user_agent: - if http_rq := get_http_request_or_none(): - user_agent = http_rq.headers.get('User-Agent') - if not user_agent: - user_agent = '' - - # See # https://github.com/keboola/event-schema/blob/main/schema/ext.keboola.mcp-server-tool.json - # for the JSON schema describing the 'keboola.mcp-server-tool' component's event params. - event_params: dict[str, Any] = { - 'mcpServerContext': { - 'appEnv': runtime_info.app_version, - 'version': runtime_info.server_version, - 'userAgent': user_agent, - # For the HTTP-based transports use the HTTP session ID. For other transports use the server ID. - 'sessionId': ctx.session_id or runtime_info.server_id, - 'serverTransport': runtime_info.transport.split('/')[-1], - 'conversationId': ctx.session.state.get(CONVERSATION_ID) or '', - }, - 'tool': { - 'name': tool_name, - 'arguments': [ - {'key': param_name, 'value': _JsonWrapper.encode_truncated(param_value)} - for param_name, param_value in bound_args.arguments.items() - if param_name not in [ctx_param_name, 'self', 'cls'] - ], - }, - } - if exception: - message = f'MCP tool "{tool_name}" call failed. {type(exception).__name__}: {exception}' - event_type: StorageEventType = 'error' - else: - message = f'MCP tool "{tool_name}" call succeeded.' - event_type: StorageEventType = 'success' - - client = KeboolaClient.from_state(ctx.session.state) - resp = await client.storage_client.trigger_event( - message=message, - component_id=_USER_AGENT_TO_COMPONENT_ID.get(user_agent.split(sep='/')[0]) or 'keboola.mcp-server-tool', - event_type=event_type, - params=event_params, - duration=execution_time, - ) - LOG.debug(f'Tool call SAPI event triggered: {resp}') - - -def tool_errors( - default_recovery: Optional[str] = None, - recovery_instructions: Optional[dict[Type[Exception], str]] = None, -) -> Callable[[F], F]: - """ - The MCP tool function decorator that logs exceptions and adds recovery instructions for LLMs. - - :param default_recovery: A fallback recovery instruction to use when no specific instruction - is found for the exception. - :param recovery_instructions: A dictionary mapping exception types to recovery instructions. - :return: The decorated function with error-handling logic applied. - """ - - def decorator(func: Callable): - - @wraps(func) - async def wrapped(*args, **kwargs): - exception: Exception | None = None - start = time.perf_counter() - - try: - return await func(*args, **kwargs) - except Exception as e: - recovery_msg = default_recovery - if recovery_instructions: - for exc_type, msg in recovery_instructions.items(): - if isinstance(e, exc_type): - recovery_msg = msg - break - - error_msg: str | None = None - - if isinstance(e, ValidationError): - error_msg = prettify_validation_error(e) - if recovery_msg: - error_msg += f'\nRecovery: {recovery_msg}' - elif isinstance(e, jsonschema.ValidationError): - error_msg = str(e) - if recovery_msg: - error_msg += f'\nRecovery: {recovery_msg}' - elif recovery_msg: - error_msg = f'{e}\nRecovery: {recovery_msg}' - - try: - if error_msg: - raise ToolError(error_msg) from e - else: - raise e - except Exception as e: - LOG.exception(f'MCP tool "{func.__name__}" call failed. {type(e).__name__}: {e}') - exception = e - raise - - finally: - try: - await _trigger_event(func, args, kwargs, exception, time.perf_counter() - start) - except Exception as e: - # Event logging is best-effort telemetry — never fail the tool because of it. - # The tool result (success or failure) is already determined before this point. - LOG.warning(f'Failed to trigger tool event for "{func.__name__}" tool: {e}', exc_info=True) - - return cast(F, wrapped) - - return decorator - - -def _format_validation_errors(errors: list[ErrorDetails]) -> dict[str, Any]: - """ - Formats Pydantic validation errors into a structured dictionary. - - :param errors: List of error dictionaries from ValidationError.errors() - :return: Dictionary with formatted errors including field, message, and extra fields - """ - formatted_errors: list[dict[str, Any]] = [] - for error in errors: - error_dict: dict[str, Any] = { - 'field': '.'.join(str(i) for i in error.get('loc', [])), - 'message': error.get('msg', 'Validation error'), - 'extra': {str(key): str(value) for key, value in error.items() if key not in {'loc', 'msg'}}, - } - formatted_errors.append(error_dict) - return {'errors': formatted_errors} - - -def prettify_validation_error(error: ValidationError) -> str: - """ - Formats a Pydantic ValidationError into a human and LLM-readable YAML string. - - :param error: The Pydantic ValidationError to format - :return: A formatted YAML string with error details - """ - error_count = len(error.errors()) - model_name = getattr(error, 'title', 'unknown') - header = f'Found {error_count} validation error(s) for {model_name}' - formatted = _format_validation_errors(error.errors()) - try: - yaml_str = yaml.dump(formatted, default_flow_style=False, sort_keys=False, allow_unicode=True) - except Exception: - yaml_str = str(formatted) - - return f'{header}\n{yaml_str}' - - -class ValidationErrorMiddleware(fmw.Middleware): - """ - Middleware that catches Pydantic ValidationError and formats it with explicit field locations. - - This middleware intercepts tool calls and catches any Pydantic ValidationError that occurs - during argument validation. It then formats the error message to clearly show which fields - are missing or invalid, making it easier for both humans and LLMs to understand the issue. - """ - - async def on_call_tool( - self, - context: MiddlewareContext[mt.CallToolRequestParams], - call_next: CallNext[mt.CallToolRequestParams, mt.CallToolResult], - ) -> mt.CallToolResult: - try: - return await call_next(context) - except ValidationError as e: - raise ToolError(prettify_validation_error(e)) from e diff --git a/src/keboola_mcp_server/generate_tool_docs.py b/src/keboola_mcp_server/generate_tool_docs.py deleted file mode 100644 index 1caded935..000000000 --- a/src/keboola_mcp_server/generate_tool_docs.py +++ /dev/null @@ -1,205 +0,0 @@ -import asyncio -import json -import logging -import re -import sys -from collections import defaultdict -from operator import attrgetter -from typing import Iterable, Mapping, Optional - -from fastmcp import FastMCP -from fastmcp.tools import Tool -from mcp.types import ToolAnnotations - -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.server import create_server -from keboola_mcp_server.tools.components.tools import COMPONENT_TOOLS_TAG -from keboola_mcp_server.tools.constants import FLOW_TOOLS_TAG -from keboola_mcp_server.tools.doc import DOC_TOOLS_TAG -from keboola_mcp_server.tools.jobs import JOB_TOOLS_TAG -from keboola_mcp_server.tools.oauth import OAUTH_TOOLS_TAG -from keboola_mcp_server.tools.project import PROJECT_TOOLS_TAG -from keboola_mcp_server.tools.search import SEARCH_TOOLS_TAG -from keboola_mcp_server.tools.semantic import SEMANTIC_TOOLS_TAG -from keboola_mcp_server.tools.sql import SQL_TOOLS_TAG -from keboola_mcp_server.tools.storage import STORAGE_TOOLS_TAG - -LOG = logging.getLogger(__name__) - - -class ToolCategory: - """Encapsulates rules for categorizing tools based on their name.""" - - def __init__(self, name: str, tag: str): - self.name = name - self.tag = tag - - def matches(self, tool_tag: str | Iterable[str]) -> bool: - """Checks if the category matches the tool tag, extended to check existance in the lists of tags.""" - tool_tags = [tool_tag] if isinstance(tool_tag, str) else tool_tag - return self.tag in tool_tags - - def __str__(self): - return self.name - - -OTHER_CATEGORY = ToolCategory('Other Tools', 'other') - - -class ToolDocumentationGenerator: - """Generates documentation for tools.""" - - def __init__(self, tools: list[Tool], categories: list[ToolCategory], output_path: str = 'TOOLS.md'): - self._tools = tools - self._categories = categories - self._output_path = output_path - self._categorizer = None - - def generate(self): - self._categorizer = self._group_tools(self._categories) - with open(self._output_path, mode='w', encoding='utf-8') as f: - self._write_header(f) - self._write_index(f, self._categorizer) - self._write_tool_details(f, self._categorizer) - - def _group_tools(self, categories: list[ToolCategory]) -> Mapping[ToolCategory, list[Tool]]: - assert categories, 'Categories are required' - tools_by_category: dict[ToolCategory, list[Tool]] = defaultdict(list) - for tool in self._tools: - has_category = False - for category in categories: - if category.matches(list(tool.tags)): - # We assume that the category we search for is unique per tool tags and exclusive to other - # categories - if not has_category: - has_category = True - LOG.info(f'Tool {tool.name} has category: {category}') - tools_by_category[category].append(tool) - else: - LOG.warning(f'Tool {tool.name} has multiple main mutually exclusive categories: {tool.tags}') - if not has_category: - LOG.info(f'Tool {tool.name} has no category, adding to: {OTHER_CATEGORY}') - tools_by_category[OTHER_CATEGORY].append(tool) - return tools_by_category - - def _write_header(self, f): - LOG.info(f'Writing header to {self._output_path}') - f.write('# Tools Documentation\n') - f.write('This document provides details about the tools available in the Keboola MCP server.\n\n') - - def _write_index(self, f, categorizer: Mapping[ToolCategory, list[Tool]]): - LOG.info(f'Writing index to {self._output_path}') - f.write('## Index\n') - for category in sorted(categorizer, key=attrgetter('name')): - if tools := categorizer[category]: - LOG.info(f'Writing category {category} and its tools ({len(tools)}) to {self._output_path}') - - f.write(f'\n### {category}\n') - for tool in sorted(tools, key=attrgetter('name')): - anchor = self._generate_anchor(tool.name) - first_sentence = self._get_first_sentence(tool.description) - f.write(f'- [{tool.name}](#{anchor}): {first_sentence}\n') - else: - LOG.warning(f'Category {category} has no tools') - f.write('\n---\n') - - def _get_annotations(self, annotations: Optional[ToolAnnotations]) -> str: - if annotations is None: - return '' - str_annotations = [] - if annotations.readOnlyHint: - str_annotations.append('read-only') - if annotations.destructiveHint: - str_annotations.append('destructive') - if annotations.idempotentHint: - str_annotations.append('idempotent') - return f'`{", ".join(sorted(str_annotations))}`' if str_annotations else '' - - def _get_tags(self, tags: set[str]) -> str: - return f'`{", ".join(sorted(tags))}`' if tags else '' - - def _get_first_sentence(self, text: Optional[str]) -> str: - """Extracts the first sentence from the given text.""" - if not text: - return 'No description available.' - first_sentence = text.split('.')[0] + '.' - return first_sentence.strip() - - def _generate_anchor(self, text: str) -> str: - """Generate GitHub-style markdown anchor from a header text.""" - anchor = text.lower() - anchor = re.sub(r'[^\w\s-]', '', anchor) - anchor = re.sub(r'\s+', '-', anchor) - return anchor - - def _write_tool_details(self, f, categorizer: Mapping[ToolCategory, list[Tool]]): - LOG.info(f'Writing tool details to {self._output_path}') - for category in categorizer: - if not (tools := categorizer[category]): - LOG.warning(f'Category {category} has no tools') - continue - - f.write(f'\n# {category.name}\n') - for tool in sorted(tools, key=attrgetter('name')): - anchor = self._generate_anchor(tool.name) - f.write(f'\n') - f.write(f'## {tool.name}\n') - annotations = self._get_annotations(tool.annotations) - f.write(f'**Annotations**: {annotations}\n\n') - tags = self._get_tags(tool.tags) - f.write(f'**Tags**: {tags}\n\n') - f.write(f'**Description**:\n\n{tool.description}\n\n') - self._write_json_schema(f, tool) - f.write('\n---\n') - - def _write_json_schema(self, f, tool): - if hasattr(tool, 'model_json_schema'): - f.write('\n**Input JSON Schema**:\n') - f.write('```json\n') - f.write(json.dumps(tool.parameters, indent=2)) - f.write('\n```\n') - else: - f.write('No JSON schema available for this tool.\n') - - -async def generate_docs() -> None: - """Main function to generate docs.""" - logging.basicConfig( - format='%(asctime)s %(name)s %(levelname)s: %(message)s', - level=logging.INFO, - stream=sys.stderr, - ) - - config = Config.from_dict( - { - 'storage_api_url': 'https://connection.keboola.com', - 'log_level': 'INFO', - } - ) - - try: - mcp = create_server(config, runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(mcp, FastMCP) - tools = await mcp.list_tools(run_middleware=False) - categories = [ - ToolCategory('Storage Tools', STORAGE_TOOLS_TAG), - ToolCategory('SQL Tools', SQL_TOOLS_TAG), - ToolCategory('Component Tools', COMPONENT_TOOLS_TAG), - ToolCategory('Flow Tools', FLOW_TOOLS_TAG), - ToolCategory('Jobs Tools', JOB_TOOLS_TAG), - ToolCategory('Documentation Tools', DOC_TOOLS_TAG), - ToolCategory('Search Tools', SEARCH_TOOLS_TAG), - ToolCategory('Semantic Tools', SEMANTIC_TOOLS_TAG), - ToolCategory('OAuth Tools', OAUTH_TOOLS_TAG), - ToolCategory('Project Tools', PROJECT_TOOLS_TAG), - # OTHER_CATEGORY - ] - doc_gen = ToolDocumentationGenerator(tools, categories) - doc_gen.generate() - except Exception as e: - LOG.exception(f'Failed to generate documentation: {e}') - sys.exit(1) - - -if __name__ == '__main__': - asyncio.run(generate_docs()) diff --git a/src/keboola_mcp_server/links.py b/src/keboola_mcp_server/links.py deleted file mode 100644 index 7313168ed..000000000 --- a/src/keboola_mcp_server/links.py +++ /dev/null @@ -1,276 +0,0 @@ -from typing import Literal, cast - -from pydantic import BaseModel, ConfigDict, Field - -from keboola_mcp_server.clients.client import ( - CONDITIONAL_FLOW_COMPONENT_ID, - DATA_APP_COMPONENT_ID, - FLOW_TYPES, - FlowType, - KeboolaClient, -) - -URLType = Literal['ui-detail', 'ui-dashboard', 'docs'] - - -class Link(BaseModel): - model_config = ConfigDict(frozen=True) - - type: URLType = Field(..., description='The type of the URL.') - title: str = Field(..., description='The name of the URL.') - url: str = Field(..., description='The URL.') - - @classmethod - def detail(cls, title: str, url: str) -> 'Link': - return cls(type='ui-detail', title=title, url=url) - - @classmethod - def dashboard(cls, title: str, url: str) -> 'Link': - return cls(type='ui-dashboard', title=title, url=url) - - @classmethod - def docs(cls, title: str, url: str) -> 'Link': - return cls(type='docs', title=title, url=url) - - -class ProjectLinksManager: - FLOW_DOCUMENTATION_URL = 'https://help.keboola.com/flows/' - - def __init__(self, *, base_url: str, project_id: str, branch_id: str | None): - self._base_url = base_url - self._project_id = project_id - self._branch_id = branch_id - - @classmethod - async def from_client(cls, client: KeboolaClient) -> 'ProjectLinksManager': - project_id = await client.storage_client.project_id() - return cls(base_url=client.storage_api_url, project_id=project_id, branch_id=client.branch_id) - - def _url(self, path: str) -> str: - parts = [self._base_url, 'admin/projects', self._project_id] - if self._branch_id: - parts += ['branch', self._branch_id] - parts.append(path) - return '/'.join(parts) - - @staticmethod - def _flow_type_from_component_id(component_id: str | None) -> FlowType | None: - if component_id in FLOW_TYPES: - return cast(FlowType, component_id) - return None - - @staticmethod - def _is_data_app_component(component_id: str | None) -> bool: - return component_id == DATA_APP_COMPONENT_ID - - @staticmethod - def _is_transformation_component(component_id: str) -> bool: - return bool(component_id and 'transformation' in component_id) - - def get_links( - self, - *, - bucket_id: str | None = None, - table_id: str | None = None, - component_id: str | None = None, - configuration_id: str | None = None, - name: str | None = None, - ) -> list[Link]: - """ - Get the most relevant links for Keboola Objects based on the provided mutually exclusive identifiers. - """ - if component_id and configuration_id: - return [ - self.get_component_config_link( - component_id=component_id, - configuration_id=configuration_id, - configuration_name=name or '', - ) - ] - - if component_id: - return [self.get_config_dashboard_link(component_id=component_id, component_name=name or '')] - - if table_id: - return [self.get_table_detail_link_from_table_id(table_id=table_id)] - - if bucket_id: - return [self.get_bucket_detail_link(bucket_id=bucket_id, bucket_name=name or bucket_id)] - return [] - - # --- Project --- - def get_project_detail_link(self) -> Link: - return Link.detail(title='Project Dashboard', url=self._url('')) - - def get_project_links(self) -> list[Link]: - return [self.get_project_detail_link()] - - # --- Flows --- - def get_flow_detail_link(self, flow_id: str | int, flow_name: str, flow_type: FlowType) -> Link: - """Get detail link for a specific flow based on its type.""" - flow_path = 'flows-v2' if flow_type == CONDITIONAL_FLOW_COMPONENT_ID else 'flows' - return Link.detail(title=f'Flow: {flow_name}', url=self._url(f'{flow_path}/{flow_id}')) - - def get_flows_dashboard_link(self, flow_type: FlowType) -> Link: - """Get dashboard link for flows based on the flow type.""" - flow_path = 'flows-v2' if flow_type == CONDITIONAL_FLOW_COMPONENT_ID else 'flows' - flow_label = 'Conditional Flows' if flow_type == CONDITIONAL_FLOW_COMPONENT_ID else 'Flows' - return Link.dashboard(title=f'{flow_label} in the project', url=self._url(flow_path)) - - def get_flows_docs_link(self) -> Link: - return Link.docs(title='Documentation for Keboola Flows', url=self.FLOW_DOCUMENTATION_URL) - - def get_flow_links(self, flow_id: str | int, flow_name: str, flow_type: FlowType) -> list[Link]: - """Get all relevant links for a flow based on its type.""" - return [ - self.get_flow_detail_link(flow_id, flow_name, flow_type), - self.get_flows_dashboard_link(flow_type), - self.get_flows_docs_link(), - ] - - # --- Schedulers --- - def get_scheduler_detail_link(self, flow_id: str | int, flow_type: FlowType) -> Link: - flow_path = 'flows-v2' if flow_type == CONDITIONAL_FLOW_COMPONENT_ID else 'flows' - return Link.detail(title='Schedules', url=self._url(f'{flow_path}/{flow_id}/schedules')) - - # --- Components --- - def get_component_config_link( - self, - component_id: str, - configuration_id: str, - configuration_name: str, - ) -> Link: - """ - Get the link to the configuration of a component based on its type, transformation, data app, flow, - or component. - """ - if self._is_transformation_component(component_id): - return self.get_transformation_config_link( - transformation_type=component_id, - transformation_id=configuration_id, - transformation_name=configuration_name, - ) - if self._is_data_app_component(component_id): - return self.get_data_app_config_link( - configuration_id=configuration_id, - configuration_name=configuration_name, - uses_basic_authentication=False, - ) - if flow_type := self._flow_type_from_component_id(component_id): - return self.get_flow_detail_link( - flow_id=configuration_id, flow_name=configuration_name, flow_type=flow_type - ) - return Link.detail( - title=f'Configuration: {configuration_name}', url=self._url(f'components/{component_id}/{configuration_id}') - ) - - def get_config_dashboard_link(self, component_id: str, component_name: str | None) -> Link: - component_name = f'{component_name}' if component_name else f'Component "{component_id}"' - return Link.dashboard( - title=f'{component_name} Configurations Dashboard', url=self._url(f'components/{component_id}') - ) - - def get_used_components_link(self) -> Link: - return Link.dashboard(title='Used Components Dashboard', url=self._url('components/configurations')) - - def get_configuration_links(self, component_id: str, configuration_id: str, configuration_name: str) -> list[Link]: - return [ - self.get_component_config_link( - component_id=component_id, configuration_id=configuration_id, configuration_name=configuration_name - ), - self.get_config_dashboard_link(component_id=component_id, component_name=None), - ] - - # --- Data Apps --- - def get_data_app_config_link( - self, configuration_id: str, configuration_name: str, uses_basic_authentication: bool - ) -> Link: - title = ( - f'Data App Configuration (To see password, click on "OPEN DATA APP"): {configuration_name}' - if uses_basic_authentication - else f'Data App Configuration: {configuration_name}' - ) - return Link.detail(title=title, url=self._url(f'data-apps/{configuration_id}')) - - def get_data_app_dashboard_link(self) -> Link: - return Link.dashboard(title='Data Apps in the project', url=self._url('data-apps')) - - def get_data_app_deployment_link(self, deployment_link: str) -> Link: - return Link.detail(title='Data App Deployment', url=deployment_link) - - def get_data_app_links( - self, - configuration_id: str, - configuration_name: str, - deployment_link: str | None = None, - uses_basic_authentication: bool = False, - ) -> list[Link]: - links = [ - self.get_data_app_config_link( - configuration_id=configuration_id, - configuration_name=configuration_name, - uses_basic_authentication=uses_basic_authentication, - ), - self.get_data_app_dashboard_link(), - ] - if deployment_link: - links.append(self.get_data_app_deployment_link(deployment_link)) - return links - - # --- Transformations --- - def get_transformations_dashboard_link(self) -> Link: - return Link.dashboard(title='Transformations dashboard', url=self._url('transformations-v2')) - - def get_transformation_config_link( - self, transformation_type: str, transformation_id: str, transformation_name: str - ) -> Link: - return Link.detail( - title=f'Transformation: {transformation_name}', - url=self._url(f'transformations-v2/{transformation_type}/{transformation_id}'), - ) - - def get_transformation_links( - self, transformation_type: str, transformation_id: str, transformation_name: str - ) -> list[Link]: - return [ - self.get_transformation_config_link(transformation_type, transformation_id, transformation_name), - self.get_transformations_dashboard_link(), - ] - - # --- Jobs --- - def get_job_detail_link(self, job_id: str) -> Link: - return Link.detail(title=f'Job: {job_id}', url=self._url(f'queue/{job_id}')) - - def get_jobs_dashboard_link(self) -> Link: - return Link.dashboard(title='Jobs in the project', url=self._url('queue')) - - def get_job_links(self, job_id: str) -> list[Link]: - return [self.get_job_detail_link(job_id), self.get_jobs_dashboard_link()] - - # --- Buckets --- - def get_bucket_detail_link(self, bucket_id: str, bucket_name: str) -> Link: - return Link.detail(title=f'Bucket: {bucket_name}', url=self._url(f'storage/{bucket_id}')) - - def get_bucket_dashboard_link(self) -> Link: - return Link.dashboard(title='Buckets in the project', url=self._url('storage')) - - def get_bucket_links(self, bucket_id: str, bucket_name: str) -> list[Link]: - return [ - self.get_bucket_detail_link(bucket_id, bucket_name), - self.get_bucket_dashboard_link(), - ] - - # --- Tables --- - def get_table_detail_link(self, bucket_id: str, table_name: str) -> Link: - return Link.detail(title=f'Table: {table_name}', url=self._url(f'storage/{bucket_id}/table/{table_name}')) - - def get_table_detail_link_from_table_id(self, table_id: str) -> Link: - table_name = table_id.split('.')[-1] - bucket_id = '.'.join(table_id.split('.')[:-1]) - return self.get_table_detail_link(bucket_id=bucket_id, table_name=table_name) - - def get_table_links(self, bucket_id: str, bucket_name: str, table_name: str) -> list[Link]: - return [ - self.get_table_detail_link(bucket_id, table_name), - self.get_bucket_detail_link(bucket_id=bucket_id, bucket_name=bucket_name), - ] diff --git a/src/keboola_mcp_server/mcp.py b/src/keboola_mcp_server/mcp.py deleted file mode 100644 index 1cc80f120..000000000 --- a/src/keboola_mcp_server/mcp.py +++ /dev/null @@ -1,644 +0,0 @@ -""" -This module overrides FastMCP.add_tool() to improve conversion of tool function docstrings -into tool descriptions. -It also provides a decorator that MCP tool functions can use to inject session state into their Context parameter -and other utilities for the MCP server. -""" - -import asyncio -import dataclasses -import logging -import textwrap -from collections.abc import Awaitable, Callable, Iterable -from typing import Any, TypeVar -from unittest.mock import MagicMock - -import toon_format -from fastmcp import Context, FastMCP -from fastmcp.exceptions import ToolError -from fastmcp.server import middleware as fmw -from fastmcp.server.dependencies import get_http_request -from fastmcp.server.middleware import CallNext, MiddlewareContext -from fastmcp.tools import Tool -from mcp import types as mt -from mcp.server.auth.middleware.bearer_auth import AuthenticatedUser -from pydantic import BaseModel -from pydantic_core import to_json -from starlette.applications import Starlette -from starlette.requests import Request -from starlette.types import ASGIApp, Receive, Scope, Send - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.oauth import ProxyAccessToken -from keboola_mcp_server.tools.constants import MODIFY_FLOW_TOOL_NAME, SEMANTIC_TOOLS_TAG, UPDATE_FLOW_TOOL_NAME -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) -CONVERSATION_ID = 'conversation_id' - -R = TypeVar('R') -T = TypeVar('T') - -DEFAULT_CONCURRENCY = 10 - -SEMANTIC_TOOLING_FEATURE = 'mcp-semantic-tooling' -SEMANTIC_TOOL_NAMES = { - 'search_semantic_context', - 'get_semantic_context', - 'get_semantic_schema', - 'validate_semantic_query', -} -# Data app tools are supported only in the main/production branch. This single set is the source of -# truth for both the on_list_tools filter and the on_call_tool guard — keeping them in sync is what -# prevents a new (possibly destructive) data app tool from leaking onto non-main branches. -DATA_APP_BRANCH_GATED_TOOLS = { - 'modify_streamlit_data_app', - 'modify_python_js_data_app', - 'create_python_js_data_app_git_credential', - 'get_data_apps', - 'deploy_data_app', - 'delete_python_js_data_app_draft', -} - - -def is_read_only_tool(tool: Tool) -> bool: - """Check if a tool has readOnlyHint=True annotation.""" - if tool.annotations is None: - return False - return tool.annotations.readOnlyHint is True - - -def is_semantic_tool(tool: Tool) -> bool: - """Check whether a tool belongs to semantic tooling.""" - return SEMANTIC_TOOLS_TAG in (tool.tags or set()) or tool.name in SEMANTIC_TOOL_NAMES - - -@dataclasses.dataclass(frozen=True) -class ServerState: - config: Config - runtime_info: ServerRuntimeInfo - - @classmethod - def from_context(cls, ctx: Context) -> 'ServerState': - server_state = ctx.request_context.lifespan_context - if not isinstance(server_state, ServerState): - raise ValueError('ServerState is not available in the context.') - return server_state - - @classmethod - def from_starlette(cls, app: Starlette) -> 'ServerState': - server_state = app.state.server_state - if not isinstance(server_state, ServerState): - raise ValueError('ServerState is not available in the Starlette app.') - return server_state - - -class ForwardSlashMiddleware: - def __init__(self, app: ASGIApp): - self._app = app - - async def __call__(self, scope: Scope, receive: Receive, send: Send): - LOG.debug(f'ForwardSlashMiddleware: scope={scope}') - - if scope['type'] == 'http': - path = scope['path'] - if path in ['/mcp']: - scope = dict(scope) - scope['path'] = f'{path}/' - - await self._app(scope, receive, send) - - -class KeboolaMcpServer(FastMCP): - def add_tool(self, tool: Tool) -> None: - """Applies `textwrap.dedent()` function to the tool's docstring, if no explicit description is provided.""" - update = {} - if tool.description: - description = textwrap.dedent(tool.description).strip() - if description != tool.description: - update['description'] = description - if not tool.serializer: - update['serializer'] = _exclude_none_serializer - - if update: - tool = tool.model_copy(update=update) - - super().add_tool(tool) - - -def get_http_request_or_none() -> Request | None: - try: - return get_http_request() - except RuntimeError: - return None - - -class SessionStateMiddleware(fmw.Middleware): - """ - FastMCP middleware that manages session state in the Context parameter. - - This middleware sets up the session state containing instances of `KeboolaClient` and `WorkspaceManager` - in the tool function's Context. These are initialized using the MCP server configuration, which is - composed of the following parameter sources: - - * Initial configuration obtained from CLI parameters when starting the server - * Environment variables - * HTTP headers - * URL query parameters - - Note: HTTP headers and URL query parameters are only used when the server runs on HTTP-based transport. - """ - - async def on_request( - self, - context: fmw.MiddlewareContext[mt.Request[Any, Any]], - call_next: fmw.CallNext[mt.Request[Any, Any], Any], - ) -> Any: - """ - Manages session state in the Context parameter. This middleware sets up the session state for all the other - MCP functions down the chain. It is called for each tool, prompt, resource, etc. calls. - - In fastmcp 2.13.0+, this must run in on_request rather than on_message because ctx.session - requires the request context to be available. - - :param context: Middleware context containing FastMCP context. - :param call_next: Next middleware in the chain to call. - :returns: Result from executing the middleware chain. - """ - # Skip session setup for initialize request - session state is only needed for actual operations - if context.method == 'initialize': - return await call_next(context) - - ctx = context.fastmcp_context - assert isinstance(ctx, Context), f'Expecting Context, got {type(ctx)}.' - - if not isinstance(ctx.session, MagicMock): - server_state = ServerState.from_context(ctx) - config: Config = server_state.config - runtime_info: ServerRuntimeInfo = server_state.runtime_info - - # IMPORTANT: Since mcp 1.12.4 and fastmcp 2.11 the fastmcp.server.dependencies.get_http_request() - # returns the same object as ctx.request_context.request. - - if http_rq := get_http_request_or_none(): - config = self.apply_request_config(http_rq, config) - - # TODO: We could probably get rid of the 'state' attribute set on ctx.session and just - # pass KeboolaClient and WorkspaceManager instances to a tool as extra parameters. - - # Skip branch validation for /list requests (tools/list, resources/list, prompts/list, etc.) - # so that clients can discover available tools even when the configured branch ID doesn't - # exist yet. For these requests the client is created without a branch ID. Otherwise, the branch is - # validated via a SAPI call. - if context.method.endswith('/list'): - if config.branch_id: - LOG.info(f'Skipping branch validation for {context.method} request.') - config = dataclasses.replace(config, branch_id=None) - - state = await self.create_session_state(config, runtime_info) - ctx.session.state = state - - try: - return await call_next(context) - finally: - # NOTE: This line is commented following a bug related to session state clearance in Claude client - # ctx.session.state = {} - pass - - @classmethod - def _get_headers(cls, runtime_info: ServerRuntimeInfo) -> dict[str, Any]: - """ - :param runtime_info: Runtime information - :return: Additional headers for the requests used for tracing the MCP server - """ - return { - 'User-Agent': ( - f'Keboola MCP Server/{runtime_info.server_version} app_env={runtime_info.app_env} ' - f'transport={runtime_info.transport}' - ), - 'MCP-Server-Transport': runtime_info.transport or 'NA', - 'MCP-Server-Versions': ( - f'keboola-mcp-server/{runtime_info.server_version} mcp/{runtime_info.mcp_library_version} ' - f'fastmcp/{runtime_info.fastmcp_library_version}' - ), - } - - @classmethod - def apply_request_config(cls, http_rq: Request, config: Config) -> Config: - LOG.debug(f'Injecting headers: http_rq={http_rq}, headers={http_rq.headers}') - config = config.replace_by(http_rq.headers) - - if user := http_rq.scope.get('user'): - LOG.debug(f'Injecting bearer and SAPI tokens: user={user}, access_token={user.access_token}') - assert isinstance(user, AuthenticatedUser), f'Expecting AuthenticatedUser, got: {type(user)}' - assert isinstance( - user.access_token, ProxyAccessToken - ), f'Expecting ProxyAccessToken, got: {type(user.access_token)}' - config = dataclasses.replace( - config, - storage_token=user.access_token.sapi_token, - bearer_token=user.access_token.delegate.token, - ) - - return config - - @classmethod - async def create_session_state( - cls, - config: Config, - runtime_info: ServerRuntimeInfo, - readonly: bool | None = None, - ) -> dict[str, Any]: - """ - Creates `KeboolaClient` and `WorkspaceManager` instances and returns them in the session state. - - :param config: The MCP server configuration. - :param runtime_info: The MCP server runtime information. - :param readonly: If True, the `KeboolaClient` will only use HTTP GET, HEAD operations. - :return: The session state dictionary containing the created client and workspace manager instances. - """ - LOG.info(f'Creating SessionState from config: {config}.') - - state: dict[str, Any] = {} - try: - if not config.storage_token: - raise ValueError('Storage API token is not provided.') - if not config.storage_api_url: - raise ValueError('Storage API URL is not provided.') - - client = await KeboolaClient( - storage_api_url=config.storage_api_url, - storage_api_token=config.storage_token, - bearer_token=config.bearer_token, - headers=cls._get_headers(runtime_info), - readonly=readonly, - ).with_branch_id(config.branch_id) - - state[KeboolaClient.STATE_KEY] = client - LOG.info('Successfully initialized Storage API client.') - except Exception as e: - LOG.error(f'Failed to initialize Keboola client: {e}') - raise - - try: - workspace_manager = await WorkspaceManager.create(client, config.workspace_schema) - state[WorkspaceManager.STATE_KEY] = workspace_manager - LOG.info('Successfully initialized Storage API Workspace manager.') - except Exception as e: - LOG.error(f'Failed to initialize Storage API Workspace manager: {e}') - raise - - state[CONVERSATION_ID] = config.conversation_id - return state - - -class ToolsFilteringMiddleware(fmw.Middleware): - """ - This middleware filters out tools that are not available in the current project. The filtering is based on the - project features. - - The middleware intercepts the `on_list_tools()` call and removes the unavailable tools - from the list. The AI assistants should not even see the tools that are not available in the current project. - - The middleware also intercepts the `on_call_tool()` call and raises an exception if a call is attempted to a tool - that is not available in the current project. - - Tool visibility for modify_flow and update_flow: - - | Token Type | Role | modify_flow | update_flow | Read-Only Tools | - |-----------------|-------------|-------------|-------------|-----------------| - | OAuth (any) | any | ✅ | ❌ | ✅ | - | SAPI | admin/share | ✅ | ❌ | ✅ | - | SAPI | ''/guest | ❌ | ✅ | ✅ | - | SAPI/OAuth | readOnly | ❌ | ❌ | ✅ | - """ - - @staticmethod - def _is_oauth_authenticated(ctx: Context) -> bool: - """ - Detect if the user is authenticated via OAuth. - - Returns True if bearer token is present, False otherwise. - """ - keboola_client = KeboolaClient.from_state(ctx.session.state) - return bool(keboola_client.bearer_token) - - @staticmethod - async def get_token_info(ctx: Context) -> JsonDict: - assert isinstance(ctx, Context), f'Expecting Context, got {type(ctx)}.' - client = KeboolaClient.from_state(ctx.session.state) - return await client.storage_client.verify_token() - - @staticmethod - def get_project_features(token_info: JsonDict) -> set[str]: - owner_data = token_info.get('owner', {}) - if not isinstance(owner_data, dict): - return set() - return set(filter(None, owner_data.get('features', []))) - - @staticmethod - def get_token_role(token_info: JsonDict) -> str: - admin_data = token_info.get('admin', {}) - if isinstance(admin_data, dict): - role = admin_data.get('role') - if isinstance(role, str): - return role - return '' - - @staticmethod - def is_client_using_main_branch(ctx: Context) -> bool: - """ - Checks if the current branch is the main/production branch. - """ - client = KeboolaClient.from_state(ctx.session.state) - branch_id = client.branch_id - - # We use None for the branch id referring to the main/production branch in the KeboolaClient. - return branch_id is None - - async def on_list_tools( - self, context: MiddlewareContext[mt.ListToolsRequest], call_next: CallNext[mt.ListToolsRequest, list[Tool]] - ) -> list[Tool]: - tools = await call_next(context) - token_info = await self.get_token_info(context.fastmcp_context) - features = self.get_project_features(token_info) - token_role = self.get_token_role(token_info).lower() - - if 'hide-conditional-flows' in features: - tools = [t for t in tools if t.name != 'create_conditional_flow'] - else: - tools = [t for t in tools if t.name != 'create_flow'] - - # Show modify_flow to: admin, share, OR OAuth users - # Show update_flow to: everyone else (except readOnly, handled below) - is_oauth = self._is_oauth_authenticated(context.fastmcp_context) - if token_role in ('admin', 'share') or is_oauth: - tools = [t for t in tools if t.name != UPDATE_FLOW_TOOL_NAME] - else: - tools = [t for t in tools if t.name != MODIFY_FLOW_TOOL_NAME] - - if not self.is_client_using_main_branch(context.fastmcp_context): - # Filter out data app tools when the client is not using the main/production branch - tools = [t for t in tools if t.name not in DATA_APP_BRANCH_GATED_TOOLS] - - if token_role == 'readonly': - tools = [t for t in tools if is_read_only_tool(t)] - LOG.debug(f'Read-only access: filtered to {len(tools)} read-only tools for role={token_role}') - - if SEMANTIC_TOOLING_FEATURE not in features: - tools = [t for t in tools if not is_semantic_tool(t)] - - return tools - - @staticmethod - def authorize_tool_call( - *, - tool_name: str, - is_read_only: bool, - is_semantic: bool, - token_role: str, - features: set[str], - is_oauth: bool, - is_main_branch: bool, - ) -> str | None: - """ - Decide whether a call to ``tool_name`` is allowed given the project features, the token role, - the authentication mode and the branch. - - This is the single source of truth for the project-feature / token-role / branch gating. - :meth:`on_call_tool` applies it to MCP tool calls; the raw ``/preview/configuration`` Starlette - route reuses it (see ``preview.py``) so the preview path enforces exactly the same rules. - - :return: A denial message if the call is not allowed, or ``None`` if it is allowed. - """ - token_role = token_role.lower() - - if token_role == 'readonly' and not is_read_only: - return ( - f'Access denied: The tool "{tool_name}" requires write permissions. ' - f'Your current role ({token_role}) only allows read-only operations. ' - f'Contact your administrator to request write access.' - ) - - if SEMANTIC_TOOLING_FEATURE not in features and is_semantic: - return ( - f'The tool "{tool_name}" is not available in this project. ' - 'Please ask Keboola support to enable "Semantic Layer Tooling" feature.' - ) - - if 'hide-conditional-flows' in features: - if tool_name == 'create_conditional_flow': - return ( - 'The "create_conditional_flow" tool is not available in this project. ' - 'Please ask Keboola support to enable "Conditional Flows" feature ' - 'or use "create_flow" tool instead.' - ) - else: - if tool_name == 'create_flow': - return ( - 'The "create_flow" tool is not available in this project. ' - 'This project uses "Conditional Flows", ' - 'please use "create_conditional_flow" tool instead.' - ) - - if token_role in ('admin', 'share') or is_oauth: - if tool_name == UPDATE_FLOW_TOOL_NAME: - return ( - 'The "update_flow" tool is not available for admin/OAuth tokens. ' - f'Use "{MODIFY_FLOW_TOOL_NAME}" to manage schedules instead.' - ) - else: - if tool_name == MODIFY_FLOW_TOOL_NAME: - return ( - f'The "{MODIFY_FLOW_TOOL_NAME}" tool is not available for this token. ' - f'Use "{UPDATE_FLOW_TOOL_NAME}" to update flow configuration instead.' - ) - - if tool_name in DATA_APP_BRANCH_GATED_TOOLS and not is_main_branch: - return 'Data apps are supported only in the main production branch.' - - return None - - async def on_call_tool( - self, - context: MiddlewareContext[mt.CallToolRequestParams], - call_next: CallNext[mt.CallToolRequestParams, mt.CallToolResult], - ) -> mt.CallToolResult: - tool = await context.fastmcp_context.fastmcp.get_tool(context.message.name) - token_info = await self.get_token_info(context.fastmcp_context) - - denial = self.authorize_tool_call( - tool_name=tool.name, - is_read_only=is_read_only_tool(tool), - is_semantic=is_semantic_tool(tool), - token_role=self.get_token_role(token_info), - features=self.get_project_features(token_info), - is_oauth=self._is_oauth_authenticated(context.fastmcp_context), - is_main_branch=self.is_client_using_main_branch(context.fastmcp_context), - ) - if denial: - raise ToolError(denial) - - return await call_next(context) - - -def _to_python(data: Any, exclude_none: bool = True) -> Any | None: - if isinstance(data, BaseModel): - return data.model_dump(exclude_none=exclude_none, by_alias=False) - elif isinstance(data, (list, tuple)): - # Handle sequences of BaseModels - cleaned = [] - for item in data: - if isinstance(item, BaseModel): - cleaned.append(item.model_dump(exclude_none=exclude_none, by_alias=False)) - elif item is not None: - cleaned.append(_to_python(item, exclude_none=exclude_none)) - elif not exclude_none: - cleaned.append(None) - return cleaned - elif isinstance(data, dict): - # Handle dictionaries that might contain BaseModels - cleaned = {} - for key, value in data.items(): - if isinstance(value, BaseModel): - cleaned[key] = value.model_dump(exclude_none=exclude_none, by_alias=False) - elif value is not None: - cleaned[key] = _to_python(value, exclude_none=exclude_none) - elif not exclude_none: - cleaned[key] = None - return cleaned - elif data is not None: - return data - else: - return None - - -def _filter_toon_nulls(data: Any) -> Any: - """ - Drops None fields while keeping TOON's list-of-dicts alignment. - Single-item lists drop keys that have None assigned. - Multi-item lists drop keys that have None assigned in all items. - """ - if isinstance(data, list): - if not data: - return data - - elif all(isinstance(item, dict) for item in data): - if len(data) == 1: - return [_filter_toon_nulls(data[0])] - - ordered_keys_with_values: list[str] = [] - seen_keys_with_values: set[str] = set() - for item in data: - for key, value in item.items(): - if value is not None and key not in seen_keys_with_values: - seen_keys_with_values.add(key) - ordered_keys_with_values.append(key) - - cleaned_items: list[dict[str, Any]] = [] - for item in data: - cleaned_item: dict[str, Any] = {} - for key in ordered_keys_with_values: - value = item.get(key) - if value is None: - cleaned_item[key] = None - else: - cleaned_item[key] = _filter_toon_nulls(value) - cleaned_items.append(cleaned_item) - - return cleaned_items - - else: - return [_filter_toon_nulls(item) if item is not None else None for item in data] - - if isinstance(data, dict): - cleaned: dict[str, Any] = {} - for key, value in data.items(): - if value is None: - continue - cleaned[key] = _filter_toon_nulls(value) - return cleaned - - return data - - -def _exclude_none_serializer(data: Any) -> str: - if (cleaned := _to_python(data)) is not None: - return to_json(cleaned, fallback=str).decode('utf-8') - else: - return '' - - -def toon_serializer(data: Any) -> str: - return toon_format.encode(_to_python(data, exclude_none=False)) - - -def toon_serializer_compact(data: Any) -> str: - return toon_format.encode(_filter_toon_nulls(_to_python(data, exclude_none=False))) - - -async def process_concurrently( - items: Iterable[T], - afunc: Callable[[T], Awaitable[R]], - max_concurrency: int = DEFAULT_CONCURRENCY, -) -> list[R | BaseException]: - """ - Asynchronously process a collection of items with a specified concurrency limit. - - :param items: The collection of items to process. - :param afunc: An asynchronous function to apply to each item. - :param max_concurrency: The maximum number of concurrent executions allowed. - :return: A list of results or exceptions from processing each item. - The order of results corresponds to the order of the input items. - """ - if max_concurrency <= 0: - raise ValueError('max_concurrency must be a positive integer.') - - semaphore = asyncio.Semaphore(max_concurrency) - - async def process_item_with_semaphore(item: T) -> R: - async with semaphore: - return await afunc(item) - - tasks = [asyncio.create_task(process_item_with_semaphore(item)) for item in items] - - return await asyncio.gather(*tasks, return_exceptions=True) - - -class AggregateError(Exception): - """Exception that aggregates multiple exceptions (Python 3.10 compatible alternative to ExceptionGroup).""" - - def __init__(self, message: str, exceptions: Iterable[BaseException]): - self.message = message - self.exceptions = list(exceptions) - super().__init__(message, self.exceptions) - - def __str__(self) -> str: - error_details = '; '.join(f'{type(e).__name__}: {e}' for e in self.exceptions) - return f'{self.message} ({len(self.exceptions)} errors): {error_details}' - - -def unwrap_results(results: Iterable[R | BaseException], message: str = 'Multiple errors occurred') -> list[R]: - """ - Unwrap results from process_concurrently, raising an AggregateError if any exceptions occurred. - - :param results: List of results or exceptions from process_concurrently. - :param message: Message for the AggregateError if exceptions are present. - :return: List of successful results. - :raises AggregateError: If any results are exceptions. - """ - successes: list[R] = [] - exceptions: list[BaseException] = [] - - for result in results: - if isinstance(result, BaseException): - exceptions.append(result) - else: - successes.append(result) - - if exceptions: - raise AggregateError(message, exceptions) - - return successes diff --git a/src/keboola_mcp_server/oauth.py b/src/keboola_mcp_server/oauth.py deleted file mode 100644 index fcc8f5099..000000000 --- a/src/keboola_mcp_server/oauth.py +++ /dev/null @@ -1,680 +0,0 @@ -import gzip -import json -import logging -import math -import os -import re -import secrets -import time -from http.client import HTTPException -from typing import Any, Mapping, cast -from urllib.parse import urljoin - -import httpx -import jwt.api_jws -from fastmcp.server.auth.auth import OAuthProvider -from mcp.server.auth.provider import ( - AccessToken, - AuthorizationCode, - AuthorizationParams, - RefreshToken, - construct_redirect_uri, -) -from mcp.server.auth.settings import ClientRegistrationOptions -from mcp.shared.auth import InvalidRedirectUriError, OAuthClientInformationFull, OAuthToken -from pydantic import AnyHttpUrl, AnyUrl - -LOG = logging.getLogger(__name__) -_OAUTH_LOG_ALL = bool(os.getenv('KEBOOLA_MCP_SERVER_OAUTH_LOG_ALL')) -_RE_LOCALHOST = re.compile(r'^(localhost|127\.0\.0\.1|\[::1]|::1)$', re.IGNORECASE) -_ALLOWED_DOMAINS = { - 'https': [ - # Any keboola.com/dev subdomain EXCEPT user-deployable data-app subdomains, which live under a - # '*.hub..keboola.com' host. A free-trial user can deploy a data app whose '/callback' would - # otherwise capture the OAuth code, so we reject any host that contains a 'hub' DNS label (RISK-76). - re.compile(r'^(?!(?:.*\.)?hub\.).+\.keboola\.(com|dev)$', re.IGNORECASE), - re.compile(r'^(.*\.)?chatgpt\.com$', re.IGNORECASE), - re.compile(r'^(.*\.)?claude\.ai$', re.IGNORECASE), - re.compile(r'^librechat\.glami-ml\.com$', re.IGNORECASE), # no subdomains allowed - re.compile(r'^(.*\.)?make\.com$', re.IGNORECASE), - re.compile(r'^api\.devin\.ai$', re.IGNORECASE), # devin.ai API domain - re.compile(r'^cloud\.onyx\.app$', re.IGNORECASE), # onyx.app OAuth callback - re.compile(r'^global\.consent\.azure-apim\.net$', re.IGNORECASE), # Azure APIM consent domain - re.compile(r'^n8n\.groupondev\.com$', re.IGNORECASE), - re.compile(r'^n8n-business\.groupondev\.com$', re.IGNORECASE), - re.compile(r'^n8n-merchant\.groupondev\.com$', re.IGNORECASE), - re.compile(r'^n8n-llm-traffic\.groupondev\.com$', re.IGNORECASE), - re.compile(r'^n8n-finance\.groupondev\.com$', re.IGNORECASE), - re.compile(r'^n8n-playground\.groupondev\.com$', re.IGNORECASE), - re.compile(r'^n8n-staging\.groupondev\.com$', re.IGNORECASE), - ], - 'http': [_RE_LOCALHOST], - 'cursor': [re.compile(r'^(anysphere\.cursor-retrieval|anysphere\.cursor-mcp)$', re.IGNORECASE)], -} - - -def _log_debug(msg: str) -> None: - """ - Logs the message at the DEBUG level if the environment variable KEBOOLA_MCP_SERVER_OAUTH_LOG_ALL is set. - Use this function for logging sensitive information. It logs nothing by default. - """ - if _OAUTH_LOG_ALL: - LOG.debug(msg) - - -class _OAuthClientInformationFull(OAuthClientInformationFull): - def validate_scope(self, requested_scope: str | None) -> list[str] | None: - # This is supposed to verify that the requested scopes are a subset of the scopes that the client registered. - # That, however, would require a persistent registry of clients. - # So, instead we pretend that all the requested scopes have been registered. - if requested_scope: - return requested_scope.split(' ') - else: - return None - - def validate_redirect_uri(self, redirect_uri: AnyUrl | None) -> AnyUrl: - # Ideally, this should verify the redirect_uri against the URI registered by the client. - # That, however, would require a persistent registry of clients. - # So, instead we require the clients to send their redirect URI in the authorization request, - # and we discard all URIs that are not on a whitelist. - if not redirect_uri: - LOG.warning('[validate_redirect_uri] No redirect_uri specified.') - raise InvalidRedirectUriError('The redirect_uri must be specified.') - - stripped_uri = self._strip_redirect_uri(redirect_uri) - if not redirect_uri.scheme: - LOG.warning(f'[validate_redirect_uri] No scheme in redirect_uri: {stripped_uri}') - raise InvalidRedirectUriError(f'Invalid redirect_uri: {stripped_uri}') - - # The custom schemes (e.g. cursor://) require a custom handler registered in a browser. - # They are used for redirecting a browser to a locally running app. - - if allowed_domains := _ALLOWED_DOMAINS.get(redirect_uri.scheme): - if not any(p.fullmatch(redirect_uri.host or '') for p in allowed_domains): - LOG.warning(f'[validate_redirect_uri] Unknown domain in redirect_uri: {stripped_uri}') - raise InvalidRedirectUriError(f'Invalid redirect_uri: {stripped_uri}') - - else: - LOG.warning(f'[validate_redirect_uri] Forbidden scheme in redirect_uri: {stripped_uri}') - raise InvalidRedirectUriError(f'Invalid redirect_uri: {stripped_uri}') - - LOG.info(f'[validate_redirect_uri] Accepted redirect_uri: {stripped_uri}]') - return redirect_uri - - @staticmethod - def _strip_redirect_uri(redirect_uri: AnyUrl) -> AnyUrl: - return AnyUrl.build(scheme=redirect_uri.scheme or '', host=redirect_uri.host or '', port=redirect_uri.port) - - -class _ExtendedAuthorizationCode(AuthorizationCode): - oauth_access_token: AccessToken - oauth_refresh_token: RefreshToken - - -class ProxyAccessToken(AccessToken): - delegate: AccessToken - # This token is created by the MCP server and used for calling AI Service and Jobs Queue, - # which do not support 'Authorization: Bearer ' header yet. - sapi_token: str - - -class ProxyRefreshToken(RefreshToken): - delegate: RefreshToken - - -class SimpleOAuthProvider(OAuthProvider): - - def __init__( - self, - *, - storage_api_url: str, - mcp_server_url: str, - callback_endpoint: str, - client_id: str, - client_secret: str, - server_url: str, - scope: str, - jwt_secret: str | None = None, - ) -> None: - """ - Creates OAuth provider implementation. - - :param storage_api_url: The URL of the Storage API service. - :param mcp_server_url: The URL of the MCP server itself. - :param callback_endpoint: The endpoint where the OAuth server redirects to after the user authorizes. - :param client_id: The client ID registered with the OAuth server. - :param client_secret: The client secret registered with the OAuth server - :param server_url: The URL of the OAuth server that the MCP server should authenticate to. - :param scope: The scope of access to request from the OAuth server. - :param jwt_secret: The secret key for encoding and decoding JWT tokens. - """ - super().__init__( - base_url=mcp_server_url, - client_registration_options=ClientRegistrationOptions(enabled=True), - ) - - self._sapi_tokens_url = urljoin(storage_api_url, '/v2/storage/tokens') - self._mcp_callback_url = urljoin(mcp_server_url, callback_endpoint) - self._oauth_client_id = client_id - self._oauth_client_secret = client_secret - self._oauth_server_auth_url = urljoin(server_url, '/oauth/authorize') - self._oauth_server_token_url = urljoin(server_url, '/oauth/token') - self._oauth_scope = scope - self._jwt_secret = jwt_secret or secrets.token_hex(32) - - async def get_client(self, client_id: str) -> OAuthClientInformationFull | None: - """ - Gets the information about a registered OAuth client by its client ID. - This specific implementation is a no-op to avoid having to persist the registered clients. - - :param client_id: A string representing the unique OAuth client identifier. - :return: An `_OAuthClientInformationFull` instance which contains just the client ID - and turns off all the client-based validations (e.g. redirect URI and scopes). - """ - client = _OAuthClientInformationFull( - # Use a fake redirect URI. Normally, we would retrieve the client from a persistent registry - # and return the registered redirect URI. - redirect_uris=[AnyHttpUrl('http://foo')], - client_id=client_id, - token_endpoint_auth_method='none', - ) - LOG.debug(f'Client loaded: client_id={client_id}') - return client - - async def register_client(self, client_info: OAuthClientInformationFull) -> None: - """ - Registers an OAuth client. This specific implementation is a no-op to avoid having to persist the registered - clients. It simply logs the client registration details for debugging purposes. - - :param client_info: The full information of the OAuth client to be registered. - """ - # This is a no-op. We don't register clients, otherwise we would need a persistent registry. - LOG.debug(f'Client registered: client_id={client_info.client_id}') - - async def authorize(self, client: OAuthClientInformationFull, params: AuthorizationParams) -> str: - """ - Creates a URL that redirects to the OAuth server for authorization. - - The authorization URL's state parameter is an encrypted JWT that contains all the authorization parameters. - The state expires after 5 minutes. - - :param client: The OAuth client details. - :param params: The authorization parameters provided by the client, such as redirect URI, state, scopes, etc. - - :return: The authorization URL that redirects to the OAuth server. - """ - # Create and encode the authorization state. - # We don't store the authentication states that we create here to avoid having to persist them. - # Instead, we encode them to JWT and pass them back to the client. - # The states expire after 5 minutes. - scopes = cast(list[str], params.scopes or []) - state = { - 'redirect_uri': str(params.redirect_uri), - 'redirect_uri_provided_explicitly': str(params.redirect_uri_provided_explicitly), - # the scopes sent by the MCP server's OAuth client (e.g. claude.ai) - 'scopes': scopes, - 'code_challenge': params.code_challenge, - 'state': params.state, - 'client_id': client.client_id, - 'expires_at': time.time() + 5 * 60, # 5 minutes from now - } - state_jwt = self._encode(state) - - LOG.debug(f'[authorize] client_id={client.client_id}, params={params}, state={state}') - - # create the authorization URL - url_params = { - 'client_id': self._oauth_client_id, - 'response_type': 'code', - 'redirect_uri': self._mcp_callback_url, - 'state': state_jwt, - # send no scopes to Keboola OAuth server and let it use its own default scope - } - - auth_url = construct_redirect_uri(self._oauth_server_auth_url, **url_params) - LOG.debug(f'[authorize] client_id={client.client_id}, params={params}, {auth_url}') - - return auth_url - - async def handle_oauth_callback(self, code: str, state: str) -> str: - """ - Handles the callback from the OAuth server. - - :param code: The authorization code provided by the OAuth server. - :param state: The state originally generated in the authorize() function. - - :return: The URL that redirects back to the AI assistant OAuth client. - """ - # Validate the state first to prevent calling OAuth server with invalid authorization code. - try: - state_data = self._decode(state) - except jwt.InvalidTokenError: - LOG.debug(f'[handle_oauth_callback] Invalid state: {state}', exc_info=True) - raise HTTPException(400, 'Invalid state parameter') - - if not state_data: - LOG.debug(f'[handle_oauth_callback] Invalid state: {state_data}', exc_info=True) - raise HTTPException(400, 'Invalid state parameter') - - if state_data['expires_at'] < time.time(): - LOG.debug(f'[handle_oauth_callback] Expired state: {state_data}', exc_info=True) - raise HTTPException(400, 'Invalid state parameter') - - # Exchange the authorization code for the access token with the OAuth server. - async with self._create_http_client() as http_client: - response = await http_client.post( - self._oauth_server_token_url, - data={ - 'client_id': self._oauth_client_id, - 'client_secret': self._oauth_client_secret, - 'code': code, - 'grant_type': 'authorization_code', - # FYI: Some tutorials use the redirect_uri here, but it does not seem to be required. - # The Keboola OAuth server requires it, but the GitHub OAuth server does not. - 'redirect_uri': self._mcp_callback_url, - }, - headers={'Accept': 'application/json'}, - ) - - if response.status_code != 200: - LOG.error( - '[handle_oauth_callback] Failed to exchange code for token, ' - f'OAuth server response: status={response.status_code}, text={response.text}' - ) - raise HTTPException( - 400, 'Failed to exchange code for token: ' f'status={response.status_code}, text={response.text}' - ) - - data = response.json() - _log_debug(f'[handle_oauth_callback] OAuth server response: {data}') - - if 'error' in data: - LOG.error(f'[handle_oauth_callback] Error when exchanging code for token: data={data}') - raise HTTPException(400, data.get('error_description', data['error'])) - - redirect_uri = cast(str, state_data['redirect_uri']) - scopes = cast(list[str], state_data['scopes']) - access_token, refresh_token = self._read_oauth_tokens(data, scopes) - - # Create MCP authorization code - # This is deserialized into _ExtendedAuthorizationCode instance in load_authorization_code() function. - auth_code = { - 'code': f'mcp_{secrets.token_hex(16)}', - 'client_id': state_data['client_id'], - 'redirect_uri': redirect_uri, - 'redirect_uri_provided_explicitly': (state_data['redirect_uri_provided_explicitly'] == 'True'), - 'expires_at': int(time.time() + 5 * 60), # 5 minutes from now - 'scopes': scopes, - 'code_challenge': state_data['code_challenge'], - 'oauth_access_token': access_token.model_dump(), - 'oauth_refresh_token': refresh_token.model_dump(), - } - auth_code_jwt = self._encode(auth_code) - - mcp_redirect_uri = construct_redirect_uri( - redirect_uri_base=redirect_uri, - code=auth_code_jwt, - state=state_data['state'], - code_challenge=state_data['code_challenge'], - ) - LOG.debug(f'[handle_oauth_callback] mcp_redirect_uri={mcp_redirect_uri}') - - return mcp_redirect_uri - - async def load_authorization_code( - self, client: OAuthClientInformationFull, authorization_code: str - ) -> AuthorizationCode | None: - """ - Loads and validates the authorization code. - This function decrypts a JWT authorization code and returns an `_ExtendedAuthorizationCode` object - if the authorization code is valid. It returns `None` otherwise. - - :param client: The OAuth client details. - :param authorization_code: The JWT authorization code to be loaded and validated. - - :return: An `_ExtendedAuthorizationCode` instance if the authorization code is valid, otherwise `None`. - """ - try: - auth_code_raw = self._decode(authorization_code) - except jwt.InvalidTokenError: - LOG.debug(f'[load_authorization_code] Invalid authorization_code: {authorization_code}', exc_info=True) - return None - - auth_code = _ExtendedAuthorizationCode.model_validate( - auth_code_raw | {'redirect_uri': AnyUrl(auth_code_raw['redirect_uri'])} - ) - _log_debug( - f'[load_authorization_code] client_id={client.client_id}, authorization_code={authorization_code}, ' - f'auth_code={auth_code}' - ) - - # Log the expired authorization code. - # The mcp library itself performs the check and returns a proper response, but no logs. - now = time.time() - if auth_code.expires_at and auth_code.expires_at < now: - LOG.info( - f'[load_authorization_code] Expired authorization code: ' - f'auth_code.expires_at={auth_code.expires_at}, now={now}' - ) - - return auth_code - - async def exchange_authorization_code( - self, client: OAuthClientInformationFull, authorization_code: AuthorizationCode - ) -> OAuthToken: - """ - Swaps the authorization code for a new access and refresh tokens from the OAuth server. - The function also creates a new Storage API token for accessing the AI Service and Jobs Queue APIs. - - :param client: The OAuth client details. - :param authorization_code: The authorization code issued earlier by the `authorize()` function. - - :return: A new OAuthToken containing the access and refresh tokens. - - :raises HTTPException: If the OAuth server response indicates an error. - """ - _log_debug( - f'[exchange_authorization_code] authorization_code={authorization_code}, ' f'client_id={client.client_id}' - ) - # Check that we get the instance loaded by load_authorization_code() function. - assert isinstance(authorization_code, _ExtendedAuthorizationCode) - - expires_in = max(0, int(authorization_code.oauth_access_token.expires_at - time.time())) # seconds - sapi_token = await self._create_sapi_token( - oauth_access_token=authorization_code.oauth_access_token.token, - expires_in=self._ceil_to_hour(expires_in * 2), # twice as much as the access token's time out - ) - - # wrap the access_token from the OAuth into our own access_token - access_token = ProxyAccessToken( - token=f'mcp_{secrets.token_hex(32)}', - client_id=client.client_id, - scopes=authorization_code.scopes, - expires_at=authorization_code.oauth_access_token.expires_at, - delegate=authorization_code.oauth_access_token, - sapi_token=sapi_token, - ) - access_token_jwt = self._encode(access_token.model_dump()) - - # wrap the refresh_token from the OAuth into our own refresh_token - refresh_token = ProxyRefreshToken( - token=f'mcp_{secrets.token_hex(32)}', - client_id=client.client_id, - scopes=authorization_code.scopes, - expires_at=authorization_code.oauth_refresh_token.expires_at, - delegate=authorization_code.oauth_refresh_token, - ) - refresh_token_jwt = self._encode(refresh_token.model_dump()) - - oauth_token = OAuthToken( - access_token=access_token_jwt, - refresh_token=refresh_token_jwt, - token_type='Bearer', - expires_in=expires_in, - scope=' '.join(access_token.scopes), - ) - - _log_debug( - f'[exchange_authorization_code] access_token={access_token}, refresh_token={refresh_token},' - f'oauth_token={oauth_token}' - ) - - return oauth_token - - async def load_access_token(self, token: str) -> AccessToken | None: - """ - Loads and validates an access token. - The method decrypts a JWT access token, validates its content, and returns a `ProxyAccessToken` object - if the token is valid and not expired. Returns `None` if the token is invalid or expired. - - :param token: The JWT access token to be loaded and validated. - :return: A `ProxyAccessToken` instance if the token is valid and not expired, otherwise `None`. - """ - try: - access_token_raw = self._decode(token) - except jwt.InvalidTokenError: - LOG.debug(f'[load_access_token] Invalid token: {token}', exc_info=True) - return None - - proxy_token = ProxyAccessToken.model_validate(access_token_raw) - _log_debug(f'[load_access_token] token={token}, proxy_token={proxy_token}') - - # Log the expired authorization code. - # The mcp library itself performs the check and returns a proper response, but no logs. - now = time.time() - if proxy_token.expires_at and proxy_token.expires_at < now: - LOG.info( - f'[load_access_token] Expired access token: proxy_token.expires_at={proxy_token.expires_at}, ' - f'now={now}' - ) - - return proxy_token - - async def load_refresh_token(self, client: OAuthClientInformationFull, refresh_token: str) -> RefreshToken | None: - """ - Loads and validates a refresh token. - The method decrypts a JWT refresh token, validates its content, and returns a `RefreshToken` object - if the token is valid and not expired. Returns `None` if the token is invalid or expired. - - :param client: The OAuth client details. - :param refresh_token: A string representing the refresh token in JWT format. - :return: A `ProxyRefreshToken` instance if the token is valid and not expired, otherwise `None`. - """ - try: - refresh_token_raw = self._decode(refresh_token) - except jwt.InvalidTokenError: - LOG.debug(f'[load_refresh_token] Invalid token: {refresh_token}', exc_info=True) - return None - - proxy_token = ProxyRefreshToken.model_validate(refresh_token_raw) - _log_debug(f'[load_refresh_token] token={refresh_token}, proxy_token={proxy_token}') - - # Log the expired authorization code. - # The mcp library itself performs the check and returns a proper response, but no logs. - now = time.time() - if proxy_token.expires_at and proxy_token.expires_at < now: - LOG.info( - f'[load_refresh_token] Expired refresh token: proxy_token.expires_at={proxy_token.expires_at}, ' - f'now={now}' - ) - - return proxy_token - - async def exchange_refresh_token( - self, - client: OAuthClientInformationFull, - refresh_token: RefreshToken, - scopes: list[str], - ) -> OAuthToken: - """ - Swaps the refresh token for a new access and refresh tokens from the OAuth server. The function also creates - a new Storage API token for accessing the AI Service and Jobs Queue APIs. - - :param client: The OAuth client details. - :param refresh_token: The refresh token to use for renewing the tokens. - :param scopes: List of scopes to associate with the new tokens. If not provided, the scopes - from the original access token will be used. This can be used to reduce the scopes. - - :return: A new OAuthToken containing the access and refresh tokens. - - :raises HTTPException: If the OAuth server response indicates an error. - """ - _log_debug( - f'[exchange_refresh_token] client_id={client.client_id}, refresh_token={refresh_token}, ' f'scopes={scopes}' - ) - - assert isinstance(refresh_token, ProxyRefreshToken), f'Expected ProxyRefreshToken, got {type(refresh_token)}' - - # get new access and refresh tokens from the OAuth server - async with self._create_http_client() as http_client: - response = await http_client.post( - self._oauth_server_token_url, - data={ - 'client_id': self._oauth_client_id, - 'client_secret': self._oauth_client_secret, - 'grant_type': 'refresh_token', - 'refresh_token': refresh_token.delegate.token, - }, - headers={'Accept': 'application/json'}, - ) - - if response.status_code != 200: - LOG.exception( - '[exchange_refresh_token] Failed to refresh token, ' - f'OAuth server response: status={response.status_code}, text={response.text}' - ) - raise HTTPException( - 400, 'Failed to refresh token: ' f'status={response.status_code}, text={response.text}' - ) - - data = response.json() - _log_debug(f'[exchange_refresh_token] OAuth server response: {data}') - - if 'error' in data: - LOG.exception(f'[exchange_refresh_token] Error when refreshing token: data={data}') - raise HTTPException(400, data.get('error_description', data['error'])) - - oauth_access_token, oauth_refresh_token = self._read_oauth_tokens(data, scopes or refresh_token.scopes) - expires_in = max(0, int(oauth_access_token.expires_at - time.time())) # seconds - sapi_token = await self._create_sapi_token( - oauth_access_token=oauth_access_token.token, - expires_in=self._ceil_to_hour(expires_in * 2), # twice as much as the access token's time out - ) - - # wrap the access_token from the OAuth into our own access_token - access_token = ProxyAccessToken( - token=f'mcp_{secrets.token_hex(32)}', - client_id=client.client_id, - scopes=oauth_access_token.scopes, - expires_at=oauth_access_token.expires_at, - delegate=oauth_access_token, - sapi_token=sapi_token, - ) - access_token_jwt = self._encode(access_token.model_dump()) - - # wrap the refresh_token from the OAuth into our own refresh_token - refresh_token = ProxyRefreshToken( - token=f'mcp_{secrets.token_hex(32)}', - client_id=client.client_id, - scopes=oauth_refresh_token.scopes, - expires_at=oauth_refresh_token.expires_at, - delegate=oauth_refresh_token, - ) - refresh_token_jwt = self._encode(refresh_token.model_dump()) - - oauth_token = OAuthToken( - access_token=access_token_jwt, - refresh_token=refresh_token_jwt, - token_type='Bearer', - expires_in=max(0, int(access_token.expires_at - time.time())), - scope=' '.join(access_token.scopes), - ) - - _log_debug( - f'[exchange_refresh_token] access_token={access_token}, refresh_token={refresh_token}, ' - f'oauth_token={oauth_token}' - ) - - return oauth_token - - async def revoke_token(self, token: str, token_type_hint: str | None = None) -> None: - """ - Revokes a token. - - This is a no-op function as the tokens are not stored and so there is no way to revoke tokens that have already - been issued. - - :param token: The token to be revoked. - :param token_type_hint: An optional hint about the type of the token. - """ - _log_debug(f'[revoke_token] token={token}, token_type_hint={token_type_hint}') - # This is no-op as we don't store the tokens. - - def _read_oauth_tokens(self, data: dict[str, Any], scopes: list[str]) -> tuple[AccessToken, RefreshToken]: - """ - Reads the access and refresh tokens from the OAuth server response. - """ - expires_in = int(data['expires_in']) # seconds - if expires_in <= 0: - LOG.exception(f'[_read_oauth_tokens] Received already expired token: data={data}') - raise HTTPException(400, 'The original OAuth access token has already expired.') - - current_time = int(time.time()) - - access_token = AccessToken( - token=data['access_token'], - client_id=self._oauth_client_id, - scopes=scopes, - # this is slightly different from 'expires_at' kept by the OAuth server - expires_at=current_time + expires_in, - ) - refresh_token = RefreshToken( - token=data['refresh_token'], - client_id=self._oauth_client_id, - scopes=scopes, - # The expires_in refers to the access token. - # There is no way of knowing when the refresh token expires. - # The Keboola OAuth server issues refresh tokens that expire in 1 month and access tokens that - # expire in 1 hour. - # We derive the lifespan of a refresh token from the lifespan of an access token and make it approximately - # 1 week long under the default circumstances. - expires_at=current_time + self._ceil_to_hour(min(168 * expires_in, 168 * 3600)), - ) - - return access_token, refresh_token - - async def _create_sapi_token(self, oauth_access_token: str, expires_in: int) -> str: - """ - Creates a new Storage API token for accessing AI and Jobs Queue services that do not support bearer tokens yet. - """ - async with self._create_http_client() as http_client: - response = await http_client.post( - self._sapi_tokens_url, - json={ - 'description': 'Created by the MCP server.', - 'expiresIn': expires_in, - 'canReadAllFileUploads': True, - 'canManageBuckets': True, - }, - headers={ - 'Accept': 'application/json', - 'Authorization': f'Bearer {oauth_access_token}', - }, - ) - - if response.status_code != 200: - LOG.error( - '[_create_sapi_token] Failed to create Storage API token, ' - f'Storage API response: status={response.status_code}, text={response.text}' - ) - raise HTTPException( - response.status_code, - f'Failed to create Storage API token: status={response.status_code}, text={response.text}', - ) - - data = response.json() - _log_debug(f'[_create_sapi_token] Storage API response: {data}') - - return data['token'] - - @staticmethod - def _ceil_to_hour(seconds: int) -> int: - return math.ceil(seconds / 3600) * 3600 - - @staticmethod - def _create_http_client(): - return httpx.AsyncClient(follow_redirects=True, timeout=httpx.Timeout(30.0)) - - def _encode(self, data: Mapping[str, Any], *, key: str | None = None) -> str: - json_str = json.dumps(data) - json_bytes = json_str.encode('utf-8') - json_gzip = gzip.compress(json_bytes) - json_encrypted = jwt.api_jws.encode(json_gzip, key or self._jwt_secret) - return json_encrypted - - def _decode(self, data: str, *, key: str | None = None) -> dict[str, Any]: - json_gzip = jwt.api_jws.decode(data, key or self._jwt_secret, algorithms=['HS256']) - json_bytes = gzip.decompress(json_gzip) - json_str = json_bytes.decode('utf-8') - json_obj = json.loads(json_str) - return json_obj diff --git a/src/keboola_mcp_server/preview.py b/src/keboola_mcp_server/preview.py deleted file mode 100644 index 40e0fc3c3..000000000 --- a/src/keboola_mcp_server/preview.py +++ /dev/null @@ -1,373 +0,0 @@ -import copy -import logging -from typing import Any - -import jsonschema -import pydantic -import yaml -from pydantic import AliasChoices, BaseModel, Field, TypeAdapter, field_validator -from starlette.requests import Request -from starlette.responses import JSONResponse, Response - -from keboola_mcp_server.authorization import ToolAuthorizationMiddleware -from keboola_mcp_server.clients import KeboolaClient -from keboola_mcp_server.clients.client import DATA_APP_COMPONENT_ID, get_metadata_property -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.mcp import ServerState, SessionStateMiddleware, ToolsFilteringMiddleware -from keboola_mcp_server.tools import data_apps as data_app_tools -from keboola_mcp_server.tools.components import tools as components_tools -from keboola_mcp_server.tools.components.model import ConfigParamUpdate, TfParamUpdate -from keboola_mcp_server.tools.components.utils import get_sql_transformation_id_from_sql_dialect -from keboola_mcp_server.tools.constants import MODIFY_FLOW_TOOL_NAME, UPDATE_FLOW_TOOL_NAME -from keboola_mcp_server.tools.flow import tools as flow_tools -from keboola_mcp_server.tools.flow.scheduler_model import ScheduleRequest -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - - -class PreviewConfigDiffRq(BaseModel): - tool_name: str = Field( - validation_alias=AliasChoices('toolName', 'tool_name', 'tool-name', 'ToolName'), - serialization_alias='toolName', - ) - tool_params: dict[str, Any] = Field( - validation_alias=AliasChoices('toolParams', 'tool_params', 'tool-params', 'ToolParams'), - serialization_alias='toolParams', - ) - - -class ConfigCoordinates(BaseModel): - component_id: str | None = Field( - default=None, - validation_alias=AliasChoices('componentId', 'component_id', 'component-id', 'ComponentId'), - serialization_alias='componentId', - ) - configuration_id: str | None = Field( - default=None, - validation_alias=AliasChoices('configurationId', 'configuration_id', 'configuration-id', 'ConfigurationId'), - serialization_alias='configurationId', - ) - configuration_row_id: str | None = Field( - default=None, - validation_alias=AliasChoices( - 'configurationRowId', 'configuration_row_id', 'configuration-row-id', 'ConfigurationRowId' - ), - serialization_alias='configurationRowId', - ) - - @field_validator('component_id', 'configuration_id', 'configuration_row_id', mode='before') - @classmethod - def convert_to_string(cls, v): - """Convert any value to string, preserving None.""" - return None if v is None else str(v) - - -class PreviewConfigDiffResp(BaseModel): - coordinates: ConfigCoordinates = Field( - validation_alias=AliasChoices('coordinates', 'Coordinates'), - serialization_alias='coordinates', - ) - original_config: dict[str, Any] | None = Field( - validation_alias=AliasChoices('originalConfig', 'original_config', 'original-config', 'OriginalConfig'), - serialization_alias='originalConfig', - ) - updated_config: dict[str, Any] | None = Field( - validation_alias=AliasChoices('updatedConfig', 'updated_config', 'updated-config', 'UpdatedConfig'), - serialization_alias='updatedConfig', - ) - is_valid: bool = Field( - validation_alias=AliasChoices('isValid', 'is_valid', 'is-valid', 'IsValid'), - serialization_alias='isValid', - ) - validation_errors: list[str] | None = Field( - default=None, - validation_alias=AliasChoices('validationErrors', 'validation_errors', 'validation-errors', 'ValidationErrors'), - serialization_alias='validationErrors', - ) - - -async def _extract_coordinates( - tool_name: str, tool_params: dict[str, Any], workspace_manager: WorkspaceManager -) -> ConfigCoordinates: - """Extract configuration coordinates from tool parameters.""" - if tool_name == 'update_config': - return ConfigCoordinates( - component_id=tool_params.get('component_id'), - configuration_id=tool_params.get('configuration_id'), - ) - elif tool_name == 'update_config_row': - return ConfigCoordinates( - component_id=tool_params.get('component_id'), - configuration_id=tool_params.get('configuration_id'), - configuration_row_id=tool_params.get('configuration_row_id'), - ) - elif tool_name == 'update_sql_transformation': - sql_dialect = await workspace_manager.get_sql_dialect() - return ConfigCoordinates( - component_id=get_sql_transformation_id_from_sql_dialect(sql_dialect), - configuration_id=tool_params.get('configuration_id'), - ) - elif tool_name in {UPDATE_FLOW_TOOL_NAME, MODIFY_FLOW_TOOL_NAME}: - return ConfigCoordinates( - component_id=tool_params.get('flow_type'), - configuration_id=tool_params.get('configuration_id'), - ) - elif tool_name == 'modify_streamlit_data_app': - return ConfigCoordinates( - component_id=DATA_APP_COMPONENT_ID, - configuration_id=tool_params.get('configuration_id'), - ) - else: - raise ValueError(f'Invalid tool name: "{tool_name}"') - - -async def _validate_tool_params( - tool_name: str, - tool_params: dict[str, Any], - tool_input_schema: dict[str, Any], -) -> tuple[bool, str | None]: - """ - Validate tool parameters against the tool's input schema using JSON schema validation. - - This validates the parameters without executing the tool function. - - :param tool_name: Name of the MCP tool to validate against - :param tool_params: Parameters to validate (raw user-provided params) - :param tool_input_schema: - :return: Tuple of (is_valid, validation_errors) - - is_valid: True if validation passed, False otherwise - - validation_errors: List of error messages if validation failed, None if successful - """ - try: - jsonschema.validate(instance=tool_params, schema=tool_input_schema) - return True, None - - except jsonschema.ValidationError as e: - # Format a validation error message similarly to errors.prettify_validation_error() function - header = f'Found 1 validation error for {tool_name}:' - formatted = { - 'errors': [ - { - 'field': '.'.join(str(p) for p in e.path or []), - 'message': e.message, - 'extra': { - 'schema': e.schema, - }, - } - ] - } - try: - yaml_str = yaml.dump(formatted, default_flow_style=False, sort_keys=False, allow_unicode=True) - except Exception: - yaml_str = str(formatted) - - return False, f'{header}\n{yaml_str}' - - except jsonschema.SchemaError as e: - # Schema itself is invalid - LOG.exception(f"[validate_tool_params] Invalid schema for tool '{tool_name}': {e}") - return False, 'Internal error: Invalid tool schema' - - except Exception as e: - # Handle other unexpected errors - LOG.exception(f'[validate_tool_params] Unexpected error: {e}') - return False, f'Validation error: {str(e)}' - - -def _prepare_mutator( - preview_rq: PreviewConfigDiffRq, client: KeboolaClient, workspace_manager: WorkspaceManager -) -> tuple[Any, dict[str, Any]]: - """ - Prepare mutator function and parameters for config diff preview. - - :param preview_rq: PreviewConfigDiffRq object containing tool parameters and name. - :param client: KeboolaClient instance for API operations. - :param workspace_manager: WorkspaceManager instance for workspace operations. - :return: Tuple containing mutator function and parameters. - """ - mutator_params: dict[str, Any] = { - **preview_rq.tool_params, - 'client': client, - } - - if preview_rq.tool_name == 'update_config': - folder = mutator_params.pop( - 'folder', None - ) # folder is metadata-only; update_config_internal does not accept it - if parameter_updates := mutator_params.get('parameter_updates'): - type_adapter = TypeAdapter(list[ConfigParamUpdate]) - mutator_params['parameter_updates'] = type_adapter.validate_python(parameter_updates) - - if folder is None: - mutator_fn = components_tools.update_config_internal - else: - _folder = folder - - async def _update_config_with_folder_preview(**kwargs: Any) -> tuple: - orig, updated = await components_tools.update_config_internal(**kwargs) - normalized = _folder.strip() - folder_preview: dict | None = None - try: - meta = await client.storage_client.configuration_metadata_get( - component_id=kwargs['component_id'], - configuration_id=kwargs['configuration_id'], - ) - current = get_metadata_property(meta, MetadataField.CONFIGURATION_FOLDER_NAME, default='') or '' - if normalized != current: - folder_preview = {'original_folder': current, 'updated_folder': normalized} - except Exception as e: - LOG.warning( - 'Failed to fetch configuration metadata for folder preview ' - '(component_id=%s, configuration_id=%s): %s. Proceeding without folder preview.', - kwargs['component_id'], - kwargs['configuration_id'], - e, - ) - return orig, updated, folder_preview - - mutator_fn = _update_config_with_folder_preview - - elif preview_rq.tool_name == 'update_config_row': - mutator_fn = components_tools.update_config_row_internal - if parameter_updates := mutator_params.get('parameter_updates'): - type_adapter = TypeAdapter(list[ConfigParamUpdate]) - mutator_params['parameter_updates'] = type_adapter.validate_python(parameter_updates) - - elif preview_rq.tool_name == 'update_sql_transformation': - mutator_fn = components_tools.update_sql_transformation_internal - mutator_params['workspace_manager'] = workspace_manager - if parameter_updates := mutator_params.get('parameter_updates'): - type_adapter = TypeAdapter(list[TfParamUpdate]) - mutator_params['parameter_updates'] = type_adapter.validate_python(parameter_updates) - - elif preview_rq.tool_name in {UPDATE_FLOW_TOOL_NAME, MODIFY_FLOW_TOOL_NAME}: - mutator_fn = flow_tools.update_flow_internal - if schedules := mutator_params.get('schedules'): - type_adapter = TypeAdapter(list[ScheduleRequest]) - mutator_params['schedules'] = type_adapter.validate_python(schedules) - - elif preview_rq.tool_name == 'modify_streamlit_data_app': - mutator_fn = data_app_tools.modify_streamlit_data_app_internal - mutator_params['workspace_manager'] = workspace_manager - - else: - raise ValueError(f'Invalid tool name: "{preview_rq.tool_name}"') - - return mutator_fn, mutator_params - - -async def preview_config_diff(rq: Request) -> Response: - preview_rq = PreviewConfigDiffRq.model_validate(await rq.json()) - - # This route is a raw Starlette route outside the FastMCP middleware chain, so the tool - # authorization that ToolAuthorizationMiddleware applies to MCP tool calls does not run here. - # Enforce the same X-Allowed-Tools / X-Disallowed-Tools / X-Read-Only-Mode headers explicitly - # so a restricted client cannot drive the mutator-preview path for a tool it cannot call. - read_only_tools = getattr(rq.app.state, 'mcp_read_only_tools', set()) - is_read_only = preview_rq.tool_name in read_only_tools - allowed, disallowed, read_only_mode = ToolAuthorizationMiddleware._get_authorization_config(rq) - if not ToolAuthorizationMiddleware._is_tool_name_authorized( - preview_rq.tool_name, is_read_only, allowed, disallowed, read_only_mode - ): - LOG.info(f'[preview_config_diff] Tool authorization denied (headers): {preview_rq.tool_name}') - return JSONResponse( - status_code=403, - content={'message': f'The tool "{preview_rq.tool_name}" is not authorized for this client.'}, - ) - - # Log only non-sensitive metadata; tool_params can carry user-supplied secrets. - LOG.info(f'[preview_config_diff] tool_name={preview_rq.tool_name} param_keys={sorted(preview_rq.tool_params)}') - - server_state = ServerState.from_starlette(rq.app) - config = SessionStateMiddleware.apply_request_config(rq, server_state.config) - state = await SessionStateMiddleware.create_session_state(config, server_state.runtime_info, readonly=True) - client = KeboolaClient.from_state(state) - workspace_manager = WorkspaceManager.from_state(state) - - # Apply the same project-feature / token-role / branch gating that ToolsFilteringMiddleware applies - # to MCP tool calls. Header authorization above does not cover these, so without this a caller could - # preview e.g. a data-app tool on a non-main branch, or a write tool with a read-only token. - semantic_tools = getattr(rq.app.state, 'mcp_semantic_tools', set()) - token_info = await client.storage_client.verify_token() - denial = ToolsFilteringMiddleware.authorize_tool_call( - tool_name=preview_rq.tool_name, - is_read_only=is_read_only, - is_semantic=preview_rq.tool_name in semantic_tools, - token_role=ToolsFilteringMiddleware.get_token_role(token_info), - features=ToolsFilteringMiddleware.get_project_features(token_info), - is_oauth=bool(client.bearer_token), - is_main_branch=client.branch_id is None, - ) - if denial: - LOG.info(f'[preview_config_diff] Tool authorization denied (project/role/branch): {preview_rq.tool_name}') - return JSONResponse(status_code=403, content={'message': denial}) - - coordinates = await _extract_coordinates(preview_rq.tool_name, preview_rq.tool_params, workspace_manager) - - if tool_input_schema := rq.app.state.mcp_tools_input_schema.get(preview_rq.tool_name): - is_valid, validation_errors = await _validate_tool_params( - tool_name=preview_rq.tool_name, - tool_params=preview_rq.tool_params, - tool_input_schema=tool_input_schema, - ) - - if not is_valid: - preview_resp = PreviewConfigDiffResp( - coordinates=coordinates, - original_config={}, - updated_config={}, - is_valid=False, - validation_errors=[validation_errors], - ) - return JSONResponse(preview_resp.model_dump(by_alias=True, exclude_none=True)) - else: - LOG.warning(f'[preview_config_diff] No input schema found for tool "{preview_rq.tool_name}"') - - mutator_fn, mutator_params = _prepare_mutator(preview_rq, client, workspace_manager) - - try: - original_config, new_config, *mutator_preview = await mutator_fn(**mutator_params) - mutator_preview = next((x for x in mutator_preview if isinstance(x, dict)), None) - if isinstance(original_config, BaseModel): - original_config = original_config.model_dump() - - updated_config = copy.deepcopy(original_config) - updated_config['configuration'] = new_config - if name := preview_rq.tool_params.get('name'): - updated_config['name'] = name - description = preview_rq.tool_params.get('description') - if description: - updated_config['description'] = description - if (is_disabled := preview_rq.tool_params.get('is_disabled')) is not None: - updated_config['isDisabled'] = is_disabled - if change_description := preview_rq.tool_params.get('change_description'): - updated_config['changeDescription'] = change_description - if mutator_preview is not None and isinstance(mutator_preview, dict): - for key, value in mutator_preview.items(): - if key.startswith('original_'): - original_config[key.replace('original_', '')] = value - elif key.startswith('updated_'): - updated_config[key.replace('updated_', '')] = value - else: - raise ValueError(f'Invalid mutator preview key: "{key}"') - - preview_resp = PreviewConfigDiffResp( - coordinates=coordinates, - original_config=original_config, - updated_config=updated_config, - is_valid=True, - validation_errors=None, - ) - - except (pydantic.ValidationError, jsonschema.ValidationError, ValueError) as ex: - LOG.exception(f'[preview_config_diff] {ex}') - preview_resp = PreviewConfigDiffResp( - coordinates=coordinates, - original_config={}, - updated_config={}, - is_valid=False, - validation_errors=[str(ex)], - ) - - return JSONResponse(preview_resp.model_dump(by_alias=True, exclude_none=True)) diff --git a/src/keboola_mcp_server/prompts/__init__.py b/src/keboola_mcp_server/prompts/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/keboola_mcp_server/prompts/add_prompts.py b/src/keboola_mcp_server/prompts/add_prompts.py deleted file mode 100644 index c9694876f..000000000 --- a/src/keboola_mcp_server/prompts/add_prompts.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Module to add prompts to the Keboola MCP server.""" - -from fastmcp.prompts import Prompt - -from keboola_mcp_server.mcp import KeboolaMcpServer - - -def add_keboola_prompts(mcp: KeboolaMcpServer) -> None: - """Add all Keboola-specific prompts to the MCP server. - - The prompt names and descriptions are automatically derived from the function - names and docstrings. - """ - # Import the prompt functions here to avoid circular imports - from keboola_mcp_server.prompts.keboola_prompts import ( - analyze_project_structure, - component_usage_summary, - create_project_documentation, - data_quality_assessment, - error_analysis_report, - project_health_check, - ) - - # ONE-CLICK PROMPTS (no required parameters) - # Add project analysis prompt - mcp.add_prompt(Prompt.from_function(analyze_project_structure)) - - # Add project health check prompt - mcp.add_prompt(Prompt.from_function(project_health_check)) - - # Add data quality assessment prompt - mcp.add_prompt(Prompt.from_function(data_quality_assessment)) - - # Add component usage summary prompt - mcp.add_prompt(Prompt.from_function(component_usage_summary)) - - # Add error analysis report prompt - mcp.add_prompt(Prompt.from_function(error_analysis_report)) - - # Add documentation generator prompt - mcp.add_prompt(Prompt.from_function(create_project_documentation)) diff --git a/src/keboola_mcp_server/prompts/keboola_prompts.py b/src/keboola_mcp_server/prompts/keboola_prompts.py deleted file mode 100644 index 2360f4f38..000000000 --- a/src/keboola_mcp_server/prompts/keboola_prompts.py +++ /dev/null @@ -1,676 +0,0 @@ -"""Keboola-specific prompts for the MCP server.""" - -from typing import List - -from fastmcp.prompts import Message - - -async def analyze_project_structure() -> List[Message]: - """Generate a comprehensive analysis prompt for a Keboola project's structure. - - This prompt analyzes the project's components, data flow, buckets, tables, - and configurations to provide insights into capabilities and applications. - """ - return [ - Message( - role='user', - content="""Based on the components that are being used and the data available from all -of the buckets in the project, give me a high-level understanding of what is going on inside -of this project and the types of use cases that are being performed. - -**Analysis Requirements:** -Highlight the key functionalities being implemented, emphasizing the project's -capability to address specific problems or tasks. Explore the range of use cases the -project is designed for, detailing examples of real-world scenarios it can handle. Be sure to also include -the names of real example buckets, tables & configurations that are within the project. - -**Structure your output in the following format:** - -## High-level Summary -• Bullet-point summary of the activities and use cases being performed - -## Data Sources & Integrations -• List all data sources and external integrations -• Include specific extractor components and their configurations -• Mention connection types and data refresh patterns - -## Data Processing & Transformation -• Detail transformation workflows and SQL logic -• Highlight data cleaning, enrichment, and aggregation processes -• Include specific transformation component names and examples - -## Data Storage & Management -• Describe bucket organization and table structures -• Include real bucket and table names from the project -• Explain data retention and archival strategies - -## Use Cases -• Identify specific business use cases and scenarios -• Provide real-world examples the project can handle -• Connect technical capabilities to business outcomes - -Please provide a comprehensive analysis with specific examples and names from the actual project data.""", - ) - ] - - -async def project_health_check() -> List[Message]: - """Generate a comprehensive health check analysis for the entire Keboola project. - - This one-click prompt analyzes project health, identifies issues, and provides recommendations. - """ - return [ - Message( - role='user', - content="""Perform a comprehensive health check of this Keboola project and identify -any issues, risks, or optimization opportunities. - -**Health Check Areas:** - -## 1. Component Health -• Analyze all components for errors, warnings, or performance issues -• Check component configurations for best practices -• Identify unused or redundant components -• Review component update status and versions - -## 2. Data Quality Assessment -• Examine tables for data completeness and consistency -• Identify tables with potential data quality issues -• Check for empty tables or tables with unusual patterns -• Analyze data freshness and update frequencies - -## 3. Performance Analysis -• Identify slow-running transformations or jobs -• Check for resource-intensive operations -• Analyze job execution patterns and bottlenecks -• Review storage usage and optimization opportunities - -## 4. Security & Access Review -• Review bucket and table permissions -• Check for potential security vulnerabilities -• Analyze token usage and access patterns -• Identify overprivileged configurations - -## 5. Cost Optimization -• Identify cost optimization opportunities -• Review storage usage and retention policies -• Analyze job execution efficiency -• Suggest resource optimization strategies - -## 6. Recommendations -• Prioritized list of issues to address -• Quick wins for immediate improvement -• Long-term optimization strategies -• Best practices implementation suggestions - -Please provide specific findings with component and table names and actionable recommendations.""", - ) - ] - - -async def data_quality_assessment() -> List[Message]: - """Generate a comprehensive data quality assessment for all project data. - - One-click analysis of data quality across all buckets and tables. - """ - return [ - Message( - role='user', - content="""Conduct a comprehensive data quality assessment across all data in this Keboola project. - -**Data Quality Analysis:** - -## 1. Completeness Analysis -• Identify tables with missing or null values -• Calculate completeness percentages for key columns -• Flag tables with significant data gaps -• Analyze data volume trends and anomalies - -## 2. Consistency Checks -• Check for data format inconsistencies -• Identify duplicate records across tables -• Analyze referential integrity between related tables -• Flag inconsistent naming conventions - -## 3. Accuracy Assessment -• Identify potential data accuracy issues -• Check for outliers and anomalous values -• Analyze data validation patterns -• Review data transformation logic for accuracy - -## 4. Timeliness Evaluation -• Assess data freshness across all tables -• Identify stale or outdated data -• Review data update frequencies -• Flag tables with irregular update patterns - -## 5. Data Profiling Summary -• Statistical overview of each table -• Data type distribution and usage -• Value distribution analysis -• Schema evolution and changes - -## 6. Quality Scores & Recommendations -• Overall quality score for each table -• Prioritized list of data quality issues -• Specific improvement recommendations -• Data governance suggestions - -Please analyze the actual project data and provide specific findings with table names, -metrics, and actionable recommendations.""", - ) - ] - - -async def security_audit() -> List[Message]: - """Generate a security audit for the Keboola project. - - One-click security assessment covering permissions, access, and best practices. - """ - return [ - Message( - role='user', - content="""Perform a comprehensive security audit of this Keboola project to -identify potential vulnerabilities and security best practice violations. - -**Security Audit Areas:** - -## 1. Access Control Review -• Analyze bucket and table permissions -• Identify overly permissive access settings -• Review token usage and scopes -• Check for unused or stale access credentials - -## 2. Data Privacy Assessment -• Identify tables containing sensitive or PII data -• Review data encryption and protection measures -• Check for proper data masking in non-production environments -• Analyze data retention and deletion policies - -## 3. Component Security -• Review component configurations for security issues -• Check for hardcoded credentials or sensitive information -• Analyze external connection security -• Verify secure communication protocols - -## 4. Compliance Check -• Review adherence to data governance policies -• Check for GDPR/data protection compliance -• Analyze audit trail and logging capabilities -• Verify backup and disaster recovery measures - -## 5. Network & Infrastructure Security -• Review API access patterns and restrictions -• Check for suspicious or anomalous access attempts -• Analyze IP whitelisting and access controls -• Review integration security with external systems - -## 6. Security Recommendations -• Critical security issues requiring immediate attention -• Medium-priority security improvements -• Security best practices implementation -• Compliance enhancement suggestions - -Please provide specific findings with component and bucket names and prioritized security recommendations.""", - ) - ] - - -async def performance_optimization_analysis() -> List[Message]: - """Generate a performance analysis and optimization recommendations. - - One-click performance audit identifying bottlenecks and optimization opportunities. - """ - return [ - Message( - role='user', - content="""Analyze the performance characteristics of this Keboola project and -identify optimization opportunities. - -**Performance Analysis Areas:** - -## 1. Job Execution Performance -• Identify slow-running transformations and extractions -• Analyze job execution patterns and frequencies -• Check for failed or frequently retried jobs -• Review job queue efficiency and resource utilization - -## 2. SQL Query Optimization -• Analyze transformation SQL for performance issues -• Identify queries with potential optimization opportunities -• Check for inefficient joins, subqueries, or aggregations -• Review indexing strategies and table design - -## 3. Data Pipeline Efficiency -• Analyze end-to-end pipeline execution times -• Identify bottlenecks in data flow -• Review parallel processing opportunities -• Check for unnecessary data movement or duplication - -## 4. Storage Optimization -• Analyze table sizes and growth patterns -• Identify opportunities for data archiving or compression -• Review partitioning and clustering strategies -• Check for unused or redundant data storage - -## 5. Resource Utilization -• Review compute resource allocation and usage -• Analyze memory and processing requirements -• Check for resource contention or conflicts -• Identify cost-performance optimization opportunities - -## 6. Optimization Recommendations -• High-impact performance improvements -• Quick wins for immediate performance gains -• Long-term optimization strategies -• Resource allocation recommendations - -Please analyze actual project performance data and provide specific -recommendations with component names and expected performance improvements.""", - ) - ] - - -async def component_usage_summary() -> List[Message]: - """Generate a comprehensive summary of all components and their usage patterns. - - One-click overview of project components, configurations, and usage analytics. - """ - return [ - Message( - role='user', - content="""Generate a comprehensive summary of all components in this Keboola -project, their configurations, and usage patterns. - -**Component Analysis:** - -## 1. Component Inventory -• Complete list of all components by type (extractors, transformations, writers) -• Component versions and update status -• Configuration count per component -• Active vs inactive component status - -## 2. Usage Analytics -• Job execution frequency per component -• Success/failure rates and reliability metrics -• Resource consumption patterns -• Peak usage times and scheduling analysis - -## 3. Configuration Analysis -• Number of configurations per component -• Configuration complexity and parameter usage -• Shared vs component-specific configurations -• Configuration change history and evolution - -## 4. Data Flow Mapping -• Input and output relationships between components -• Data dependencies and lineage -• Critical path analysis in data pipelines -• Component interdependency mapping - -## 5. Health & Status Overview -• Component error rates and common issues -• Performance metrics and execution times -• Maintenance and update requirements -• Deprecated or outdated component usage - -## 6. Optimization Opportunities -• Underutilized or redundant components -• Configuration consolidation opportunities -• Component upgrade recommendations -• Efficiency improvement suggestions - -Please provide specific details including component names, configuration IDs, and -actionable insights for project optimization.""", - ) - ] - - -async def error_analysis_report() -> List[Message]: - """Generate an analysis of recent errors and failures across the project. - - One-click error analysis with troubleshooting recommendations. - """ - return [ - Message( - role='user', - content="""Analyze recent errors and failures across this Keboola project and -provide troubleshooting recommendations. - -**Error Analysis:** - -## 1. Error Frequency & Patterns -• Most common error types across all components -• Error frequency trends over time -• Components with highest failure rates -• Recurring vs one-time error patterns - -## 2. Critical Errors -• High-priority errors affecting data pipelines -• Errors causing data quality issues -• Security-related errors or warnings -• Errors impacting business-critical processes - -## 3. Component-Specific Issues -• Transformation errors and SQL issues -• Extractor connection and authentication problems -• Writer destination errors and data delivery failures -• Orchestration and scheduling conflicts - -## 4. Root Cause Analysis -• Infrastructure vs configuration-related errors -• Data-related errors (missing files, schema changes) -• Permission and access-related issues -• External service dependency failures - -## 5. Impact Assessment -• Business impact of each error category -• Data pipeline disruption analysis -• SLA and delivery timeline impacts -• Downstream system effect analysis - -## 6. Resolution Recommendations -• Immediate fixes for critical errors -• Preventive measures for recurring issues -• Configuration improvements to reduce errors -• Monitoring and alerting enhancements - -Please analyze actual error logs and job histories to provide specific error -instances with component names and detailed troubleshooting guidance.""", - ) - ] - - -async def create_project_documentation() -> List[Message]: - """Generate comprehensive project documentation automatically. - - One-click documentation creation for the entire Keboola project. - """ - return [ - Message( - role='user', - content="""Generate comprehensive, professional documentation for this Keboola -project that can be used for onboarding, maintenance, and knowledge sharing. - -**Documentation Structure:** - -## 1. Project Overview -• Executive summary of project purpose and objectives -• Key stakeholders and business owners -• Project scope and data processing capabilities -• Success metrics and KPIs - -## 2. Architecture Documentation -• High-level system architecture diagram description -• Data flow and pipeline overview -• Component interaction and dependencies -• Technical infrastructure and requirements - -## 3. Data Dictionary -• Complete inventory of all buckets and tables with names -• Column definitions and business meanings -• Data types, constraints, and validation rules -• Data lineage and source system mappings - -## 4. Component Documentation -• Detailed description of each component and its purpose -• Configuration parameters and their meanings -• Input/output specifications -• Business logic and transformation rules - -## 5. Operational Procedures -• Data pipeline monitoring and maintenance procedures -• Error handling and troubleshooting guides -• Backup and disaster recovery processes -• Change management and deployment procedures - -## 6. User Guides -• End-user access and data consumption guides -• Report and dashboard usage instructions -• Data quality and validation procedures -• FAQ and common troubleshooting scenarios - -## 7. Technical Reference -• API endpoints and integration specifications -• Security and access control documentation -• Performance tuning and optimization guides -• Development and testing procedures - -Please create detailed, professional documentation using actual project data -including specific names, configurations, and real examples.""", - ) - ] - - -async def generate_project_descriptions( - focus_area: str = 'all', include_technical_details: bool = True -) -> List[Message]: - """Generate comprehensive descriptions for all tables and buckets in a Keboola project. - - The focus can be on buckets, tables, or all components. Technical details such as - schema information and metadata can be optionally included. - """ - technical_section = '' - if include_technical_details: - technical_section = """ -## Technical Details (for each item) -• Schema information and column definitions -• Data types and constraints -• Row counts and data volume metrics -• Last update timestamps and refresh patterns""" - - focus_instruction = { - 'buckets': 'Focus specifically on bucket-level descriptions and organization.', - 'tables': 'Focus specifically on table-level descriptions and data structures.', - 'all': 'Provide comprehensive descriptions for both buckets and tables.', - }.get(focus_area, 'Provide comprehensive descriptions for both buckets and tables.') - - # Pre-calculate conditional sections to avoid long lines - bucket_tech_section = technical_section if focus_area in ['buckets', 'all'] else '' - table_tech_section = technical_section if focus_area in ['tables', 'all'] else '' - - return [ - Message( - role='user', - content=f"""Generate comprehensive, business-friendly descriptions for all tables and buckets -in this Keboola project. -{focus_instruction} - -**Requirements:** -Create clear, informative descriptions that help users understand: -1. What data each bucket/table contains -2. The business purpose and use cases -3. Data lineage and relationships -4. Quality and completeness indicators - -**Structure your output as follows:** - -## Bucket Descriptions -For each bucket, provide: -• **Bucket Name**: [bucket.name] -• **Purpose**: Business purpose and data category -• **Contents**: Types of tables and data contained -• **Use Cases**: How this data is typically used -• **Data Sources**: Where the data originates from{bucket_tech_section} - -## Table Descriptions -For each table, provide: -• **Table Name**: [bucket.table] -• **Description**: Clear business description of the data -• **Key Columns**: Most important fields and their meanings -• **Data Quality**: Completeness, accuracy, and freshness indicators -• **Relationships**: How it connects to other tables -• **Business Value**: Why this data matters and how it's used{table_tech_section} - -## Summary -• Overall data architecture insights -• Recommendations for improving descriptions -• Suggestions for better data organization - -Please analyze the actual project data and provide specific, actionable descriptions for each component.""", - ) - ] - - -async def debug_transformation(transformation_name: str) -> List[Message]: - """Generate a prompt to help debug a specific transformation. - - Provides debugging assistance for transformation logic, SQL errors, performance - problems, and optimization strategies. - """ - return [ - Message( - role='user', - content=f"""I need help debugging a Keboola transformation called "{transformation_name}". - -Please help me: -1. Identify potential issues in the transformation logic -2. Check for common SQL errors or performance problems -3. Suggest optimization strategies -4. Recommend debugging approaches -5. Provide best practices for transformation development - -What specific information would you need to effectively debug this transformation?""", - ) - ] - - -async def create_data_pipeline_plan( - source_description: str, target_description: str, requirements: str = '' -) -> List[Message]: - """Generate a prompt to create a data pipeline plan. - - Creates a comprehensive data pipeline design based on source and target specifications - with optional additional requirements. - """ - requirements_text = f'\n\nAdditional requirements:\n{requirements}' if requirements else '' - - return [ - Message( - role='user', - content=f"""I need to create a data pipeline in Keboola Connection with the following specifications: - -**Source:** {source_description} -**Target:** {target_description}{requirements_text} - -Please help me design a comprehensive data pipeline plan that includes: - -1. **Data Extraction Strategy** - - Recommended extractors or data sources - - Connection configuration considerations - - Data refresh frequency recommendations - -2. **Data Transformation Plan** - - Required data cleaning and preparation steps - - Transformation logic and SQL queries - - Data quality checks and validation - -3. **Data Loading Strategy** - - Target storage configuration - - Output format and structure - - Performance optimization considerations - -4. **Orchestration and Monitoring** - - Recommended orchestration flow - - Error handling and alerting - - Monitoring and logging strategies - -5. **Best Practices** - - Security considerations - - Scalability recommendations - - Maintenance and documentation - -Please provide a detailed, step-by-step implementation plan with specific Keboola components and configurations.""", - ) - ] - - -async def optimize_sql_query(sql_query: str, context: str = '') -> List[Message]: - """Generate a prompt to optimize an SQL query for Keboola transformations. - - Analyzes the provided SQL query and suggests performance optimizations, - best practices, and alternative approaches. - """ - context_text = f'\n\nContext: {context}' if context else '' - - return [ - Message( - role='user', - content=f"""Please analyze and optimize this SQL query for use in a Keboola transformation:{context_text} - -```sql -{sql_query} -``` - -I need help with: - -1. **Performance Optimization** - - Identify potential bottlenecks - - Suggest indexing strategies - - Recommend query restructuring - -2. **Best Practices** - - Code readability and maintainability - - Keboola-specific optimizations - - Resource usage efficiency - -3. **Error Prevention** - - Common pitfalls to avoid - - Data type considerations - - Null handling improvements - -4. **Alternative Approaches** - - Different ways to achieve the same result - - Trade-offs between approaches - - Scalability considerations - -Please provide the optimized query with explanations for each improvement.""", - ) - ] - - -async def troubleshoot_component_error( - component_name: str, error_message: str, component_type: str = 'unknown' -) -> List[Message]: - """Generate a prompt to troubleshoot a component error. - - Provides comprehensive troubleshooting guidance for component errors including - diagnosis, solutions, and prevention strategies. - """ - return [ - Message( - role='user', - content=f"""I'm experiencing an error with a Keboola component and need troubleshooting help: - -**Component:** {component_name} -**Type:** {component_type} -**Error Message:** -``` -{error_message} -``` - -Please help me: - -1. **Diagnose the Issue** - - Interpret the error message - - Identify the root cause - - Determine if it's a configuration, data, or system issue - -2. **Provide Solutions** - - Step-by-step troubleshooting guide - - Configuration fixes or adjustments - - Alternative approaches if needed - -3. **Prevention Strategies** - - How to avoid this error in the future - - Best practices for component configuration - - Monitoring and alerting recommendations - -4. **Additional Investigation** - - What additional information might be needed - - Logs or metrics to check - - Related components that might be affected - -Please provide a comprehensive troubleshooting guide with specific actions I can take.""", - ) - ] diff --git a/src/keboola_mcp_server/resources/__init__.py b/src/keboola_mcp_server/resources/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/keboola_mcp_server/resources/data_app/__init__.py b/src/keboola_mcp_server/resources/data_app/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/keboola_mcp_server/resources/prompts/__init__.py b/src/keboola_mcp_server/resources/prompts/__init__.py deleted file mode 100644 index 6301b8397..000000000 --- a/src/keboola_mcp_server/resources/prompts/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -import logging -from importlib import resources - -LOG = logging.getLogger(__name__) - -_DIALECT_CONFIGS: dict[str, dict] = { - 'BigQuery': { - 'delimiter': 'backtick (`` ` ``)', - 'col': '`column_name`', - 'fqn': '`project`.`dataset`.`table`', - 'new_table': '`table_name`', - 'extra': [], - }, - 'Snowflake': { - 'delimiter': 'double quote (`"`)', - 'col': '"column_name"', - 'fqn': '"DATABASE"."SCHEMA"."TABLE"', - 'new_table': '"table_name"', - 'extra': [ - 'Unquoted identifiers and column aliases are auto-uppercased by Snowflake — ' - 'always use delimited identifiers to preserve case.', - 'Use `LISTAGG` instead of `STRING_AGG`.', - 'In CTEs, use delimited identifiers for every column alias so the name survives ' - 'into the outer query unchanged.', - ], - }, -} - - -def _build_dialect_section(sql_dialect: str) -> str: - cfg = _DIALECT_CONFIGS.get(sql_dialect) - if not cfg: - LOG.warning('Unknown SQL dialect %r — no dialect-specific identifier guidance will be emitted.', sql_dialect) - return f'### SQL Identifiers\n\nSQL dialect: **{sql_dialect}**.\n' - lines = [ - '### SQL Identifiers\n', - f'This project uses **{sql_dialect}** SQL dialect.', - f'The delimited identifier character is the {cfg["delimiter"]}.', - '**Always wrap every identifier** (column name, table name, alias) ' 'in delimited identifiers:\n', - f'- Column reference: {cfg["col"]}', - f'- Fully qualified table name: {cfg["fqn"]}', - f'- New table in CREATE TABLE (table name only, no FQN): {cfg["new_table"]}', - '- Never mix delimiter styles within a single query.\n', - ] - for note in cfg['extra']: - lines.append(f'- {note}') - return '\n'.join(lines) - - -def load_prompt(name: str) -> str: - return resources.files(__package__).joinpath(name).read_text(encoding='utf-8') - - -def get_project_system_prompt(sql_dialect: str = '') -> str: - base = load_prompt('project_system_prompt.md') - if not sql_dialect: - return base - return _build_dialect_section(sql_dialect) + '\n\n---\n\n' + base diff --git a/src/keboola_mcp_server/server.py b/src/keboola_mcp_server/server.py deleted file mode 100644 index f0494030d..000000000 --- a/src/keboola_mcp_server/server.py +++ /dev/null @@ -1,259 +0,0 @@ -"""MCP server implementation for Keboola Connection.""" - -import dataclasses -import logging -import os -from collections.abc import AsyncIterator -from contextlib import AbstractAsyncContextManager, asynccontextmanager -from typing import Callable, Literal - -from fastmcp import FastMCP -from fastmcp.server.middleware.logging import LoggingMiddleware -from pydantic import AliasChoices, BaseModel, Field -from starlette.applications import Starlette -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.responses import JSONResponse, RedirectResponse, Response - -from keboola_mcp_server.authorization import ToolAuthorizationMiddleware -from keboola_mcp_server.config import Config, ServerRuntimeInfo, Transport -from keboola_mcp_server.errors import ValidationErrorMiddleware -from keboola_mcp_server.mcp import ( - KeboolaMcpServer, - ServerState, - SessionStateMiddleware, - ToolsFilteringMiddleware, -) -from keboola_mcp_server.oauth import SimpleOAuthProvider -from keboola_mcp_server.preview import preview_config_diff -from keboola_mcp_server.prompts.add_prompts import add_keboola_prompts -from keboola_mcp_server.tools.components.tools import add_component_tools -from keboola_mcp_server.tools.data_apps import add_data_app_tools -from keboola_mcp_server.tools.doc import add_doc_tools -from keboola_mcp_server.tools.flow.tools import add_flow_tools -from keboola_mcp_server.tools.jobs import add_job_tools -from keboola_mcp_server.tools.oauth import add_oauth_tools -from keboola_mcp_server.tools.project import add_project_tools -from keboola_mcp_server.tools.search import add_search_tools -from keboola_mcp_server.tools.semantic import add_semantic_tools -from keboola_mcp_server.tools.sql import add_sql_tools -from keboola_mcp_server.tools.storage import add_storage_tools - -LOG = logging.getLogger(__name__) - - -class StatusApiResp(BaseModel): - status: str - - -class ServiceInfoApiResp(BaseModel): - app_name: str = Field( - default='KeboolaMcpServer', - validation_alias=AliasChoices('appName', 'app_name', 'app-name'), - serialization_alias='appName', - ) - app_version: str = Field( - validation_alias=AliasChoices('appVersion', 'app_version', 'app-version'), serialization_alias='appVersion' - ) - server_version: str = Field( - validation_alias=AliasChoices('serverVersion', 'server_version', 'server-version'), - serialization_alias='serverVersion', - ) - mcp_library_version: str = Field( - validation_alias=AliasChoices('mcpLibraryVersion', 'mcp_library_version', 'mcp-library-version'), - serialization_alias='mcpLibraryVersion', - ) - fastmcp_library_version: str = Field( - validation_alias=AliasChoices('fastmcpLibraryVersion', 'fastmcp_library_version', 'fastmcp-library-version'), - serialization_alias='fastmcpLibraryVersion', - ) - server_transport: Transport | None = Field( - validation_alias=AliasChoices('serverTransport', 'server_transport', 'server-transport'), - serialization_alias='serverTransport', - default=None, - ) - server_id: str = Field( - validation_alias=AliasChoices('serverId', 'server_id', 'server-id'), - serialization_alias='serverId', - ) - - -def create_keboola_lifespan( - server_state: ServerState, -) -> Callable[[FastMCP[ServerState]], AbstractAsyncContextManager[ServerState]]: - @asynccontextmanager - async def keboola_lifespan(server: FastMCP) -> AsyncIterator[ServerState]: - """ - Manage Keboola server lifecycle - - This method is called when the server starts, initializes the server state and returns it within a - context manager. The lifespan state is accessible across the whole server as well as within the tools as - `context.life_span`. When the server shuts down, it cleans up the server state. - - :param server: FastMCP server instance - - Usage: - def tool(ctx: Context): - ... = ctx.request_context.life_span.config # ctx.life_span is type of ServerState - - Ideas: - - it could handle OAuth token, client access, Redis database connection for storing sessions, access - to the Relational DB, etc. - """ - yield server_state - - return keboola_lifespan - - -class CustomRoutes: - """Routes which are not part of the MCP protocol.""" - - def __init__(self, server_state: ServerState, oauth_provider: SimpleOAuthProvider | None = None) -> None: - self.server_state = server_state - self.oauth_provider = oauth_provider - - async def get_status(self, _rq: Request) -> Response: - """Checks the service is up and running.""" - resp = StatusApiResp(status='ok') - return JSONResponse(resp.model_dump(by_alias=True)) - - async def get_info(self, _rq: Request) -> Response: - """Returns basic information about the service.""" - resp = ServiceInfoApiResp( - app_version=self.server_state.runtime_info.app_version, - server_version=self.server_state.runtime_info.server_version, - mcp_library_version=self.server_state.runtime_info.mcp_library_version, - fastmcp_library_version=self.server_state.runtime_info.fastmcp_library_version, - server_transport=self.server_state.runtime_info.transport, - server_id=self.server_state.runtime_info.server_id, - ) - return JSONResponse(resp.model_dump(by_alias=True)) - - async def oauth_callback_handler(self, request: Request) -> Response: - """Handle GitHub OAuth callback.""" - code = request.query_params.get('code') - state = request.query_params.get('state') - - if not code or not state: - raise HTTPException(400, 'Missing code or state parameter') - - try: - assert self.oauth_provider # this must have been set if we are handling OAuth callbacks - redirect_uri = await self.oauth_provider.handle_oauth_callback(code, state) - return RedirectResponse(status_code=302, url=redirect_uri) - except HTTPException: - raise - except Exception as e: - LOG.exception(f'Failed to handle OAuth callback: {e}') - return JSONResponse(status_code=500, content={'message': f'Unexpected error: {e}'}) - - def add_to_mcp(self, mcp: FastMCP) -> None: - """Add custom routes to an MCP server. - - :param mcp: MCP server instance. - """ - mcp.custom_route('/', methods=['GET'])(self.get_info) - mcp.custom_route('/health-check', methods=['GET'])(self.get_status) - mcp.custom_route('/preview/configuration', methods=['POST'])(preview_config_diff) - if self.oauth_provider: - mcp.custom_route('/oauth/callback', methods=['GET'])(self.oauth_callback_handler) - - def add_to_starlette(self, app: Starlette) -> None: - """Add custom routes to a Starlette app. - - :param app: Starlette app instance. - """ - app.state.server_state = self.server_state - app.add_route('/', self.get_info, methods=['GET']) - app.add_route('/health-check', self.get_status, methods=['GET']) - app.add_route('/preview/configuration', preview_config_diff, methods=['POST']) - if self.oauth_provider: - app.add_route('/oauth/callback', self.oauth_callback_handler, methods=['GET']) - for route in self.oauth_provider.get_routes(): - app.add_route(route.path, route.endpoint, methods=route.methods) - - -def create_server( - config: Config, - *, - runtime_info: ServerRuntimeInfo, - custom_routes_handling: Literal['add', 'return'] | None = 'add', -) -> FastMCP | tuple[FastMCP, CustomRoutes]: - """Create and configure the MCP server. - - :param config: Server configuration. - :param runtime_info: Server runtime information holding the server versions, transport, etc. - :param custom_routes_handling: Add custom routes (health check etc.) to the server. If 'add', - the routes are added to the MCP server instance. If 'return', the routes are returned as a CustomRoutes - instance. If None, no custom routes are added. The 'return' mode is a workaround for the 'http-compat' - mode, where we need to add the custom routes to the parent app. - :return: Configured FastMCP server instance. - """ - config = config.replace_by(os.environ) - - hostname_suffix = os.environ.get('HOSTNAME_SUFFIX') - if not config.storage_api_url and hostname_suffix: - config = dataclasses.replace(config, storage_api_url=f'https://connection.{hostname_suffix}') - - if config.oauth_client_id and config.oauth_client_secret: - # fall back to HOSTNAME_SUFFIX if no URLs are specified for the OAUth server or the MCP server itself - if not config.oauth_server_url and hostname_suffix: - config = dataclasses.replace(config, oauth_server_url=f'https://connection.{hostname_suffix}') - if not config.mcp_server_url and hostname_suffix: - config = dataclasses.replace(config, mcp_server_url=f'https://mcp.{hostname_suffix}') - if not config.oauth_scope: - config = dataclasses.replace(config, oauth_scope='email') - - oauth_provider = SimpleOAuthProvider( - storage_api_url=config.storage_api_url, - client_id=config.oauth_client_id, - client_secret=config.oauth_client_secret, - server_url=config.oauth_server_url, - scope=config.oauth_scope, - # This URL must be reachable from the internet. - mcp_server_url=config.mcp_server_url, - # The path corresponds to oauth_callback_handler() set up below. - callback_endpoint='/oauth/callback', - jwt_secret=config.jwt_secret, - ) - else: - oauth_provider = None - - # Initialize FastMCP server with system lifespan - LOG.info(f'Creating server with config: {config}') - server_state = ServerState(config=config, runtime_info=runtime_info) - mcp = KeboolaMcpServer( - name='Keboola MCP Server', - lifespan=create_keboola_lifespan(server_state), - auth=oauth_provider, - middleware=[ - LoggingMiddleware(log_level=logging.DEBUG), - SessionStateMiddleware(), - ToolAuthorizationMiddleware(), - ToolsFilteringMiddleware(), - ValidationErrorMiddleware(), - ], - ) - - if custom_routes_handling: - custom_routes = CustomRoutes(server_state=server_state, oauth_provider=oauth_provider) - if custom_routes_handling == 'add': - custom_routes.add_to_mcp(mcp) - - add_component_tools(mcp) - add_data_app_tools(mcp) - add_doc_tools(mcp) - add_flow_tools(mcp) - add_job_tools(mcp) - add_oauth_tools(mcp) - add_project_tools(mcp) - add_search_tools(mcp) - add_semantic_tools(mcp) - add_sql_tools(mcp) - add_storage_tools(mcp) - add_keboola_prompts(mcp) - - if custom_routes_handling != 'return': - return mcp - else: - return mcp, custom_routes diff --git a/src/keboola_mcp_server/tools/__init__.py b/src/keboola_mcp_server/tools/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/keboola_mcp_server/tools/components/__init__.py b/src/keboola_mcp_server/tools/components/__init__.py deleted file mode 100644 index 48e60cb9f..000000000 --- a/src/keboola_mcp_server/tools/components/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public exports for component tools.""" - -from keboola_mcp_server.tools.components.tools import ( - add_config_row, - create_config, - create_sql_transformation, - get_components, - get_config_examples, - get_configs, - run_sync_action, - update_config, - update_config_row, - update_sql_transformation, -) - -__all__ = [ - 'get_configs', - 'get_components', - 'get_config_examples', - 'create_sql_transformation', - 'update_sql_transformation', - 'create_config', - 'add_config_row', - 'update_config', - 'update_config_row', -] diff --git a/src/keboola_mcp_server/tools/components/api_models.py b/src/keboola_mcp_server/tools/components/api_models.py deleted file mode 100644 index 9b4fd7026..000000000 --- a/src/keboola_mcp_server/tools/components/api_models.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -Raw API Models - Pure data transfer objects that mirror API responses exactly. - -These models represent the raw data returned by Keboola APIs. -They contain no business logic and use the exact field names and structures from the APIs. -""" - -from typing import Any, Optional - -from pydantic import AliasChoices, BaseModel, Field - - -class ComponentAPIResponse(BaseModel): - """ - Raw component response that can handle both Storage API and AI Service API responses. - - Storage API (/v2/storage/components/{id}) returns just the core fields. - AI Service API (/docs/components/{id}) returns core fields + optional documentation metadata. - - The optional fields will be None when parsing Storage API responses. - """ - - # Core fields present in both APIs (SAPI and AI service) - component_id: str = Field( - description='The ID of the component', - validation_alias=AliasChoices('id', 'component_id', 'componentId', 'component-id'), - ) - component_name: str = Field( - description='The name of the component', - validation_alias=AliasChoices( - 'name', - 'component_name', - 'componentName', - 'component-name', - ), - ) - type: str = Field( - description='Component type (extractor, writer, application)', - validation_alias=AliasChoices('type', 'component_type', 'componentType', 'component-type'), - ) - flags: list[str] = Field( - default_factory=list, - description='Developer portal flags', - validation_alias=AliasChoices('flags', 'component_flags', 'componentFlags', 'component-flags'), - ) - categories: list[str] = Field( - default_factory=list, - description='Component categories', - validation_alias=AliasChoices( - 'categories', - 'component_categories', - 'componentCategories', - 'component-categories', - ), - ) - - # Optional metadata fields only present in AI Service API responses - documentation_url: str | None = Field( - default=None, - description='Documentation URL', - validation_alias=AliasChoices('documentationUrl', 'documentation_url', 'documentation-url'), - ) - documentation: str | None = Field( - default=None, - description='Component documentation', - validation_alias=AliasChoices('documentation'), - ) - configuration_schema: dict[str, Any] | None = Field( - default=None, - description='Configuration schema', - validation_alias=AliasChoices('configurationSchema', 'configuration_schema', 'configuration-schema'), - ) - configuration_row_schema: dict[str, Any] | None = Field( - default=None, - description='Configuration row schema', - validation_alias=AliasChoices('configurationRowSchema', 'configuration_row_schema', 'configuration-row-schema'), - ) - data: dict[str, Any] | None = Field( - default=None, - description='Additional component metadata', - validation_alias=AliasChoices('data'), - ) - - -class ConfigurationAPIResponse(BaseModel): - """ - Raw API response for configuration endpoints. - - Mirrors the actual JSON structure returned by Keboola Storage API for: - - configuration_detail() - - configuration_list() - - configuration_create() - - configuration_update() - """ - - component_id: str = Field( - description='The ID of the component', - validation_alias=AliasChoices('component_id', 'componentId', 'component-id'), - ) - configuration_id: str = Field( - description='The ID of the configuration', - validation_alias=AliasChoices('id', 'configuration_id', 'configurationId', 'configuration-id'), - ) - name: str = Field(description='The name of the configuration') - description: Optional[str] = Field(default=None, description='The description of the configuration') - version: int = Field(description='The version of the configuration') - is_disabled: bool = Field( - default=False, - description='Whether the configuration is disabled', - validation_alias=AliasChoices('isDisabled', 'is_disabled', 'is-disabled'), - ) - is_deleted: bool = Field( - default=False, - description='Whether the configuration is deleted', - validation_alias=AliasChoices('isDeleted', 'is_deleted', 'is-deleted'), - ) - configuration: dict[str, Any] = Field( - description='The nested configuration object containing parameters and storage' - ) - rows: Optional[list[dict[str, Any]]] = Field( - default=None, description='The row configurations within this configuration' - ) - change_description: Optional[str] = Field( - default=None, - description='The description of the latest changes', - validation_alias=AliasChoices('changeDescription', 'change_description', 'change-description'), - ) - metadata: list[dict[str, Any]] = Field( - default_factory=list, - description='Configuration metadata', - validation_alias=AliasChoices('metadata', 'configuration_metadata', 'configurationMetadata'), - ) diff --git a/src/keboola_mcp_server/tools/components/model.py b/src/keboola_mcp_server/tools/components/model.py deleted file mode 100644 index 1e840fefc..000000000 --- a/src/keboola_mcp_server/tools/components/model.py +++ /dev/null @@ -1,913 +0,0 @@ -""" -Domain models for Keboola component and configuration management. - -This module contains the business domain models used throughout the MCP server for representing -Keboola components and their configurations. The models are organized into logical groups: - -## Component Models -- Component: Full component details with schemas and documentation -- ComponentSummary: Lightweight component info for list operations -- ComponentCapabilities: What a component can do (derived from developer portal flags) - -## Configuration Models -The new configuration models provide a structured approach separating shared settings -from individual tasks: - - ### Detail Models (for get operations) - - Configuration: Complete config with root + rows + component context - - ConfigurationRoot: Shared settings (credentials, global config) - - ConfigurationRow: Individual tasks (table mappings, specific parameters) - - ### Summary Models (for list operations) - - ConfigurationSummary: Lightweight config structure - - ConfigurationRootSummary: Essential root metadata only - - ConfigurationRowSummary: Essential row metadata only - -## Tool Output Models -- ConfigToolOutput: Standard response for config create/update operations -- GetConfigsListOutput: Response for get_configs tool (list mode) -- GetConfigsDetailOutput: Response for get_configs tool (detail mode) -- GetConfigsOutput: Union of list and detail output for get_configs tool - -## Legacy Models -- ComponentConfigurationResponseBase: Base class used by Flow tools (FlowConfigurationResponse) -""" - -import asyncio -from datetime import datetime -from typing import Annotated, Any, Literal, Optional, Sequence, Union, get_args - -from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator - -from keboola_mcp_server.clients.client import get_metadata_property -from keboola_mcp_server.clients.encryption import redact_secrets -from keboola_mcp_server.clients.storage import ComponentAPIResponse, ComponentType, ConfigurationAPIResponse -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.links import Link - -# ============================================================================ -# TYPE DEFINITIONS -# ============================================================================ - - -class VariableDefinition(BaseModel): - """A single variable definition to attach to a configuration.""" - - name: str = Field(description='Variable name.') - type: Literal['string', 'vault'] = Field(default='string', description='Variable type: "string" or "vault".') - default_value: Optional[str] = Field(default=None, description='Optional default value bound at creation time.') - - -ALL_COMPONENT_TYPES = tuple(component_type for component_type in get_args(ComponentType)) - - -# ============================================================================ -# COMPONENT MODELS -# ============================================================================ - - -class ComponentCapabilities(BaseModel): - """ - Component capabilities derived from developer portal flags. - - Represents what a component can do in terms of data processing: - - Row-based: Can have multiple configuration rows for different tasks - - Table I/O: Can read from or write to data tables - - File I/O: Can read from or write to files - - OAuth: Requires OAuth authentication setup - """ - - is_row_based: bool = Field(default=False, description='Whether the component supports row configurations') - has_table_input: bool = Field(default=False, description='Whether the component can read from tables') - has_table_output: bool = Field(default=False, description='Whether the component can write to tables') - has_file_input: bool = Field(default=False, description='Whether the component can read from files') - has_file_output: bool = Field(default=False, description='Whether the component can write to files') - requires_oauth: bool = Field(default=False, description='Whether the component requires OAuth authorization') - - @classmethod - def from_flags(cls, flags: list[str]) -> 'ComponentCapabilities': - """ - Derive component capabilities from developer portal flags. - - :param flags: List of developer portal flags from API response - :return: Structured component capabilities - """ - return cls( - is_row_based='genericDockerUI-rows' in flags, - has_table_input=any( - flag in flags for flag in ['genericDockerUI-tableInput', 'genericDockerUI-simpleTableInput'] - ), - has_table_output='genericDockerUI-tableOutput' in flags, - has_file_input='genericDockerUI-fileInput' in flags, - has_file_output='genericDockerUI-fileOutput' in flags, - requires_oauth='genericDockerUI-authorization' in flags, - ) - - -class ComponentSummary(BaseModel): - """Lightweight component representation for list operations.""" - - component_id: str = Field(description='Component ID') - component_name: str = Field(description='Component name') - component_type: str = Field(description='Component type') - capabilities: ComponentCapabilities = Field(description='Component capabilities') - links: list[Link] = Field(default_factory=list, description='Navigation links for the web interface') - - @classmethod - def from_api_response(cls, api_response: ComponentAPIResponse) -> 'ComponentSummary': - """ - Create ComponentSummary from API response. - - :param api_response: Parsed API response from Storage or AI Service API - :return: Lightweight component domain model for list operations - """ - capabilities = ComponentCapabilities.from_flags(api_response.flags) - - return cls.model_construct( - component_id=api_response.component_id, - component_name=api_response.component_name, - component_type=api_response.type, - capabilities=capabilities, - ) - - -class Component(BaseModel): - """ - Complete component representation with full details. - - Contains comprehensive component information including documentation, - configuration schemas, and metadata. Used by get tools where detailed - component information is needed. - """ - - # Core component metadata (shared with ComponentSummary) - component_id: str = Field(description='Component ID') - component_name: str = Field(description='Component name') - component_type: str = Field(description='Component type') - component_categories: list[str] = Field( - default_factory=list, - description='Component categories', - ) - capabilities: ComponentCapabilities = Field(description='Component capabilities') - - # Additional metadata - documentation_url: str | None = Field( - default=None, - description='URL to component documentation', - ) - documentation: str | None = Field( - default=None, - description='Component documentation text', - ) - configuration_schema: dict[str, Any] | None = Field( - default=None, - description='JSON schema for configuration root validation', - ) - configuration_row_schema: dict[str, Any] | None = Field( - default=None, - description='JSON schema for configuration row validation', - ) - - links: list[Link] = Field(default_factory=list, description='Links for UI navigation') - - sync_actions: list[str] | None = Field( - default=None, - description='Synchronous actions supported by the component (e.g., "testConnection")', - ) - - @classmethod - def from_api_response(cls, api_response: ComponentAPIResponse) -> 'Component': - """ - Create Component from API response. - - :param api_response: Parsed API response from Storage or AI Service API - :return: Complete component domain model with detailed metadata - """ - capabilities = ComponentCapabilities.from_flags(api_response.flags) - - return cls.model_construct( - component_id=api_response.component_id, - component_name=api_response.component_name, - component_type=api_response.type, - component_categories=api_response.categories, - capabilities=capabilities, - documentation_url=api_response.documentation_url, - documentation=api_response.documentation, - configuration_schema=api_response.configuration_schema, - configuration_row_schema=api_response.configuration_row_schema, - sync_actions=(api_response.data or {}).get('synchronous_actions'), - ) - - -class GetComponentsOutput(BaseModel): - """Output of the get_components tool.""" - - components: list[Component] = Field(description='The components') - links: list[Link] = Field(description='Navigation links for the web interface.', default_factory=list) - - -# ============================================================================ -# CONFIGURATION MODELS -# ============================================================================ - - -class FullConfigId(BaseModel, frozen=True): - """Composite configuration ID (component ID + configuration ID).""" - - component_id: str = Field(description='ID of the component') - configuration_id: str = Field(description='ID of the configuration') - - -class ConfigurationRoot(BaseModel): - """ - Complete configuration root with all data. - - Represents the shared configuration settings for a component including - credentials, global parameters, and shared storage mappings. For row-based - components, this contains the common settings that apply to all rows. - """ - - component_id: str = Field(description='The ID of the component') - configuration_id: str = Field(description='The ID of this configuration root') - name: str = Field(description='The name of the configuration') - description: Optional[str] = Field(default=None, description='The description of the configuration') - version: int = Field(description='The version of the configuration') - is_disabled: bool = Field(default=False, description='Whether the configuration is disabled') - is_deleted: bool = Field(default=False, description='Whether the configuration is deleted') - folder: str = Field(default='', description='The UI folder this configuration is organized into') - parameters: dict[str, Any] = Field( - description='The configuration parameters, adhering to the configuration root schema' - ) - storage: Optional[dict[str, Any]] = Field( - default=None, description='The table and/or file input/output mapping configuration' - ) - processors: Optional[dict[str, Any]] = Field( - default=None, description='The processors that run before or after the configured component.' - ) - variables_id: Optional[str] = Field(default=None, description='ID of the linked keboola.variables configuration') - variables_values_id: Optional[str] = Field( - default=None, description='ID of the Default Values row in the linked keboola.variables configuration' - ) - variables: Optional[list[dict[str, Any]]] = Field( - default=None, description='Variable definitions (keboola.variables configs only)' - ) - configuration_metadata: list[dict[str, Any]] = Field( - default_factory=list, description='Configuration metadata including MCP tracking' - ) - - @field_validator('processors', mode='before') - @classmethod - def validate_processors(cls, value: Any) -> Any: - # Storage API returns [] when no processors are configured instead of None or {} - if value == []: - return None - return value - - @classmethod - def from_api_response(cls, api_config: 'ConfigurationAPIResponse') -> 'ConfigurationRoot': - """ - Create ConfigurationRoot from API response. - - Handles the flattening of nested configuration.parameters and configuration.storage - from the API response structure into the domain model. - - :param api_config: Validated API configuration response - :return: Complete configuration root domain model - """ - return cls( - component_id=api_config.component_id, - configuration_id=api_config.configuration_id, - name=api_config.name, - description=api_config.description, - version=api_config.version, - is_disabled=api_config.is_disabled, - is_deleted=api_config.is_deleted, - folder=get_metadata_property(api_config.metadata, MetadataField.CONFIGURATION_FOLDER_NAME) or '', - # Plaintext '#'-prefixed secret values are redacted so that they never reach the model's context. - # 'KBC::' ciphers are kept as they are opaque. - parameters=redact_secrets(api_config.configuration.get('parameters', {})), - storage=api_config.configuration.get('storage'), - processors=redact_secrets(api_config.configuration.get('processors')), - variables_id=api_config.configuration.get('variables_id'), - variables_values_id=api_config.configuration.get('variables_values_id'), - variables=api_config.configuration.get('variables'), - configuration_metadata=api_config.metadata, - ) - - -class ConfigurationRow(BaseModel): - """ - Complete configuration row with all data. - - Represents an individual task or extraction within a configuration. - For row-based components, each row typically handles a specific data source, - destination, or transformation operation. - """ - - component_id: str = Field(description='The ID of the component') - configuration_id: str = Field(description='The ID of the corresponding configuration root') - configuration_row_id: str = Field(description='The ID of this configuration row') - name: str = Field(description='The name of the configuration row') - description: Optional[str] = Field(default=None, description='The description of the configuration row') - version: int = Field(description='The version of the configuration row') - is_disabled: bool = Field(default=False, description='Whether the configuration row is disabled') - is_deleted: bool = Field(default=False, description='Whether the configuration row is deleted') - parameters: dict[str, Any] = Field( - description='The configuration row parameters, adhering to the configuration row schema' - ) - storage: Optional[dict[str, Any]] = Field( - default=None, description='The table and/or file input/output mapping configuration' - ) - processors: Optional[dict[str, Any]] = Field( - default=None, description='The processors that run before or after the configured component row.' - ) - values: Optional[list[dict[str, Any]]] = Field( - default=None, description='Variable default values (keboola.variables rows only)' - ) - configuration_metadata: list[dict[str, Any]] = Field(default_factory=list, description='Configuration row metadata') - - @field_validator('processors', mode='before') - @classmethod - def validate_processors(cls, value: Any) -> Any: - # Storage API returns [] when no processors are configured instead of None or {} - if value == []: - return None - return value - - @classmethod - def from_api_row_data( - cls, - row_data: dict[str, Any], - component_id: str, - configuration_id: str, - ) -> 'ConfigurationRow': - """ - Create ConfigurationRow from API row data. - - Converts individual row data from the API into a structured domain model. - Handles the nested structure of configuration row data. - - :param row_data: Raw row data from API response - :param component_id: ID of the parent component - :param configuration_id: ID of the parent configuration - :return: Complete configuration row domain model - """ - row_cfg = row_data.get('configuration', {}) - return cls( - component_id=component_id, - configuration_id=configuration_id, - configuration_row_id=row_data['id'], - name=row_data['name'], - description=row_data.get('description'), - version=row_data['version'], - is_disabled=row_data.get('isDisabled', False), - is_deleted=row_data.get('isDeleted', False), - # Plaintext '#'-prefixed secret values are redacted so that they never reach the model's context. - parameters=redact_secrets(row_cfg.get('parameters', {})), - storage=row_cfg.get('storage'), - processors=redact_secrets(row_cfg.get('processors')), - values=row_cfg.get('values'), - configuration_metadata=row_cfg.get('metadata', []), - ) - - -class ConfigurationRootSummary(BaseModel): - """Lightweight configuration root for list operations.""" - - component_id: str = Field(description='The ID of the component') - configuration_id: str = Field(description='The ID of this configuration root') - name: str = Field(description='The name of the configuration') - description: Optional[str] = Field(default=None, description='The description of the configuration') - is_disabled: bool = Field(default=False, description='Whether the configuration is disabled') - is_deleted: bool = Field(default=False, description='Whether the configuration is deleted') - folder: str = Field(default='', description='The UI folder this configuration is organized into') - - @classmethod - def from_api_response(cls, api_config: 'ConfigurationAPIResponse') -> 'ConfigurationRootSummary': - """Create lightweight configuration root summary from API response.""" - return cls.model_construct( - component_id=api_config.component_id, - configuration_id=api_config.configuration_id, - name=api_config.name, - description=api_config.description, - is_disabled=api_config.is_disabled, - is_deleted=api_config.is_deleted, - folder=get_metadata_property(api_config.metadata, MetadataField.CONFIGURATION_FOLDER_NAME) or '', - ) - - -class ConfigurationRowSummary(BaseModel): - """Lightweight configuration row for list operations.""" - - component_id: str = Field(description='The ID of the component') - configuration_id: str = Field(description='The ID of the corresponding configuration root') - row_configuration_id: str = Field(description='The ID of this configuration row') - name: str = Field(description='The name of the configuration row') - description: Optional[str] = Field(default=None, description='The description of the configuration row') - is_disabled: bool = Field(default=False, description='Whether the configuration row is disabled') - is_deleted: bool = Field(default=False, description='Whether the configuration row is deleted') - - @classmethod - def from_api_row_data( - cls, - row_data: dict[str, Any], - component_id: str, - configuration_id: str, - ) -> 'ConfigurationRowSummary': - """Create lightweight configuration row summary from API row data.""" - return cls( - component_id=component_id, - configuration_id=configuration_id, - row_configuration_id=row_data['id'], - name=row_data['name'], - description=row_data.get('description'), - is_disabled=row_data.get('isDisabled', False), - is_deleted=row_data.get('isDeleted', False), - ) - - -class ConfigSummary(BaseModel): - """ - Lightweight configuration structure for list operations. - - Container model that mirrors the structure of the full Configuration model - but with lightweight summary data. Used by list operations where many - configurations are returned. - """ - - configuration_root: ConfigurationRootSummary = Field(description='The configuration root summary') - configuration_rows: Optional[list[ConfigurationRowSummary]] = Field( - default=None, description='The configuration row summaries' - ) - links: list[Link] = Field(default_factory=list, description='Navigation links for the web interface') - - @classmethod - def from_api_response(cls, api_config: 'ConfigurationAPIResponse') -> 'ConfigSummary': - """ - Create ConfigurationSummary from API response. - - Builds a lightweight configuration structure by creating summary models - for both configuration root and configurations row from the API response data. - - :param api_config: Validated API configuration response - :return: Lightweight configuration structure for list operations - """ - configuration_root = ConfigurationRootSummary.from_api_response(api_config) - - configuration_rows = None - if api_config.rows: - configuration_rows = [ - ConfigurationRowSummary.from_api_row_data( - row_data=row, - component_id=api_config.component_id, - configuration_id=api_config.configuration_id, - ) - for row in api_config.rows - ] - - return cls.model_construct( - configuration_root=configuration_root, - configuration_rows=configuration_rows, - ) - - -class Configuration(BaseModel): - """ - Complete configuration structure for detailed views. - - Container model that holds both configuration root and configuration rows along with - component context and UI links. Used by get operations where detailed - configuration information is needed. - """ - - configuration_root: ConfigurationRoot = Field(description='The complete configuration root') - configuration_rows: Optional[list[ConfigurationRow]] = Field( - default=None, description='The complete configuration rows' - ) - component: Optional[ComponentSummary] = Field( - default=None, description='The component this configuration belongs to' - ) - links: list[Link] = Field(default_factory=list, description='Navigation links for the web interface') - - @classmethod - def from_api_response( - cls, - api_config: 'ConfigurationAPIResponse', - component: Optional[ComponentSummary] = None, - links: Optional[list[Link]] = None, - ) -> 'Configuration': - """ - Create Configuration from API response. - - Builds the complete configuration structure including full root and row - data, along with component context and UI links when provided. - - :param api_config: Validated API configuration response - :param component: Lightweight component context (optional) - :param links: UI navigation links (optional) - :return: Complete configuration model for detailed operations - """ - configuration_root = ConfigurationRoot.from_api_response(api_config) - - configuration_rows = None - if api_config.rows: - configuration_rows = [ - ConfigurationRow.from_api_row_data( - row_data=row, - component_id=api_config.component_id, - configuration_id=api_config.configuration_id, - ) - for row in api_config.rows - ] - - return cls.model_construct( - configuration_root=configuration_root, - configuration_rows=configuration_rows, - component=component, - links=links or [], - ) - - -class ComponentWithConfigs(BaseModel, frozen=True): - """Grouping of a component and its associated configuration summaries.""" - - component: ComponentSummary = Field(description='The Keboola component.') - configs: list[ConfigSummary] = Field( - description='List of configuration summaries associated with the component.', - ) - - -# ============================================================================ -# CONFIGURATION PARAMETER UPDATE MODELS -# ============================================================================ - - -class ConfigParamSet(BaseModel, frozen=True): - """ - Set or create a parameter value at the specified path. - - Use this operation to: - - Update an existing parameter value - - Create a new parameter key - - Replace a nested parameter value - """ - - op: Literal['set'] # name 'op' inspired by JSON Patch (https://datatracker.ietf.org/doc/html/rfc6902) - path: str = Field(description='JSONPath to the parameter key to set (e.g., "api_key", "database.host")') - value: Any = Field(description='New value to set') - - -class ConfigParamReplace(BaseModel, frozen=True): - """Replace a substring in a string parameter.""" - - op: Literal['str_replace'] - path: str = Field(description='JSONPath to the parameter key to modify') - search_for: str = Field(description='Substring to search for (non-empty)') - replace_with: str = Field(description='Replacement string (can be empty for deletion)') - - -class ConfigParamRemove(BaseModel, frozen=True): - """Remove a parameter key.""" - - op: Literal['remove'] - path: str = Field(description='JSONPath to the parameter key to remove') - - -class ConfigParamListAppend(BaseModel, frozen=True): - """Append a value to a list parameter.""" - - op: Literal['list_append'] - path: str = Field(description='JSONPath to the list parameter') - value: Any = Field(description='Value to append to the list') - - -# Discriminated union of all parameter update operations -ConfigParamUpdate = Annotated[ - Union[ConfigParamSet, ConfigParamReplace, ConfigParamRemove, ConfigParamListAppend], Field(discriminator='op') -] - - -# ============================================================================ -# TRANSFORMATION MODELS -# ============================================================================ - - -class TransformationConfiguration(BaseModel): - """ - Creates the transformation configuration, a schema for the transformation configuration in the API. - Currently, the storage configuration uses only input and output tables, excluding files, etc. - """ - - class Parameters(BaseModel): - """The parameters for the transformation.""" - - class Block(BaseModel): - """The transformation block.""" - - class Code(BaseModel): - """The code block for the transformation block.""" - - name: str = Field(description='The name of the current code block describing the purpose of the block') - script: Sequence[str] = Field( - description=( - 'The executable SQL query statements written in the current SQL dialect. ' - 'Each statement must be executable and a separate item in the list.' - ), - ) - - name: str = Field(description='The name of the current block') - codes: list[Code] = Field(description='The code scripts') - - blocks: list[Block] = Field(description='The blocks for the transformation') - - async def to_simplified_parameters(self) -> 'SimplifiedTfBlocks': - # Avoid circular import - from keboola_mcp_server.tools.components.sql_utils import join_sql_statements - - """Convert the raw parameters to simplified parameters.""" - return SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name=block.name, - codes=[ - SimplifiedTfBlocks.Block.Code(name=code.name, script=join_sql_statements(code.script)) - for code in block.codes - ], - ) - for block in self.blocks - ] - ) - - class Storage(BaseModel): - """The storage configuration for the transformation. For now it stores only input and output tables.""" - - class Destination(BaseModel): - """Tables' destinations for the transformation. Either input or output tables.""" - - class Table(BaseModel): - """The table used in the transformation""" - - destination: Optional[str] = Field(description='The destination table name', default=None) - source: Optional[str] = Field(description='The source table name', default=None) - - tables: list[Table] = Field(description='The tables used in the transformation', default_factory=list) - - input: Destination = Field(description='The input tables for the transformation', default_factory=Destination) - output: Destination = Field(description='The output tables for the transformation', default_factory=Destination) - - parameters: Parameters = Field(description='The parameters for the transformation') - storage: Storage = Field(description='The storage configuration for the transformation') - - -# Type alias for TransformationConfiguration.Parameters for convenience -TransformationBlocks = TransformationConfiguration.Parameters - - -class SimplifiedTfBlocks(BaseModel): - """ - Transformations parameters blocks simplified for the agent use: - `script` is a string instead of a list of statements - """ - - class Block(BaseModel): - """The transformation block.""" - - class Code(BaseModel): - """The code block for the transformation block.""" - - name: str = Field(description='A descriptive name for the code block') - script: str = Field(description='The SQL script of the code block') - - async def to_raw_code(self) -> TransformationConfiguration.Parameters.Block.Code: - # Avoid circular import - from keboola_mcp_server.tools.components.sql_utils import split_sql_statements - - """Convert the simplified code to the raw code.""" - return TransformationConfiguration.Parameters.Block.Code( - name=self.name, script=await split_sql_statements(self.script) - ) - - name: str = Field(description='A descriptive name for the code block') - codes: list[Code] = Field(description='SQL code sub-blocks') - - blocks: list[Block] = Field(description='SQL code blocks') - - async def to_raw_parameters(self) -> TransformationConfiguration.Parameters: - """Convert the simplified transformation parameters to raw (SAPI) parameters.""" - return TransformationConfiguration.Parameters( - blocks=[ - TransformationConfiguration.Parameters.Block( - name=block.name, codes=await asyncio.gather(*[code.to_raw_code() for code in block.codes]) - ) - for block in self.blocks - ] - ) - - -# ============================================================================ -# TRANSFORMATION PARAMETER UPDATE MODELS -# ============================================================================ - - -TfPosition = Literal['start', 'end'] - - -class TfAddBlock(BaseModel, frozen=True): - """Add a new block to the transformation.""" - - op: Literal['add_block'] - block: SimplifiedTfBlocks.Block = Field(description='The block to add') - position: TfPosition = Field(description='The position of the block to add', default='end') - - -class TfRemoveBlock(BaseModel, frozen=True): - """Remove an existing block from the transformation.""" - - op: Literal['remove_block'] - block_id: str = Field(description='The ID of the block to remove') - - -class TfRenameBlock(BaseModel, frozen=True): - """Rename an existing block in the transformation.""" - - op: Literal['rename_block'] - block_id: str = Field(description='The ID of the block to rename') - block_name: str = Field(description='The new name of the block') - - -class TfAddCode(BaseModel, frozen=True): - """Add a new code to an existing block in the transformation.""" - - op: Literal['add_code'] - block_id: str = Field(description='The ID of the block to add the code to') - code: SimplifiedTfBlocks.Block.Code = Field(description='The code to add') - position: TfPosition = Field(description='The position of the code to add', default='end') - - -class TfRemoveCode(BaseModel, frozen=True): - """Remove an existing code from an existing block in the transformation.""" - - op: Literal['remove_code'] - block_id: str = Field(description='The ID of the block to remove the code from') - code_id: str = Field(description='The ID of the code to remove') - - -class TfRenameCode(BaseModel, frozen=True): - """Rename an existing code in an existing block in the transformation.""" - - op: Literal['rename_code'] - block_id: str = Field(description='The ID of the block to rename the code in') - code_id: str = Field(description='The ID of the code to rename') - code_name: str = Field(description='The new name of the code') - - -class TfSetCode(BaseModel, frozen=True): - """Set the SQL script of an existing code in an existing block in the transformation.""" - - op: Literal['set_code'] - block_id: str = Field(description='The ID of the block to set the code in') - code_id: str = Field(description='The ID of the code to set') - script: str = Field(description='The SQL script of the code to set') - - -class TfAddScript(BaseModel, frozen=True): - """Append or prepend SQL script text to an existing code in an existing block in the transformation.""" - - op: Literal['add_script'] - block_id: str = Field(description='The ID of the block to add the script to') - code_id: str = Field(description='The ID of the code to add the script to') - script: str = Field(description='The SQL script to add') - position: TfPosition = Field(description='The position of the script to add', default='end') - - -class TfStrReplace(BaseModel, frozen=True): - """Replace a substring in SQL statements in the transformation.""" - - op: Literal['str_replace'] - block_id: Optional[str] = Field( - description='The ID of the block to replace substrings in. If not provided, all blocks will be updated.', - default=None, - ) - code_id: Optional[str] = Field( - description='The ID of the code to replace substrings in. ' - 'If not provided, all codes in the block will be updated.', - default=None, - ) - search_for: str = Field(description='Substring to search for (non-empty)') - replace_with: str = Field(description='Replacement string (can be empty for deletion)') - - @model_validator(mode='after') - def validate_code_id_requires_block_id(self) -> 'TfStrReplace': - """Validate that code_name can only be specified if block_name is also specified.""" - if self.block_id is None and self.code_id is not None: - raise ValueError('code_id must be None if block_id is None') - return self - - -# Discriminated union of all transformation parameter update operations -TfParamUpdate = Annotated[ - Union[ - TfAddBlock, - TfRemoveBlock, - TfRenameBlock, - TfAddCode, - TfRemoveCode, - TfRenameCode, - TfSetCode, - TfAddScript, - TfStrReplace, - ], - Field(discriminator='op'), -] - - -# ============================================================================ -# TOOL OUTPUT MODELS -# ============================================================================ - - -class ConfigToolOutput(BaseModel): - """Response model for configuration tool operations.""" - - component_id: str = Field(description='The ID of the component.') - configuration_id: str = Field(description='The ID of the configuration.') - description: str = Field(description='The description of the configuration.') - version: int = Field(description='The version number of the configuration.') - timestamp: datetime = Field(description='The timestamp of the operation.') - success: bool = Field(default=True, description='Indicates if the operation succeeded.') - links: list[Link] = Field(description='The links relevant to the configuration.') - change_summary: Optional[str] = Field( - description="Optional summary of the change to update the agent's context.", - default=None, - ) - - -class GetConfigsListOutput(BaseModel, frozen=True): - components_with_configs: list[ComponentWithConfigs] = Field(description='The components with their configurations') - links: list[Link] = Field( - description='Links relevant to the listing of components with configurations.', - ) - - -class GetConfigsDetailOutput(BaseModel, frozen=True): - configs: list[Configuration] = Field(description='List of configurations') - - -GetConfigsOutput = Union[GetConfigsListOutput, GetConfigsDetailOutput] - - -# ============================================================================ -# LEGACY MODELS (minimal set for Flow tools compatibility) -# ============================================================================ - - -class ComponentConfigurationResponseBase(BaseModel): - """ - Legacy base model for component configurations. - - DEPRECATED: Use ConfigurationRootSummary or ConfigurationRowSummary instead. - Maintained for backward compatibility with existing code. - """ - - component_id: str = Field( - description='The ID of the component', - validation_alias=AliasChoices('component_id', 'componentId', 'component-id'), - ) - configuration_id: str = Field( - description='The ID of the component configuration', - validation_alias=AliasChoices( - 'configuration_id', - 'id', - 'configurationId', - 'configuration-id', - ), - ) - configuration_name: str = Field( - description='The name of the component configuration', - validation_alias=AliasChoices( - 'configuration_name', - 'name', - 'configurationName', - 'configuration-name', - ), - ) - configuration_description: Optional[str] = Field( - description='The description of the component configuration', - validation_alias=AliasChoices( - 'configuration_description', - 'description', - 'configurationDescription', - 'configuration-description', - ), - default=None, - ) - is_disabled: bool = Field( - description='Whether the component configuration is disabled', - validation_alias=AliasChoices('is_disabled', 'isDisabled', 'is-disabled'), - default=False, - ) - is_deleted: bool = Field( - description='Whether the component configuration is deleted', - validation_alias=AliasChoices('is_deleted', 'isDeleted', 'is-deleted'), - default=False, - ) diff --git a/src/keboola_mcp_server/tools/components/sql_utils.py b/src/keboola_mcp_server/tools/components/sql_utils.py deleted file mode 100644 index 200bac452..000000000 --- a/src/keboola_mcp_server/tools/components/sql_utils.py +++ /dev/null @@ -1,199 +0,0 @@ -""" -SQL splitting and joining utilities for SQL transformations. - -This module provides functionality to split SQL scripts into individual -statements and join them back together, using regex-based parsing similar -to the Keboola UI's splitQueriesWorker.worker.ts implementation. -""" - -import asyncio -import logging -import re -from typing import Iterable - -import sqlglot - -from keboola_mcp_server.tools.components.model import SimplifiedTfBlocks - -LOG = logging.getLogger(__name__) - -SQL_SPLIT_REGEX = re.compile( - r'\s*(' - r'(?:' # Start non-capturing group for alternatives - r"'[^'\\]*(?:\\.[^'\\]*)*'|" # Single-quoted strings - r'"[^"\\]*(?:\\.[^"\\]*)*"|' # Double-quoted strings - r'\$\$(?:(?!\$\$)[\s\S])*\$\$|' # Multi-line blocks $$...$$ (using [\s\S] for any char) - r'/\*[^*]*\*+(?:[^*/][^*]*\*+)*/|' # Multi-line comments /* ... */ - r'#[^\n\r]*|' # Hash comments - r'--[^\n\r]*|' # SQL comments - r'//[^\n\r]*|' # C-style comments - r'/(?![*/])|' # Division operator: / not followed by * or / - r'-(?!-)|' # Dash/minus: - not followed by another - - r'\$(?!\$)|' # Dollar sign: $ not followed by another $ - r'[^"\';#/$-]+' # Everything else except special chars (greedy match for performance) - r')+' # End non-capturing group, one or more times - r'(?:;|$)' # Statement terminator: semicolon or end - r')', # End capturing group - re.MULTILINE, -) - -# Regex for detecting line comments (single-line style: --, //, #) -LINE_COMMENT_REGEX = re.compile(r'(--|//|#).*$') - -# Regex patterns for parsing block/code structure markers -BLOCK_MARKER_REGEX = re.compile(r'/\*\s*=+\s*BLOCK:\s*([^*]+?)\s*=+\s*\*/', re.MULTILINE) -CODE_MARKER_REGEX = re.compile(r'/\*\s*=+\s*CODE:\s*([^*]+?)\s*=+\s*\*/', re.MULTILINE) - - -async def split_sql_statements(script: str, timeout_seconds: float = 1.0) -> list[str]: - """ - Splits a SQL script string into individual statements. - - Uses regex-based parsing similar to UI's splitQueriesWorker.worker.ts. - Includes timeout protection to prevent catastrophic backtracking. - - :param script: The SQL script string to split - :param timeout_seconds: Maximum time to allow for regex processing - (default: 1.0) - :return: List of individual SQL statements (trimmed, non-empty) - :raises ValueError: If the script is invalid or regex times out - """ - if not script or not script.strip(): - return [] - - try: - try: - statements = await asyncio.wait_for(asyncio.to_thread(_split_with_regex, script), timeout=timeout_seconds) - except asyncio.TimeoutError: - raise ValueError( - f'SQL parsing took too long (possible catastrophic backtracking). Timeout: {timeout_seconds}s' - ) - - if statements is None: - raise ValueError('SQL script is not valid (no matches found)') - - normalized = [stmt.strip() for stmt in statements if stmt.strip()] - - return normalized - - except Exception as e: - if isinstance(e, ValueError): - raise - LOG.exception(f'Failed to split SQL statements: {e}') - raise ValueError(f'Failed to parse SQL script: {e}') - - -def _split_with_regex(script: str) -> list[str]: - """ - Internal function to split SQL using regex. - - This is separated to allow timeout handling in the calling function. - - :param script: The SQL script string to split - :return: List of matched statement strings (may include empty strings) - """ - matches = SQL_SPLIT_REGEX.findall(script) - return matches if matches else [] - - -def join_sql_statements(statements: Iterable[str]) -> str: - """ - Joins SQL statements into a single script string. - - :param statements: List of SQL statements to join - :return: Concatenated SQL script string separated by double newlines - """ - if not statements: - return '' - - result_parts = [] - - for stmt in statements: - trimmed_stmt = stmt.strip() - if not trimmed_stmt: - continue - - result_parts.append(trimmed_stmt) - result_parts.append('\n\n') - - return ''.join(result_parts) - - -def format_sql(sql: str, dialect: str) -> str: - """ - Formats SQL code using sqlglot for better readability. - - :param sql: Raw SQL code (may contain multiple statements) - :param dialect: SQL dialect - :return: Formatted SQL code, or original if formatting fails - """ - try: - # transpile returns a list - one item per statement/comment - formatted_items = sqlglot.transpile(sql, read=dialect.lower(), pretty=True) - - if not formatted_items: - return sql - - def process_item(item: str) -> str | None: - """Process a single formatted item, returning None if it should be skipped.""" - item = item.rstrip() - if not item: - return None - - # Check if it's ONLY a comment (no SQL after it) - # Remove block comments and line comments, then check if anything substantial remains - sql_content = re.sub(r'/\*.*?\*/', '', item, flags=re.DOTALL) - sql_content = re.sub(r'(--.*)$', '', sql_content, flags=re.MULTILINE).strip() - - is_only_comment = not sql_content - - # Add semicolon only to actual SQL statements (not pure comments) - if not is_only_comment and not item.endswith(';'): - item += ';' - - return item - - result = [processed for item in formatted_items if (processed := process_item(item)) is not None] - - if not result: - return sql - - # Join with double newlines (consistent with join_sql_statements) - return '\n\n'.join(result) - except Exception as e: - LOG.warning(f'Failed to format SQL statement in {dialect} dialect: {sql}. Error: {e}') - return sql - - -def format_simplified_tf_code( - code: SimplifiedTfBlocks.Block.Code, dialect: str -) -> tuple[SimplifiedTfBlocks.Block.Code, bool]: - """ - Formats the simplified transformation code using sqlglot for better readability. - - :param code: The simplified transformation code - :param dialect: SQL dialect ('snowflake' or 'bigquery') - :return: Tuple of (formatted simplified transformation code, - bool representing if the script was changed by formatting) - """ - formatted_script = format_sql(sql=code.script, dialect=dialect) - - return SimplifiedTfBlocks.Block.Code(name=code.name, script=formatted_script), formatted_script != code.script - - -def format_simplified_tf_block(block: SimplifiedTfBlocks.Block, dialect: str) -> tuple[SimplifiedTfBlocks.Block, bool]: - """ - Formats the simplified transformation block using sqlglot for better readability. - - :param block: The simplified transformation block - :param dialect: SQL dialect ('snowflake' or 'bigquery') - :return: Tuple of (formatted simplified transformation block, - bool representing if the block was changed by formatting) - """ - formatted_codes = [] - is_changed = False - for code in block.codes: - formatted_code, is_changed_code = format_simplified_tf_code(code=code, dialect=dialect) - is_changed = is_changed or is_changed_code - formatted_codes.append(formatted_code) - return SimplifiedTfBlocks.Block(name=block.name, codes=formatted_codes), is_changed diff --git a/src/keboola_mcp_server/tools/components/tf_update.py b/src/keboola_mcp_server/tools/components/tf_update.py deleted file mode 100644 index 5e12990c6..000000000 --- a/src/keboola_mcp_server/tools/components/tf_update.py +++ /dev/null @@ -1,275 +0,0 @@ -""" -Functions for updating SQL transformation parameters. - -This module provides operations for modifying transformation blocks and codes -using JSONPath for locating and manipulating elements. -""" - -from jsonpath_ng.ext import parse as parse_jsonpath - -from keboola_mcp_server.tools.components.model import ( - TfAddBlock, - TfAddCode, - TfAddScript, - TfRemoveBlock, - TfRemoveCode, - TfRenameBlock, - TfRenameCode, - TfSetCode, - TfStrReplace, -) - -# Operations that change the structure of the transformation -STRUCTURAL_OPS = frozenset[str]({'add_block', 'add_code', 'remove_block', 'remove_code'}) - - -def add_block(params: dict, op: TfAddBlock, sql_dialect: str) -> tuple[dict, str]: - """ - Add a new block to the transformation. - - :param params: The transformation parameters dictionary with 'blocks' key - :param op: The add_block operation - :param sql_dialect: The SQL dialect of the transformation - :return: Tuple of (modified parameters dictionary, change summary message) - :raises ValueError: If params doesn't contain 'blocks' key or block name is empty/whitespace - """ - if 'blocks' not in params: - raise ValueError("Invalid parameters: must contain 'blocks' key") - - if not op.block.name.strip(): - raise ValueError('Invalid operation: block name cannot be empty') - - new_block_dict = op.block.model_dump() - - if op.position == 'start': - params['blocks'].insert(0, new_block_dict) - else: # 'end' - params['blocks'].append(new_block_dict) - - message = f'Added block with name "{op.block.name}"' - - return params, message - - -def remove_block(params: dict, op: TfRemoveBlock, sql_dialect: str) -> tuple[dict, str]: - """ - Remove an existing block from the transformation. - - :param params: The transformation parameters dictionary with 'blocks' key - :param op: The remove_block operation - :return: Tuple of (modified parameters dictionary, change summary message) - """ - expr = parse_jsonpath(f"$.blocks[?(@.id = '{op.block_id}')]") - matches = expr.find(params) - - if not matches: - raise ValueError(f"Block with id '{op.block_id}' does not exist") - - # Remove the block using `filter` - return expr.filter(lambda x: True, params), '' - - -def rename_block(params: dict, op: TfRenameBlock, sql_dialect: str) -> tuple[dict, str]: - """ - Rename an existing block in the transformation. - - :param params: The transformation parameters dictionary with 'blocks' key - :param op: The rename_block operation - :return: Tuple of (modified parameters dictionary, change summary message) - :raises ValueError: If block_id doesn't exist or block_name is empty/whitespace - """ - if not op.block_name.strip(): - raise ValueError('Invalid operation: block name cannot be empty') - - expr = parse_jsonpath(f"$.blocks[?(@.id = '{op.block_id}')].name") - matches = expr.find(params) - - if not matches: - raise ValueError(f"Block with id '{op.block_id}' does not exist") - - return expr.update(params, op.block_name), '' - - -def add_code(params: dict, op: TfAddCode, sql_dialect: str) -> tuple[dict, str]: - """ - Add a new code to an existing block in the transformation. - - :param params: The transformation parameters dictionary with 'blocks' key - :param op: The add_code operation - :param sql_dialect: The SQL dialect of the transformation - :return: Tuple of (modified parameters dictionary, change summary message) - :raises ValueError: If block_id doesn't exist or code name is empty/whitespace - """ - if not op.code.name.strip(): - raise ValueError('Invalid operation: code name cannot be empty') - - expr = parse_jsonpath(f"$.blocks[?(@.id = '{op.block_id}')].codes") - matches = expr.find(params) - - if not matches: - raise ValueError(f"Block with id '{op.block_id}' does not exist") - - codes = matches[0].value - - new_code_dict = op.code.model_dump() - - if op.position == 'start': - codes.insert(0, new_code_dict) - else: # 'end' - codes.append(new_code_dict) - - message = f'Added code with name "{op.code.name}"' - - return params, message - - -def remove_code(params: dict, op: TfRemoveCode, sql_dialect: str) -> tuple[dict, str]: - """ - Remove an existing code from an existing block in the transformation. - - :param params: The transformation parameters dictionary with 'blocks' key - :param op: The remove_code operation - :return: Tuple of (modified parameters dictionary, change summary message) - """ - # Target the specific code in the specific block - expr = parse_jsonpath(f"$.blocks[?(@.id = '{op.block_id}')].codes[?(@.id = '{op.code_id}')]") - matches = expr.find(params) - - if not matches: - raise ValueError(f"Code with id '{op.code_id}' in block '{op.block_id}' does not exist") - - # Remove the code using `filter` - return expr.filter(lambda x: True, params), '' - - -def rename_code(params: dict, op: TfRenameCode, sql_dialect: str) -> tuple[dict, str]: - """ - Rename an existing code in an existing block in the transformation. - - :param params: The transformation parameters dictionary with 'blocks' key - :param op: The rename_code operation - :return: Tuple of (modified parameters dictionary, change summary message) - :raises ValueError: If block_id or code_id doesn't exist or code_name is empty/whitespace - """ - if not op.code_name.strip(): - raise ValueError('Invalid operation: code name cannot be empty') - - # Target the specific code's name field directly - expr = parse_jsonpath(f"$.blocks[?(@.id = '{op.block_id}')].codes[?(@.id = '{op.code_id}')].name") - matches = expr.find(params) - - if not matches: - raise ValueError(f"Code with id '{op.code_id}' in block '{op.block_id}' does not exist") - - return expr.update(params, op.code_name), '' - - -def set_code(params: dict, op: TfSetCode, sql_dialect: str) -> tuple[dict, str]: - """ - Set the SQL script of an existing code in an existing block in the transformation. - - :param params: The transformation parameters dictionary with 'blocks' key - :param op: The set_code operation - :return: Tuple of (modified parameters dictionary, change summary message) - """ - if not op.script.strip(): - raise ValueError('Invalid operation: script cannot be empty') - - # Target the specific code's script field directly - expr = parse_jsonpath(f"$.blocks[?(@.id = '{op.block_id}')].codes[?(@.id = '{op.code_id}')].script") - matches = expr.find(params) - - if not matches: - raise ValueError(f"Code with id '{op.code_id}' in block '{op.block_id}' does not exist") - - # Update the script field - message = f"Changed code with id '{op.code_id}' in block '{op.block_id}'" - return expr.update(params, op.script), message - - -def add_script(params: dict, op: TfAddScript, sql_dialect: str) -> tuple[dict, str]: - """ - Append or prepend SQL script text to an existing code in an existing block in the transformation. - - :param params: The transformation parameters dictionary with 'blocks' key - :param op: The add_script operation - :return: Tuple of (modified parameters dictionary, change summary message) - :raises ValueError: If block_id or code_id doesn't exist or script is empty/whitespace - """ - if not op.script.strip(): - raise ValueError('Invalid operation: script cannot be empty') - - # Target the specific code's script field directly - expr = parse_jsonpath(f"$.blocks[?(@.id = '{op.block_id}')].codes[?(@.id = '{op.code_id}')].script") - matches = expr.find(params) - - if not matches: - raise ValueError(f"Code with id '{op.code_id}' in block '{op.block_id}' does not exist") - - current_script = matches[0].value - - # Compute new script based on position - if op.position == 'start': - new_script = f'{op.script} {current_script}' if current_script else op.script - else: # 'end' - new_script = f'{current_script} {op.script}' if current_script else op.script - - # Update the script field - message = f"Added script to code with id '{op.code_id}' in block '{op.block_id}'" - return expr.update(params, new_script), message - - -def str_replace(params: dict, op: TfStrReplace, sql_dialect: str) -> tuple[dict, str]: - """ - Replace a substring in SQL statements in the transformation. - - :param params: The transformation parameters dictionary with 'blocks' key - :param op: The str_replace operation - :return: Tuple of (modified parameters dictionary, change summary message) - :raises ValueError: If search string is empty, search and replace are the same, - search string not found, or target doesn't exist - """ - if not op.search_for: - raise ValueError('Invalid operation: search string is empty') - - if op.search_for == op.replace_with: - raise ValueError(f'Invalid operation: search string and replace string are the same: "{op.search_for}"') - - # Determine the JSONPath based on scope - if op.block_id is None: - # Replace in all blocks and all codes - jsonpath = '$.blocks[*].codes[*].script' - scope = 'the transformation' - elif op.code_id is None: - # Replace in all codes of a specific block - jsonpath = f"$.blocks[?(@.id = '{op.block_id}')].codes[*].script" - scope = f'block "{op.block_id}"' - else: - # Replace in a specific code of a specific block - jsonpath = f"$.blocks[?(@.id = '{op.block_id}')].codes[?(@.id = '{op.code_id}')].script" - scope = f'code "{op.code_id}", block "{op.block_id}"' - - expr = parse_jsonpath(jsonpath) - matches = expr.find(params) - - if not matches: - raise ValueError(f'No scripts found in {scope}') - - replace_cnt = 0 - - for match in matches: - script = match.value - - if op.search_for in script: - # Count actual occurrences in this script - occurrences_in_script = script.count(op.search_for) - replace_cnt += occurrences_in_script - new_script = script.replace(op.search_for, op.replace_with) - params = match.full_path.update(params, new_script) - - # Validate that at least one replacement was made - if replace_cnt == 0: - raise ValueError(f'Search string "{op.search_for}" not found in {scope}') - - occurrence_word = 'occurrence' if replace_cnt == 1 else 'occurrences' - return params, f'Replaced {replace_cnt} {occurrence_word} of "{op.search_for}" in {scope}' diff --git a/src/keboola_mcp_server/tools/components/tools.py b/src/keboola_mcp_server/tools/components/tools.py deleted file mode 100644 index 24a51b5bf..000000000 --- a/src/keboola_mcp_server/tools/components/tools.py +++ /dev/null @@ -1,2023 +0,0 @@ -""" -Keboola Component Management Tools for MCP Server. - -This module provides the core tools for managing Keboola components and their configurations -through the Model Context Protocol (MCP) interface. It serves as the main entry point for -component-related operations in the MCP server. - -## Tool Categories - -### Component/Configuration Discovery -- `get_components`: Retrieve detailed component information including schemas -- `find_component_id`: Search for components by natural language query -- `get_configs`: Get details for specific configurations or list all configurations -- `get_config_examples`: Get sample configuration examples for a component - -### Configuration Management -- `create_config`: Create new root component configurations -- `update_config`: Update existing root configurations -- `add_config_row`: Add new configuration rows to existing configurations -- `update_config_row`: Update existing configuration rows - -### SQL Transformations -- `create_sql_transformation`: Create new SQL transformations with code blocks -- `update_sql_transformation`: Update existing SQL transformation configurations -""" - -import copy -import json -import logging -from datetime import datetime, timezone -from typing import Annotated, Any, Optional, Sequence, cast - -from fastmcp import Context -from fastmcp.exceptions import ToolError -from fastmcp.tools import FunctionTool -from httpx import HTTPStatusError -from mcp.types import ToolAnnotations -from pydantic import Field - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.storage import ConfigurationAPIResponse, JsonDict -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.links import ProjectLinksManager -from keboola_mcp_server.mcp import ( - KeboolaMcpServer, - process_concurrently, - toon_serializer_compact, - unwrap_results, -) -from keboola_mcp_server.tools.components.model import ( - Component, - ComponentSummary, - ComponentType, - ConfigParamUpdate, - ConfigToolOutput, - Configuration, - FullConfigId, - GetComponentsOutput, - GetConfigsDetailOutput, - GetConfigsListOutput, - GetConfigsOutput, - SimplifiedTfBlocks, - TfParamUpdate, - TransformationConfiguration, - VariableDefinition, -) -from keboola_mcp_server.tools.components.utils import ( - BIGQUERY_TRANSFORMATION_ID, - FOLDER_SUPPORTING_COMPONENT_IDS, - SNOWFLAKE_TRANSFORMATION_ID, - VARIABLES_COMPONENT_ID, - _apply_vars_to_parent_cfg, - add_ids, - apply_configuration_variables, - apply_folder_metadata, - build_folder_hint, - check_suitable, - clear_configuration_folder_metadata, - create_transformation_configuration, - expand_component_types, - fetch_component, - folder_field_description, - get_config_folders, - get_sql_transformation_id_from_sql_dialect, - list_configs_by_ids, - list_configs_by_types, - set_cfg_creation_metadata, - set_cfg_update_metadata, - set_configuration_folder_metadata, - set_nested_value, - update_params, - update_transformation_parameters, -) -from keboola_mcp_server.tools.constants import CONFIG_DIFF_PREVIEW_TAG -from keboola_mcp_server.tools.validation import ( - validate_processors_configuration, - validate_root_parameters_configuration, - validate_root_storage_configuration, - validate_row_parameters_configuration, - validate_row_storage_configuration, -) -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - -COMPONENT_TOOLS_TAG = 'components' - - -# ============================================================================ -# TOOL REGISTRATION -# ============================================================================ - - -def add_component_tools(mcp: KeboolaMcpServer) -> None: - """Add tools to the MCP server.""" - # Component/Configuration discovery tools - mcp.add_tool( - FunctionTool.from_function( - get_components, - tags={COMPONENT_TOOLS_TAG}, - annotations=ToolAnnotations(readOnlyHint=True), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - get_configs, - tags={COMPONENT_TOOLS_TAG}, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - get_config_examples, - tags={COMPONENT_TOOLS_TAG}, - annotations=ToolAnnotations(readOnlyHint=True), - ) - ) - - # Configuration management tools - mcp.add_tool( - FunctionTool.from_function( - create_config, - tags={COMPONENT_TOOLS_TAG}, - annotations=ToolAnnotations(destructiveHint=False), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - update_config, - tags={COMPONENT_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}, - annotations=ToolAnnotations(destructiveHint=True), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - add_config_row, - tags={COMPONENT_TOOLS_TAG}, - annotations=ToolAnnotations(destructiveHint=False), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - update_config_row, - tags={COMPONENT_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}, - annotations=ToolAnnotations(destructiveHint=True), - ) - ) - - # Sync action tools - mcp.add_tool( - FunctionTool.from_function( - run_sync_action, - tags={COMPONENT_TOOLS_TAG}, - annotations=ToolAnnotations(readOnlyHint=True), - ) - ) - - # SQL transformation tools - mcp.add_tool( - FunctionTool.from_function( - create_sql_transformation, - tags={COMPONENT_TOOLS_TAG}, - annotations=ToolAnnotations(destructiveHint=False), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - update_sql_transformation, - tags={COMPONENT_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}, - annotations=ToolAnnotations(destructiveHint=True), - ) - ) - - LOG.info('Component tools added to the MCP server.') - - -# ============================================================================ -# Configuration LISTING TOOLS -# ============================================================================ - - -@tool_errors() -async def get_configs( - ctx: Context, - component_types: Annotated[ - Sequence[ComponentType], - Field( - description=( - 'Filter by component types. Options: "application", "extractor", "transformation", "writer". ' - 'Empty list [] means ALL component types will be returned. ' - 'This parameter is IGNORED when configs is provided (non-empty) or component_ids is non-empty.' - ) - ), - ] = tuple(), - component_ids: Annotated[ - Sequence[str], - Field( - description=( - 'Filter by specific component IDs (e.g., ["keboola.ex-db-mysql", "keboola.wr-google-sheets"]). ' - 'Empty list [] uses component_types filtering instead. ' - 'When provided (non-empty) and configs is empty, lists summaries for these components. ' - 'Ignored if configs is provided.' - ) - ), - ] = tuple(), - configs: Annotated[ - Sequence[FullConfigId], - Field( - description=( - 'List of specific configurations to retrieve full details for. ' - 'Each dict must have "component_id" (str) and "configuration_id" (str). ' - 'Example: [{"component_id": "keboola.ex-db-mysql", "configuration_id": "12345"}]. ' - 'If provided (non-empty), ignores other filters and returns full details only for these configs, ' - 'grouped by component. Use this for detailed retrieval.' - ) - ), - ] = tuple(), -) -> GetConfigsOutput: - """ - Retrieves component configurations in the project with optional filtering. - - Can list summaries of multiple configurations (grouped by component) or retrieve full details - for specific configurations. - - Returns a list of components, each containing: - - Component metadata (ID, name, type, description) - - Configurations for that component (summaries by default, full details if requested) - - Links to the Keboola UI - - PARAMETER BEHAVIOR: - - If configs is provided (non-empty): Returns FULL details ONLY for those configs. - - Else if component_ids is provided (non-empty): Lists config summaries for those components. - - Else: Lists configs based on component_types (all types if empty). - - WHEN TO USE: - - For listing: Use component_types/component_ids. - - For details: Use configs (can handle multiple). - - WHEN NOT TO USE: - - Do NOT list all configs just to find a configuration by name. Use `search` with - item_types=["configuration", "transformation"] instead. - - Only use broad listing (empty component_types and component_ids) when you need - a complete inventory of all configurations in the project. - - EXAMPLES: - - List all configs (summaries): component_types=[], component_ids=[] - - List extractors (summaries): component_types=["extractor"] - - Get details for specific configs: - configs=[{"component_id": "keboola.ex-db-mysql", "configuration_id": "12345"}] - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - # Case 1: specific configs provided - return full details for those configs - if configs: - - async def fetch_config_detail(spec: FullConfigId) -> Configuration: - component_id = spec.component_id - configuration_id = spec.configuration_id - - raw_configuration = cast( - JsonDict, - await client.storage_client.configuration_detail( - component_id=component_id, configuration_id=configuration_id - ), - ) - - api_config = ConfigurationAPIResponse.model_validate(raw_configuration | {'component_id': component_id}) - api_component = await fetch_component(client=client, component_id=component_id) - component_summary = ComponentSummary.from_api_response(api_component) - - links = links_manager.get_configuration_links( - component_id=component_id, - configuration_id=configuration_id, - configuration_name=str(raw_configuration.get('name', '')), - ) - - configuration = Configuration.from_api_response( - api_config=api_config, - component=component_summary, - links=links, - ) - - # Handle transformation simplification - if component_id in (SNOWFLAKE_TRANSFORMATION_ID, BIGQUERY_TRANSFORMATION_ID): - original_parameters = TransformationConfiguration.Parameters.model_validate( - configuration.configuration_root.parameters - ) - simplified_parameters: SimplifiedTfBlocks = await original_parameters.to_simplified_parameters() - configuration.configuration_root.parameters = add_ids(simplified_parameters.model_dump()) - - return configuration - - results = await process_concurrently(configs, fetch_config_detail) - fetched_configs = unwrap_results(results, 'Failed to fetch one or more configurations') - return GetConfigsDetailOutput(configs=fetched_configs) - - # Case 2: component_ids provided - list summaries by IDs - if component_ids: - components_with_configs = await list_configs_by_ids(client, component_ids, links_manager) - # Case 3: use component_types filtering (or all types if empty) - else: - component_types = expand_component_types(component_types) - components_with_configs = await list_configs_by_types(client, component_types, links_manager) - - links = [links_manager.get_used_components_link(), links_manager.get_transformations_dashboard_link()] - return GetConfigsListOutput(components_with_configs=components_with_configs, links=links) - - -# ============================================================================ -# COMPONENT DISCOVERY TOOLS -# ============================================================================ - - -@tool_errors() -async def get_components( - ctx: Context, - component_ids: Annotated[Sequence[str], Field(description='IDs of the components')], -) -> GetComponentsOutput: - """ - Retrieves detailed information about one or more components by their IDs. - - RETURNS FOR EACH COMPONENT: - - Component metadata (name, type, description) - - Documentation and usage instructions - - Configuration JSON schema (required for creating/updating configurations) - - Links to component dashboard in Keboola UI - - WHEN TO USE: - - Before creating a new configuration: fetch the component to get its configuration schema - - Before updating a configuration: fetch the component to understand valid configuration options - - When user asks about component capabilities or documentation - - PREREQUISITES: - - You must know the component_id(s). If unknown, first use `find_component_id` or `docs` tool to discover them. - - EXAMPLES: - - User: "Create a generic extractor configuration" - → First call `find_component_id` to get the component_id, then call this tool to get the schema - - User: "What options does the Snowflake writer support?" - → Call this tool with the Snowflake writer component_id to retrieve its documentation and schema - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - async def fetch_component_with_links(component_id: str) -> Component: - api_component = await fetch_component(client=client, component_id=component_id) - component = Component.from_api_response(api_component) - component.links.append( - links_manager.get_config_dashboard_link(component_id=component_id, component_name=component.component_name) - ) - return component - - results = await process_concurrently(component_ids, fetch_component_with_links) - components = unwrap_results(results, 'Failed to fetch one or more components') - return GetComponentsOutput(components=components, links=[links_manager.get_used_components_link()]) - - -# ============================================================================ -# CONFIGURATION MANAGEMENT TOOLS -# ============================================================================ - - -@tool_errors() -async def create_sql_transformation( - ctx: Context, - name: Annotated[ - str, - Field( - description='A short, descriptive name summarizing the purpose of the SQL transformation.', - ), - ], - description: Annotated[ - str, - Field( - description=( - 'The detailed description of the SQL transformation capturing the user intent, explaining the ' - 'SQL query, and the expected output.' - ), - ), - ], - sql_code_blocks: Annotated[ - Sequence[SimplifiedTfBlocks.Block.Code], - Field( - description=( - 'The SQL query code blocks, each containing a descriptive name and an executable SQL script ' - 'written in the current SQL dialect. The query will be automatically reformatted to be more readable.' - ), - ), - ], - created_table_names: Annotated[ - Sequence[str], - Field( - description=( - 'A list of created table names if they are generated within the SQL query statements ' - '(e.g., using `CREATE TABLE ...`).' - ), - ), - ] = tuple(), - folder: Annotated[ - str, - Field(description=folder_field_description('transformation', 'transformations')), - ] = '', - variables: Annotated[ - Optional[list[VariableDefinition]], - Field( - description=( - 'Variable definitions to attach to this transformation. ' - 'Each entry specifies a name, type ("string" or "vault"), and an optional default value. ' - 'On creation, both `None` (omitted) and `[]` (empty list) mean "do not attach variables" — ' - 'no `keboola.variables` config is created. To remove variables from an existing transformation, ' - 'use `update_sql_transformation` with `variables=[]`.' - ), - ), - ] = None, -) -> ConfigToolOutput: - """ - Creates an SQL transformation using the specified name, SQL query following the current SQL dialect, a detailed - description, and a list of created table names. - - CONSIDERATIONS: - - By default, SQL transformation must create at least one table to produce a result; omit only if the user - explicitly indicates that no table creation is needed. - - Each SQL code block must include descriptive name that reflects its purpose and group one or more executable - semantically related SQL statements. - - Each SQL query statement within a code block must be executable and follow the current SQL dialect. - - Use delimited identifiers for the current SQL dialect for all identifiers and FQN references. - - When referring to the input tables within the SQL query, use fully qualified table names, which can be - retrieved using appropriate tools. - - When creating a new table within the SQL query (e.g. CREATE TABLE ...): use only the table name with - delimited identifiers, without the fully qualified path; add the plain table name without delimiters - to the `created_table_names` list. - - Unless otherwise specified by user, transformation name and description are generated based on the SQL query - and user intent. - - If there are 20 or more SQL transformations in the project, consider organizing them with a folder: existing - folder names are surfaced in the response's change_summary — use one of them or create a new one. - - USAGE: - - Use when you want to create a new SQL transformation. - - EXAMPLES: - - user_input: `Can you create a new transformation out of this sql query?` - - set the sql_code_blocks to the query, and set other parameters accordingly. - - returns the created SQL transformation configuration if successful. - - user_input: `Generate me an SQL transformation which [USER INTENT]` - - set the sql_code_blocks to the query based on the [USER INTENT], and set other parameters accordingly. - - returns the created SQL transformation configuration if successful. - """ - - # Get the SQL dialect to use the correct transformation ID (Snowflake or BigQuery) - # This can raise an exception if workspace is not set or different backend than BigQuery or Snowflake is used - sql_dialect = await WorkspaceManager.from_state(ctx.session.state).get_sql_dialect() - component_id = get_sql_transformation_id_from_sql_dialect(sql_dialect) - LOG.info(f'Creating transformation. SQL dialect: {sql_dialect}, using transformation ID: {component_id}') - - # Process the data to be stored in the transformation configuration - parameters(sql statements) - # and storage (input and output tables) - transformation_configuration_payload = await create_transformation_configuration( - codes=sql_code_blocks, transformation_name=name, output_tables=created_table_names, sql_dialect=sql_dialect - ) - - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - LOG.info(f'Creating new transformation configuration: {name} for component: {component_id}.') - - new_raw_transformation_configuration = await client.storage_client.configuration_create( - component_id=component_id, - name=name, - description=description, - configuration=transformation_configuration_payload.model_dump(by_alias=True), - ) - - configuration_id = str(new_raw_transformation_configuration['id']) - - await set_cfg_creation_metadata( - client=client, - component_id=component_id, - configuration_id=configuration_id, - ) - - folder = folder.strip() - if folder: - try: - await set_configuration_folder_metadata(client, component_id, configuration_id, folder) - except Exception: - LOG.warning( - 'Unable to set folder metadata for component "%s", configuration "%s".', - component_id, - configuration_id, - ) - change_summary = None - else: - try: - total, existing_folders, lower_bound = await get_config_folders(client, component_id) - change_summary = build_folder_hint( - total, - existing_folders, - 'SQL transformations', - 'update_sql_transformation', - lower_bound=lower_bound, - ) - except Exception: - LOG.warning( - 'Unable to fetch transformation folders for component "%s" when creating configuration "%s".', - component_id, - configuration_id, - ) - change_summary = None - - LOG.info(f'Created new transformation "{component_id}" with configuration id ' f'"{configuration_id}".') - - vars_result = None - if variables: - vars_result = await apply_configuration_variables(client, component_id, configuration_id, variables) - if vars_result is not None: - await set_cfg_update_metadata(client, component_id, configuration_id, vars_result['version']) - - links = links_manager.get_transformation_links( - transformation_type=component_id, - transformation_id=configuration_id, - transformation_name=name, - ) - - return ConfigToolOutput( - component_id=component_id, - configuration_id=configuration_id, - description=description, - timestamp=datetime.now(timezone.utc), - success=True, - links=links, - version=(vars_result or new_raw_transformation_configuration)['version'], - change_summary=change_summary, - ) - - -@tool_errors() -async def update_sql_transformation( - ctx: Context, - change_description: Annotated[ - str, - Field( - description=( - 'A clear, human-readable summary of what changed in this transformation update. ' - 'Be specific: e.g., "Added JOIN with customers table", "Updated WHERE clause to filter active records".' - ), - ), - ], - configuration_id: Annotated[str, Field(description='The ID of the transformation configuration to update.')], - name: Annotated[ - str, - Field( - description=( - 'New name for the transformation. Only provide if changing the name. ' - 'Name should be short (typically under 50 characters) and descriptive.' - ) - ), - ] = '', - description: Annotated[ - str, - Field( - description=( - 'New detailed description for the transformation. Only provide if changing the description. ' - 'Should explain what the transformation does, data sources, and business logic. ' - 'Leave empty to preserve the original description.' - ), - ), - ] = '', - parameter_updates: Annotated[ - list[TfParamUpdate], - Field( - description=( - 'List of operations to apply to the transformation structure (blocks, codes, SQL scripts). ' - 'Each operation modifies specific elements using block_id and code_id identifiers. ' - 'Only provide if updating SQL code or block structure - do not use for description or storage changes. ' - '\n\n' - 'IMPORTANT: Use get_configs first to retrieve the current transformation structure and identify ' - 'the block_id and code_id values needed for your operations. IDs are automatically assigned.\n' - '\n' - 'Available operations:\n' - '1. add_block: Add a new block to the transformation\n' - ' - Fields: op="add_block", block={name, codes}, position="start"|"end"\n' - '2. remove_block: Remove an existing block\n' - ' - Fields: op="remove_block", block_id (e.g., "b0")\n' - '3. rename_block: Rename an existing block\n' - ' - Fields: op="rename_block", block_id (e.g., "b0"), block_name\n' - '4. add_code: Add a new code block to an existing block\n' - ' - Fields: op="add_code", block_id (e.g., "b0"), code={name, script}, position="start"|"end"\n' - '5. remove_code: Remove an existing code block\n' - ' - Fields: op="remove_code", block_id (e.g., "b0"), code_id (e.g., "b0.c0")\n' - '6. rename_code: Rename an existing code block\n' - ' - Fields: op="rename_code", block_id (e.g., "b0"), code_id (e.g., "b0.c0"), code_name\n' - '7. set_code: Replace the entire SQL script of a code block\n' - ' - Fields: op="set_code", block_id (e.g., "b0"), code_id (e.g., "b0.c0"), script\n' - '8. add_script: Append or prepend SQL to a code block\n' - ' - Fields: op="add_script", block_id (e.g., "b0"), code_id (e.g., "b0.c0"), script,' - ' position="start"|"end"\n' - '9. str_replace: Replace substring in SQL scripts\n' - ' - Fields: op="str_replace", search_for, replace_with, block_id (optional), code_id (optional)\n' - ' - If block_id omitted: replaces in all blocks\n' - ' - If code_id omitted: replaces in all codes of the specified block\n' - ), - ), - ] = None, - storage: Annotated[ - dict[str, Any], - Field( - description=( - 'Complete storage configuration for transformation input/output table mappings. ' - 'Only provide if updating storage mappings - this replaces the ENTIRE storage configuration. ' - '\n\n' - 'When to use:\n' - '- Adding/removing input tables for the transformation\n' - '- Modifying output table mappings and destinations\n' - '- Changing table aliases used in SQL\n' - '\n' - 'Important:\n' - '- Must conform to transformation storage schema (input/output tables)\n' - '- Replaces ALL existing storage config - include all mappings you want to keep\n' - '- Use get_configs first to see current storage configuration\n' - '- Leave unfilled to preserve existing storage configuration' - ) - ), - ] = None, - folder: Annotated[ - Optional[str], - Field(description=folder_field_description('transformation', 'transformations')), - ] = None, - variables: Annotated[ - Optional[list[VariableDefinition]], - Field( - description=( - 'Variable definitions for this transformation. ' - 'Provide a non-empty list to create or replace all variable definitions. ' - 'Provide an empty list ([]) to remove all variables. ' - 'Omit (None) to leave existing variables unchanged.' - ), - ), - ] = None, -) -> ConfigToolOutput: - """ - Updates an existing SQL transformation configuration by modifying its SQL code, storage mappings, - name or description. - - This tool allows PARTIAL parameter updates for transformation SQL blocks and code - you only need to provide - the operations you want to perform. All other fields will remain unchanged. - Use this for modifying SQL transformations created with create_sql_transformation. - - WHEN TO USE: - - SQL transformations only (Snowflake/BigQuery); use update_config for Python/R transformations - - Modifying SQL queries in transformation (add/edit/remove SQL statements) - - Updating transformation block or code block names - - Changing input/output table mappings for the transformation - - Updating the transformation name or description - - Any combination of the above - - PREREQUISITES: - - Transformation must already exist (use create_sql_transformation for new transformations) - - You must know the configuration_id of the transformation - - SQL dialect is determined automatically from the workspace - - CRITICAL: Use get_configs first to see the current transformation structure and get block_id/code_id values - - TRANSFORMATION STRUCTURE: - A transformation has this hierarchy: - transformation - └─ blocks[] - List of transformation blocks (each has a unique block_id) - └─ block.name - Descriptive name for the block - └─ block.codes[] - List of code blocks within the block (each has a unique code_id) - └─ code.name - Descriptive name for the code block - └─ code.script - SQL script (string with SQL statements) - - Example structure from get_configs: - { - "blocks": [ - { - "id": "b0", ← block_id needed for operations (format: b{index}) - "name": "Data Preparation", - "codes": [ - { - "id": "b0.c0", ← code_id needed for operations (format: b{block_index}.c{code_index}) - "name": "Load customers", - "script": "SELECT * FROM customers WHERE status = 'active';" - } - ] - } - ] - } - - PARAMETER UPDATE OPERATIONS: - All operations use block_id and code_id to identify elements (get these from get_configs first). - - ID Format: - - block_id: "b0", "b1", "b2", etc. (format: b{index}) - - code_id: "b0.c0", "b0.c1", "b1.c0", etc. (format: b{block_index}.c{code_index}) - - 1. BLOCK OPERATIONS: - - add_block: Create a new block in the transformation - {"op": "add_block", "block": {"name": "New Block", "codes": []}, "position": "end"} - - - remove_block: Delete an entire block - {"op": "remove_block", "block_id": "b0"} - - - rename_block: Change a block's name - {"op": "rename_block", "block_id": "b2", "block_name": "Updated Name"} - - 2. CODE BLOCK OPERATIONS: - - add_code: Create a new code block within an existing block - {"op": "add_code", "block_id": "b1", "code": {"name": "New Code", "script": "SELECT 1;"}, "position": "end"} - - - remove_code: Delete a code block - {"op": "remove_code", "block_id": "b0", "code_id": "b0.c0"} - - - rename_code: Change a code block's name - {"op": "rename_code", "block_id": "b1", "code_id": "b1.c2", "code_name": "Updated Name"} - - 3. SQL SCRIPT OPERATIONS: - - set_code: Replace the entire SQL script (overwrites existing) - {"op": "set_code", "block_id": "b0", "code_id": "b0.c0", "script": "SELECT * FROM new_table;"} - - - add_script: Append or prepend SQL to existing script (preserves existing) - {"op": "add_script", "block_id": "b2", "code_id": "b2.c1", "script": "WHERE date > '2024-01-01'", - "position": "end"} - - - str_replace: Find and replace text in SQL scripts - {"op": "str_replace", "search_for": "old_table", "replace_with": "new_table", "block_id": "b0",' - "code_id": "b0.c0"} - - Omit code_id to replace in all codes of a block - - Omit both block_id and code_id to replace everywhere - - IMPORTANT CONSIDERATIONS: - - Parameter updates are PARTIAL - only the operations you specify are applied - - All other parts of the transformation remain unchanged - - Each SQL script must be executable and follow the current SQL dialect: - - Use delimited identifiers for the current SQL dialect. - - Never mix delimiter styles within a single query. - - Storage configuration is COMPLETE REPLACEMENT - include ALL mappings you want to keep - - Leave updated_description empty to preserve the original description - - SCHEMA CHANGES: Destructive schema changes (removing columns, changing types, renaming columns) require - manually deleting the output table before running the updated transformation to avoid schema mismatch errors. - Non-destructive changes (adding columns) typically do not require table deletion. - - WORKFLOW: - 1. Call get_configs to retrieve current transformation structure and identify block_id/code_id values - 2. Identify what needs to change (SQL code, storage, description) - 3. For SQL changes: Prepare parameter_updates list with targeted operations - 4. For storage changes: Build complete storage configuration (include all mappings) - 5. Call update_sql_transformation with change_description and only the fields to change - - EXAMPLE WORKFLOWS: - - Example 1 - Update SQL script in existing code block: - Step 1: Get current config - result = get_configs(component_id="keboola.snowflake-transformation", configuration_id="12345") - # Note the block_id (e.g., "b0") and code_id (e.g., "b0.c1") from result - - Step 2: Update the SQL - update_sql_transformation( - configuration_id="12345", - change_description="Updated WHERE clause to filter active customers only", - parameter_updates=[ - { - "op": "set_code", - "block_id": "b0", # from step 1 - "code_id": "b0.c0", # from step 1 - "script": "SELECT * FROM customers WHERE status = 'active' AND region = 'US';" - } - ] - ) - - Example 2 - Append a new code block to the second block of an existing transformation: - update_sql_transformation( - configuration_id="12345", - change_description="Added aggregation step", - parameter_updates=[ - { - "op": "add_code", - "block_id": "b1", # second block - "code": { - "name": "Aggregate Sales", - "script": "SELECT customer_id, SUM(amount) as total FROM orders GROUP BY customer_id;" - }, - "position": "end" - } - ] - ) - - Example 3 - Replace table name across all SQL scripts: - update_sql_transformation( - configuration_id="12345", - change_description="Renamed source table from old_customers to customers", - parameter_updates=[ - { - "op": "str_replace", - "search_for": "old_customers", - "replace_with": "customers" - # No block_id or code_id = applies to all scripts - } - ] - ) - - Example 4 - Update storage mappings: - update_sql_transformation( - configuration_id="12345", - change_description="Added new input table", - storage={ - "input": { - "tables": [ - { - "source": "in.c-main.customers", - "destination": "customers" - }, - { - "source": "in.c-main.orders", - "destination": "orders" - } - ] - }, - "output": { - "tables": [ - { - "source": "result", - "destination": "out.c-main.customer_summary" - } - ] - } - } - ) - """ - client = KeboolaClient.from_state(ctx.session.state) - workspace_manager = WorkspaceManager.from_state(ctx.session.state) - sql_dialect = await workspace_manager.get_sql_dialect() - sql_transformation_id = get_sql_transformation_id_from_sql_dialect(sql_dialect) - - links_manager = await ProjectLinksManager.from_client(client) - - LOG.info( - f'Updating transformation: {sql_transformation_id} with config ID: {configuration_id}. ' - f'SQL dialect: {sql_dialect}' - ) - - _, updated_configuration, msg, *_ = await update_sql_transformation_internal( - client=client, - workspace_manager=workspace_manager, - change_description=change_description, - configuration_id=configuration_id, - name=name, - description=description, - parameter_updates=parameter_updates, - storage=storage, - ) - - vars_config_id_to_delete: str | None = None - if variables is not None: - _, vars_config_id_to_delete = await _apply_vars_to_parent_cfg( - client, sql_transformation_id, configuration_id, variables, updated_configuration - ) - - updated_raw_configuration = await client.storage_client.configuration_update( - component_id=sql_transformation_id, - configuration_id=configuration_id, - configuration=updated_configuration, - change_description=change_description, - updated_name=name, - updated_description=description, - ) - - if vars_config_id_to_delete: - await client.storage_client.configuration_delete( - component_id=VARIABLES_COMPONENT_ID, - configuration_id=vars_config_id_to_delete, - skip_trash=True, - ) - - folder_hint = None - if folder is None: - try: - total, existing_folders, lower_bound = await get_config_folders(client, sql_transformation_id) - folder_hint = build_folder_hint( - total, - existing_folders, - 'SQL transformations', - 'update_sql_transformation', - lower_bound=lower_bound, - ) - except Exception: - LOG.warning( - 'Unable to fetch transformation folders for component "%s" when updating configuration "%s".', - sql_transformation_id, - configuration_id, - ) - else: - folder_stripped = folder.strip() - if folder_stripped: - try: - await set_configuration_folder_metadata( - client, sql_transformation_id, configuration_id, folder_stripped - ) - except Exception as exc: - LOG.warning( - 'Unable to set folder metadata for component "%s", configuration "%s".', - sql_transformation_id, - configuration_id, - exc_info=exc, - ) - else: - try: - await clear_configuration_folder_metadata(client, sql_transformation_id, configuration_id) - except Exception as exc: - LOG.warning( - 'Unable to clear folder metadata for component "%s", configuration "%s".', - sql_transformation_id, - configuration_id, - exc_info=exc, - ) - - await set_cfg_update_metadata( - client=client, - component_id=sql_transformation_id, - configuration_id=configuration_id, - configuration_version=updated_raw_configuration.get('version'), - ) - - change_summary = ' '.join(filter(None, [msg, folder_hint])) or None - - links = links_manager.get_transformation_links( - transformation_type=sql_transformation_id, - transformation_id=configuration_id, - transformation_name=updated_raw_configuration.get('name') or '', - ) - - LOG.info( - f'Updated transformation configuration: {updated_raw_configuration["id"]} for ' - f'component: {sql_transformation_id}.' - ) - - return ConfigToolOutput( - component_id=sql_transformation_id, - configuration_id=configuration_id, - description=updated_raw_configuration.get('description') or '', - timestamp=datetime.now(timezone.utc), - success=True, - links=links, - version=updated_raw_configuration['version'], - change_summary=change_summary, - ) - - -# This function must use exactly the same parameters as update_sql_transformation() function. -# Except for the `ctx` and `client` parameters. -async def update_sql_transformation_internal( - *, - client: KeboolaClient, - workspace_manager: WorkspaceManager, - configuration_id: str, - change_description: str, - name: str = '', - description: str = '', - parameter_updates: list[TfParamUpdate] | None = None, - storage: dict[str, Any] | None = None, - folder: Optional[str] = None, -) -> tuple[JsonDict, JsonDict, str, dict | None]: - sql_dialect = await workspace_manager.get_sql_dialect() - sql_transformation_id = get_sql_transformation_id_from_sql_dialect(sql_dialect) - try: - config_details = await client.storage_client.configuration_detail( - component_id=sql_transformation_id, configuration_id=configuration_id - ) - except HTTPStatusError as e: - if e.response.status_code == 404: - raise ToolError( - f"Configuration '{configuration_id}' was not found under SQL transformation component " - f"'{sql_transformation_id}'. If this is a Python or R transformation, use 'update_config' " - f"with component_id 'keboola.python-transformation-v2' or 'keboola.r-transformation-v2' " - f"instead of 'update_sql_transformation'." - ) from e - raise - api_component = await fetch_component(client=client, component_id=sql_transformation_id) - transformation = Component.from_api_response(api_component) - - updated_configuration = cast(JsonDict, config_details.get('configuration', {})) - updated_configuration = copy.deepcopy(updated_configuration) - - msg: str = '' - - if parameter_updates: - current_param_dict = updated_configuration.get('parameters', {}) - current_raw_parameters = TransformationConfiguration.Parameters.model_validate(current_param_dict) - simplified_parameters = await current_raw_parameters.to_simplified_parameters() - - updated_params, msg = update_transformation_parameters( - parameters=simplified_parameters, - updates=parameter_updates, - sql_dialect=sql_dialect, - ) - updated_raw_parameters = await updated_params.to_raw_parameters() - - parameters_cfg = validate_root_parameters_configuration( - component=transformation, - parameters=updated_raw_parameters.model_dump(exclude_none=True), - initial_message='Applying the "parameter_updates" resulted in an invalid configuration.', - configuration_id=configuration_id, - ) - updated_configuration['parameters'] = parameters_cfg - - if storage is not None: - storage_cfg = validate_root_storage_configuration( - component=transformation, - storage=storage, - initial_message='The "storage" field is not valid.', - configuration_id=configuration_id, - ) - updated_configuration['storage'] = storage_cfg - - folder_preview: dict | None = None - if folder is not None: - normalized_folder = folder.strip() - try: - current_metadata = await client.storage_client.configuration_metadata_get( - component_id=sql_transformation_id, configuration_id=configuration_id - ) - current_folder = next( - ( - m.get('value', '') - for m in current_metadata - if m.get('key') == MetadataField.CONFIGURATION_FOLDER_NAME - ), - '', - ) - if normalized_folder != current_folder: - folder_preview = {'original_folder': current_folder, 'updated_folder': normalized_folder} - except Exception as e: - LOG.warning( - 'Failed to fetch configuration metadata for folder preview ' - '(component_id=%s, configuration_id=%s): %s. Proceeding without folder preview.', - sql_transformation_id, - configuration_id, - e, - ) - - return config_details, updated_configuration, msg, folder_preview - - -@tool_errors() -async def create_config( - ctx: Context, - name: Annotated[ - str, - Field( - description='A short, descriptive name summarizing the purpose of the component configuration.', - ), - ], - description: Annotated[ - str, - Field( - description=( - 'The detailed description of the component configuration explaining its purpose and functionality.' - ), - ), - ], - component_id: Annotated[str, Field(description='The ID of the component for which to create the configuration.')], - parameters: Annotated[ - dict[str, Any], - Field(description='The component configuration parameters, adhering to the configuration_schema'), - ], - storage: Annotated[ - dict[str, Any], - Field( - description=( - 'The table and/or file input / output mapping of the component configuration. ' - 'It is present only for components that have tables or file input mapping defined' - ), - ), - ] = None, - processors_before: Annotated[ - list[dict[str, Any]], - Field(description='The list of processors that will run before the configured component runs.'), - ] = None, - processors_after: Annotated[ - list[dict[str, Any]], - Field(description='The list of processors that will run after the configured component runs.'), - ] = None, - variables: Annotated[ - Optional[list[VariableDefinition]], - Field( - description=( - 'Variable definitions to attach to this configuration. ' - 'Each entry specifies a name, type ("string" or "vault"), and an optional default value. ' - 'On creation, both `None` (omitted) and `[]` (empty list) mean "do not attach variables" — ' - 'no `keboola.variables` config is created. To remove variables from an existing configuration, ' - 'use `update_config` with `variables=[]`.' - ), - ), - ] = None, -) -> ConfigToolOutput: - """ - Creates a root component configuration using the specified name, component ID, configuration JSON, and description. - - BEFORE CALLING - REQUIRED STEPS: - 1. Call `get_components([component_id])` to retrieve the component's `configuration_schema`. - 2. Read `configuration_schema.required` to find ALL mandatory top-level fields. - 3. Call `get_config_examples(component_id)` to see real-world parameter examples. - 4. Populate `parameters` with every required field before calling this tool. - Skipping these steps will cause a schema validation error. - - USAGE: - - Use when you want to create a new root configuration for a specific component. - - WHEN NOT TO USE: - - `keboola.orchestrator` / `keboola.flow` → use flows tools - - `keboola.data-apps` → use data applications tools - - `keboola.snowflake-transformation` / `keboola.google-bigquery-transformation` → use SQL transformation tools - - EXAMPLES: - - user_input: `Create a new configuration for component X with these settings` - - set the component_id and configuration parameters accordingly - - returns the created component configuration if successful. - """ - check_suitable('create_config', component_id) - - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - LOG.info(f'Creating new configuration: {name} for component: {component_id}.') - - api_component = await fetch_component(client=client, component_id=component_id) - component = Component.from_api_response(api_component) - - storage_cfg = validate_root_storage_configuration( - component=component, - storage=storage, - initial_message='The "storage" field is not valid.', - ) - parameters = validate_root_parameters_configuration( - component=component, - parameters=parameters, - initial_message='The "parameters" field is not valid.', - ) - - configuration_payload = {'storage': storage_cfg, 'parameters': parameters} - - if processors_before: - processors_before = await validate_processors_configuration( - client=client, - processors=processors_before, - initial_message='The "processors_before" field is not valid.', - ) - set_nested_value(configuration_payload, 'processors.before', processors_before) - - if processors_after: - processors_after = await validate_processors_configuration( - client=client, - processors=processors_after, - initial_message='The "processors_after" field is not valid.', - ) - set_nested_value(configuration_payload, 'processors.after', processors_after) - - new_raw_configuration = cast( - dict[str, Any], - await client.storage_client.configuration_create( - component_id=component_id, - name=name, - description=description, - configuration=configuration_payload, - ), - ) - - configuration_id = new_raw_configuration['id'] - - LOG.info(f'Created new configuration for component "{component_id}" with configuration id "{configuration_id}".') - - await set_cfg_creation_metadata(client, component_id, configuration_id) - - vars_result = None - if variables: - vars_result = await apply_configuration_variables(client, component_id, configuration_id, variables) - if vars_result is not None: - await set_cfg_update_metadata(client, component_id, configuration_id, vars_result['version']) - - links = links_manager.get_configuration_links( - component_id=component_id, configuration_id=configuration_id, configuration_name=name - ) - - return ConfigToolOutput( - component_id=component_id, - configuration_id=configuration_id, - description=description, - version=(vars_result or new_raw_configuration)['version'], - timestamp=datetime.now(timezone.utc), - success=True, - links=links, - ) - - -@tool_errors() -async def add_config_row( - ctx: Context, - name: Annotated[ - str, - Field( - description='A short, descriptive name summarizing the purpose of the component configuration.', - ), - ], - description: Annotated[ - str, - Field( - description=( - 'The detailed description of the component configuration explaining its purpose and functionality.' - ), - ), - ], - component_id: Annotated[str, Field(description='The ID of the component for which to create the configuration.')], - configuration_id: Annotated[ - str, - Field( - description='The ID of the configuration for which to create the configuration row.', - ), - ], - parameters: Annotated[ - dict[str, Any], - Field(description='The component row configuration parameters, adhering to the configuration_row_schema'), - ], - storage: Annotated[ - dict[str, Any], - Field( - description=( - 'The table and/or file input / output mapping of the component configuration. ' - 'It is present only for components that have tables or file input mapping defined' - ), - ), - ] = None, - processors_before: Annotated[ - list[dict[str, Any]], - Field(description='The list of processors that will run before the configured component row runs.'), - ] = None, - processors_after: Annotated[ - list[dict[str, Any]], - Field(description='The list of processors that will run after the configured component row runs.'), - ] = None, -) -> ConfigToolOutput: - """ - Creates a component configuration row in the specified configuration_id, using the specified name, - component ID, configuration JSON, and description. - - BEFORE CALLING - REQUIRED STEPS: - 1. Call `get_components([component_id])` to retrieve the component's `configuration_row_schema`. - 2. Read `configuration_row_schema.required` to find ALL mandatory top-level fields. - 3. Call `get_config_examples(component_id)` to see real-world row parameter examples. - 4. Populate `parameters` with every required field before calling this tool. - Skipping these steps will cause a schema validation error. - - USAGE: - - Use when you want to create a new row configuration for a specific component configuration. - - WHEN NOT TO USE: - - `keboola.orchestrator` / `keboola.flow` → use flows tools - - `keboola.data-apps` → use data applications tools - - `keboola.snowflake-transformation` / `keboola.google-bigquery-transformation` → use SQL transformation tools - - EXAMPLES: - - user_input: `Create a new configuration row for component X with these settings` - - set the component_id, configuration_id and configuration parameters accordingly - - returns the created component configuration if successful. - """ - check_suitable('add_config_row', component_id) - - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - LOG.info( - f'Creating new configuration row: {name} for component: {component_id} ' - f'and configuration {configuration_id}.' - ) - - api_component = await fetch_component(client=client, component_id=component_id) - component = Component.from_api_response(api_component) - - storage_cfg = validate_row_storage_configuration( - component=component, - storage=storage, - initial_message='The "storage" field is not valid.', - configuration_id=configuration_id, - ) - parameters = validate_row_parameters_configuration( - component=component, - parameters=parameters, - initial_message='The "parameters" field is not valid.', - configuration_id=configuration_id, - ) - - configuration_payload = {'storage': storage_cfg, 'parameters': parameters} - - if processors_before: - processors_before = await validate_processors_configuration( - client=client, - processors=processors_before, - initial_message='The "processors_before" field is not valid.', - ) - set_nested_value(configuration_payload, 'processors.before', processors_before) - - if processors_after: - processors_after = await validate_processors_configuration( - client=client, - processors=processors_after, - initial_message='The "processors_after" field is not valid.', - ) - set_nested_value(configuration_payload, 'processors.after', processors_after) - - new_raw_configuration = cast( - dict[str, Any], - await client.storage_client.configuration_row_create( - component_id=component_id, - config_id=configuration_id, - name=name, - description=description, - configuration=configuration_payload, - ), - ) - - LOG.info( - f'Created new configuration for component "{component_id}" with configuration id ' f'"{configuration_id}".' - ) - - await set_cfg_update_metadata( - client=client, - component_id=component_id, - configuration_id=configuration_id, - configuration_version=new_raw_configuration['version'], - ) - - links = links_manager.get_configuration_links( - component_id=component_id, - configuration_id=configuration_id, - configuration_name=name, - ) - - return ConfigToolOutput( - component_id=component_id, - configuration_id=configuration_id, - description=description, - version=new_raw_configuration['version'], - timestamp=datetime.now(timezone.utc), - success=True, - links=links, - ) - - -@tool_errors() -async def update_config( - ctx: Context, - change_description: Annotated[ - str, - Field( - description=( - 'A clear, human-readable summary of what changed in this update. ' - 'Be specific: e.g., "Updated API key", "Added customers table to input mapping".' - ), - ), - ], - component_id: Annotated[str, Field(description='The ID of the component the configuration belongs to.')], - configuration_id: Annotated[str, Field(description='The ID of the configuration to update.')], - name: Annotated[ - str, - Field( - description=( - 'New name for the configuration. Only provide if changing the name. ' - 'Name should be short (typically under 50 characters) and descriptive.' - ) - ), - ] = '', - description: Annotated[ - str, - Field( - description=( - 'New detailed description for the configuration. Only provide if changing the description. ' - 'Should explain the purpose, data sources, and behavior of this configuration. ' - 'Leave empty to preserve the original description.' - ), - ), - ] = '', - parameter_updates: Annotated[ - list[ConfigParamUpdate], - Field( - description=( - 'List of granular parameter update operations to apply. ' - 'Each operation (set, str_replace, remove, list_append) modifies a specific ' - 'value using JSONPath notation. Only provide if updating parameters -' - ' do not use for changing description, storage or processors. ' - 'Paths are relative to the `parameters` object, not the configuration root ' - '(e.g. use `tables`, not `parameters.tables`). ' - 'Prefer simple JSONPaths (e.g., "array_param[1]", "object_param.key") ' - 'and make the smallest possible updates - only change what needs changing. ' - 'In case you need to replace the whole parameters section, you can use the `set` operation ' - 'with `$` as path.' - ), - ), - ] = None, - storage: Annotated[ - dict[str, Any], - Field( - description=( - 'Complete storage configuration containing input/output table and file mappings. ' - 'Only provide if updating storage mappings - this replaces the ENTIRE storage configuration. ' - '\n\n' - 'When to use:\n' - '- Adding/removing input or output tables\n' - '- Modifying table/file mappings\n' - '- Updating table destinations or sources\n' - '\n' - 'Important:\n' - '- Not applicable for row-based components (they use row-level storage)\n' - '- Must conform to the Keboola storage schema\n' - '- Replaces ALL existing storage config - include all mappings you want to keep\n' - '- Use get_configs first to see current storage configuration\n' - '- Leave unfilled to preserve existing storage configuration' - ) - ), - ] = None, - processors_before: Annotated[ - list[dict[str, Any]], - Field(description='The list of processors that will run before the configured component row runs.'), - ] = None, - processors_after: Annotated[ - list[dict[str, Any]], - Field(description='The list of processors that will run after the configured component row runs.'), - ] = None, - folder: Annotated[ - Optional[str], - Field(description=folder_field_description('configuration', 'configurations')), - ] = None, - variables: Annotated[ - Optional[list[VariableDefinition]], - Field( - description=( - 'Variable definitions for this configuration. ' - 'Provide a non-empty list to create or replace all variable definitions. ' - 'Provide an empty list ([]) to remove all variables. ' - 'Omit (None) to leave existing variables unchanged.' - ), - ), - ] = None, -) -> ConfigToolOutput: - """ - Updates an existing root component configuration by modifying its parameters, storage mappings, name or description. - - This tool allows PARTIAL parameter updates - you only need to provide the fields you want to change. - All other fields will remain unchanged. - Use this tool when modifying existing configurations; for configuration rows, use update_config_row instead. - - WHEN TO USE: - - Modifying configuration parameters (credentials, settings, API keys, etc.) - - Updating storage mappings (input/output tables or files) - - Changing configuration name or description - - Any combination of the above - - WHEN NOT TO USE: - - `keboola.orchestrator` / `keboola.flow` → use flows tools - - `keboola.data-apps` → use data applications tools - - `keboola.snowflake-transformation` / `keboola.google-bigquery-transformation` → use SQL transformation tools - - PREREQUISITES: - - Configuration must already exist (use create_config for new configurations) - - You must know both component_id and configuration_id - - For parameter updates: Review the component's root_configuration_schema using get_components. - - For storage updates: Ensure mappings are valid for the component type - - IMPORTANT CONSIDERATIONS: - - Parameter updates are PARTIAL - only specify fields you want to change - - parameter_updates supports granular operations: set keys, replace strings, remove keys, or append to lists - - Parameters must conform to the component's root_configuration_schema - - Validate schemas before calling: use get_components to retrieve root_configuration_schema - - For row-based components, this updates the ROOT only (use update_config_row for individual rows) - - WORKFLOW: - 1. Retrieve current configuration using get_configs (to understand current state) - 2. Identify specific parameters/storage mappings to modify - 3. Prepare parameter_updates list with targeted operations - 4. Call update_config with only the fields to change - """ - client = KeboolaClient.from_state(ctx.session.state) - - links_manager = await ProjectLinksManager.from_client(client) - - LOG.info(f'Updating configuration for component: {component_id} and configuration ID {configuration_id}.') - - _, configuration_payload = await update_config_internal( - client=client, - change_description=change_description, - component_id=component_id, - configuration_id=configuration_id, - name=name, - description=description, - parameter_updates=parameter_updates, - storage=storage, - processors_before=processors_before, - processors_after=processors_after, - ) - - vars_config_id_to_delete: str | None = None - if variables is not None: - _, vars_config_id_to_delete = await _apply_vars_to_parent_cfg( - client, component_id, configuration_id, variables, configuration_payload - ) - - updated_raw_configuration = await client.storage_client.configuration_update( - component_id=component_id, - configuration_id=configuration_id, - configuration=configuration_payload, - change_description=change_description, - updated_name=name, - updated_description=description, - ) - - if vars_config_id_to_delete: - await client.storage_client.configuration_delete( - component_id=VARIABLES_COMPONENT_ID, - configuration_id=vars_config_id_to_delete, - skip_trash=True, - ) - - LOG.info(f'Updated configuration for component "{component_id}" with configuration id ' f'"{configuration_id}".') - - folder_hint = ( - await apply_folder_metadata(client, component_id, configuration_id, folder, 'configurations', 'update_config') - if component_id in FOLDER_SUPPORTING_COMPONENT_IDS - else None - ) - - await set_cfg_update_metadata( - client=client, - component_id=component_id, - configuration_id=configuration_id, - configuration_version=updated_raw_configuration.get('version'), - ) - - links = links_manager.get_configuration_links( - component_id=component_id, - configuration_id=configuration_id, - configuration_name=updated_raw_configuration.get('name') or '', - ) - - return ConfigToolOutput( - component_id=component_id, - configuration_id=configuration_id, - description=updated_raw_configuration.get('description') or '', - timestamp=datetime.now(timezone.utc), - success=True, - links=links, - version=updated_raw_configuration['version'], - change_summary=folder_hint, - ) - - -# This function must use exactly the same parameters as update_config() function, -# except for `ctx`, `client`, and `folder` (folder is metadata-only, not a payload field). -async def update_config_internal( - *, - client: KeboolaClient, - change_description: str, - component_id: str, - configuration_id: str, - name: str = '', - description: str = '', - parameter_updates: list[ConfigParamUpdate] | None = None, - storage: dict[str, Any] | None = None, - processors_before: list[dict[str, Any]] | None = None, - processors_after: list[dict[str, Any]] | None = None, -) -> tuple[JsonDict, JsonDict]: - check_suitable('update_config', component_id) - - current_config = await client.storage_client.configuration_detail( - component_id=component_id, configuration_id=configuration_id - ) - api_component = await fetch_component(client=client, component_id=component_id) - component = Component.from_api_response(api_component) - - configuration_payload = cast(JsonDict, current_config.get('configuration', {})) - configuration_payload = copy.deepcopy(configuration_payload) - - if storage is not None: - storage_cfg = validate_root_storage_configuration( - component=component, - storage=storage, - initial_message='The "storage" field is not valid.', - configuration_id=configuration_id, - ) - configuration_payload['storage'] = storage_cfg - - if processors_before is not None: - processors_before = await validate_processors_configuration( - client=client, - processors=processors_before, - initial_message='The "processors_before" field is not valid.', - ) - set_nested_value(configuration_payload, 'processors.before', processors_before) - - if processors_after is not None: - processors_after = await validate_processors_configuration( - client=client, - processors=processors_after, - initial_message='The "processors_after" field is not valid.', - ) - set_nested_value(configuration_payload, 'processors.after', processors_after) - - if parameter_updates: - current_params = configuration_payload.get('parameters', {}) - updated_params = update_params(current_params, parameter_updates) - - parameters_cfg = validate_root_parameters_configuration( - component=component, - parameters=updated_params, - initial_message='Applying the "parameter_updates" resulted in an invalid configuration.', - configuration_id=configuration_id, - ) - configuration_payload['parameters'] = parameters_cfg - - return current_config, configuration_payload - - -@tool_errors() -async def update_config_row( - ctx: Context, - change_description: Annotated[ - str, - Field(description=('A clear, human-readable summary of what changed in this row update. Be specific.')), - ], - component_id: Annotated[str, Field(description='The ID of the component the configuration belongs to.')], - configuration_id: Annotated[ - str, - Field(description='The ID of the parent configuration containing the row to update.'), - ], - configuration_row_id: Annotated[str, Field(description='The ID of the specific configuration row to update.')], - name: Annotated[ - str, - Field( - description=( - 'New name for the configuration row. Only provide if changing the name. ' - 'Name should be short (typically under 50 characters) and descriptive of this specific row.' - ) - ), - ] = '', - description: Annotated[ - str, - Field( - description=( - 'New detailed description for the configuration row. Only provide if changing the description. ' - 'Should explain the specific purpose and behavior of this individual row.' - ) - ), - ] = '', - parameter_updates: Annotated[ - list[ConfigParamUpdate], - Field( - description=( - 'List of granular parameter update operations to apply to this row. ' - 'Each operation (set, str_replace, remove, list_append) modifies a specific ' - 'parameter using JSONPath notation. Only provide if updating parameters - ' - 'do not use for changing description or storage. ' - "Paths are relative to the row's `parameters` object, not the row root " - '(e.g. use `tables`, not `parameters.tables`). ' - 'Prefer simple dot-delimited JSONPaths ' - 'and make the smallest possible updates - only change what needs changing. ' - 'In case you need to replace the whole parameters, you can use the `set` operation ' - 'with `$` as path.' - ), - ), - ] = None, - storage: Annotated[ - dict[str, Any], - Field( - description=( - 'Complete storage configuration for this row containing input/output table and file mappings. ' - 'Only provide if updating storage mappings - this replaces the ENTIRE storage configuration ' - 'for this row. ' - '\n\n' - 'When to use:\n' - '- Adding/removing input or output tables for this specific row\n' - '- Modifying table/file mappings for this row\n' - '- Updating table destinations or sources for this row\n' - '\n' - 'Important:\n' - "- Must conform to the component's row storage schema\n" - '- Replaces ALL existing storage config for this row - include all mappings you want to keep\n' - '- Use get_configs first to see current row storage configuration\n' - '- Leave unfilled to preserve existing storage configuration' - ) - ), - ] = None, - processors_before: Annotated[ - list[dict[str, Any]], - Field(description='The list of processors that will run before the configured component row runs.'), - ] = None, - processors_after: Annotated[ - list[dict[str, Any]], - Field(description='The list of processors that will run after the configured component row runs.'), - ] = None, - is_disabled: Annotated[ - bool | None, - Field( - description=( - "Enable or disable the configuration row. Set to True to disable execution (config row won't run), " - 'False to enable execution (config row will run). Only provide if changing the status, ' - 'leave as null to preserve current state.' - ), - ), - ] = None, -) -> ConfigToolOutput: - """ - Updates an existing component configuration row by modifying its parameters, storage mappings, name, or description. - - This tool allows PARTIAL parameter updates - you only need to provide the fields you want to change. - All other fields will remain unchanged. - Configuration rows are individual items within a configuration, often representing separate data sources, - tables, or endpoints that share the same component type and parent configuration settings. - - WHEN TO USE: - - Modifying row-specific parameters (table sources, filters, credentials, etc.) - - Updating storage mappings for a specific row (input/output tables or files) - - Changing row name or description - - Any combination of the above - - WHEN NOT TO USE: - - `keboola.orchestrator` / `keboola.flow` → use flows tools - - `keboola.data-apps` → use data applications tools - - `keboola.snowflake-transformation` / `keboola.google-bigquery-transformation` → use SQL transformation tools - - PREREQUISITES: - - The configuration row must already exist (use add_config_row for new rows) - - You must know component_id, configuration_id, and configuration_row_id - - For parameter updates: Review the component's row_configuration_schema using get_components - - For storage updates: Ensure mappings are valid for row-level storage - - IMPORTANT CONSIDERATIONS: - - Parameter updates are PARTIAL - only specify fields you want to change - - parameter_updates supports granular operations: set individual keys, replace strings, or remove keys - - Parameters must conform to the component's row_configuration_schema (not root schema) - - Validate schemas before calling: use get_components to retrieve row_configuration_schema - - Each row operates independently - changes to one row don't affect others - - Row-level storage is separate from root-level storage configuration - - WORKFLOW: - 1. Retrieve current configuration using get_configs to see existing rows - 2. Identify the specific row to modify by its configuration_row_id - 3. Prepare parameter_updates list with targeted operations for this row - 4. Call update_config_row with only the fields to change - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - LOG.info( - f'Updating configuration row for component: {component_id}, configuration id: {configuration_id}, ' - f'row id: {configuration_row_id}.' - ) - - _, configuration_payload = await update_config_row_internal( - client=client, - change_description=change_description, - component_id=component_id, - configuration_id=configuration_id, - configuration_row_id=configuration_row_id, - name=name, - description=description, - parameter_updates=parameter_updates, - storage=storage, - processors_before=processors_before, - processors_after=processors_after, - is_disabled=is_disabled, - ) - updated_raw_configuration = await client.storage_client.configuration_row_update( - component_id=component_id, - config_id=configuration_id, - configuration_row_id=configuration_row_id, - configuration=configuration_payload, - change_description=change_description, - updated_name=name, - updated_description=description, - is_disabled=is_disabled, - ) - - LOG.info( - f'Updated configuration row for component: {component_id}, configuration id: {configuration_id}, ' - f'row id: {configuration_row_id}.' - ) - - await set_cfg_update_metadata( - client=client, - component_id=component_id, - configuration_id=configuration_id, - configuration_version=updated_raw_configuration['version'], - ) - - links = links_manager.get_configuration_links( - component_id=component_id, - configuration_id=configuration_id, - configuration_name=updated_raw_configuration.get('name') or '', - ) - - return ConfigToolOutput( - component_id=component_id, - configuration_id=configuration_id, - description=updated_raw_configuration.get('description') or '', - timestamp=datetime.now(timezone.utc), - success=True, - links=links, - version=updated_raw_configuration['version'], - ) - - -# This function must use exactly the same parameters as update_config_row() function. -# Except for the `ctx` and `client` parameters. -async def update_config_row_internal( - *, - client: KeboolaClient, - change_description: str, - component_id: str, - configuration_id: str, - configuration_row_id: str, - name: str = '', - description: str = '', - parameter_updates: list[ConfigParamUpdate] | None = None, - storage: dict[str, Any] | None = None, - processors_before: list[dict[str, Any]] | None = None, - processors_after: list[dict[str, Any]] | None = None, - is_disabled: bool | None = None, -) -> tuple[JsonDict, JsonDict]: - check_suitable('update_config_row', component_id) - - current_row = await client.storage_client.configuration_row_detail( - component_id=component_id, config_id=configuration_id, configuration_row_id=configuration_row_id - ) - api_component = await fetch_component(client=client, component_id=component_id) - component = Component.from_api_response(api_component) - - configuration_payload = cast(JsonDict, current_row.get('configuration', {})) - configuration_payload = copy.deepcopy(configuration_payload) - - if storage is not None: - storage_cfg = validate_row_storage_configuration( - component=component, - storage=storage, - initial_message='The "storage" field is not valid.', - configuration_id=configuration_id, - configuration_row_id=configuration_row_id, - ) - configuration_payload['storage'] = storage_cfg - - if processors_before is not None: - processors_before = await validate_processors_configuration( - client=client, - processors=processors_before, - initial_message='The "processors_before" field is not valid.', - ) - set_nested_value(configuration_payload, 'processors.before', processors_before) - - if processors_after is not None: - processors_after = await validate_processors_configuration( - client=client, - processors=processors_after, - initial_message='The "processors_after" field is not valid.', - ) - set_nested_value(configuration_payload, 'processors.after', processors_after) - - if parameter_updates: - current_params = configuration_payload.get('parameters', {}) - updated_params = update_params(current_params, parameter_updates) - - parameters_cfg = validate_row_parameters_configuration( - component=component, - parameters=updated_params, - initial_message='Applying the "parameter_updates" resulted in an invalid row configuration.', - configuration_id=configuration_id, - configuration_row_id=configuration_row_id, - ) - configuration_payload['parameters'] = parameters_cfg - - return current_row, configuration_payload - - -@tool_errors() -async def get_config_examples( - ctx: Context, - component_id: Annotated[str, Field(description='The ID of the component to get configuration examples for.')], -) -> Annotated[ - str, - Field(description='Markdown formatted string containing configuration examples for the component.'), -]: - """ - Retrieves sample configuration examples for a specific component. - - USAGE: - - Use before calling `create_config` or `add_config_row` to understand the expected parameters structure. - - Use when you want to see example configurations for a specific component. - - EXAMPLES: - - user_input: `Show me example configurations for component X` - - set the component_id parameter accordingly - - returns a markdown formatted string with configuration examples - """ - client = KeboolaClient.from_state(ctx.session.state) - try: - raw_component = await client.ai_service_client.get_component_detail(component_id) - except HTTPStatusError: - LOG.exception(f'Error when getting component details: {component_id}') - return '' - - root_examples = raw_component.get('rootConfigurationExamples') or [] - row_examples = raw_component.get('rowConfigurationExamples') or [] - assert isinstance(root_examples, list) # pylance check - assert isinstance(row_examples, list) # pylance check - - markdown = f'# Configuration Examples for `{component_id}`\n\n' - - if root_examples: - markdown += '## Root Configuration Examples\n\n' - for i, example in enumerate(root_examples, start=1): - markdown += f'{i}. Root Configuration:\n```json\n{json.dumps(example, indent=2)}\n```\n\n' - - if row_examples: - markdown += '## Row Configuration Examples\n\n' - for i, example in enumerate(row_examples, start=1): - markdown += f'{i}. Row Configuration:\n```json\n{json.dumps(example, indent=2)}\n```\n\n' - - return markdown - - -# ============================================================================ -# SYNC ACTION TOOLS -# ============================================================================ - - -@tool_errors() -async def run_sync_action( - ctx: Context, - action_name: Annotated[ - str, - Field(description='The sync action to execute (e.g., "testConnection", "getTables").'), - ], - component_id: Annotated[ - str, - Field(description='The ID of the component (e.g., "keboola.ex-db-mysql").'), - ], - configuration_id: Annotated[ - str, - Field(description='The ID of the configuration to use for the sync action.'), - ], - configuration_row_id: Annotated[ - str | None, - Field( - description=( - 'Optional row ID for row-level actions. When provided, ' - 'the row parameters and storage are shallow-merged on top of root config.' - ), - ), - ] = None, -) -> dict[str, Any] | list[Any]: - """ - Executes a synchronous action for a component configuration or a component row configuration. - - WHEN TO USE: - - For finding available values of a configuration field - - For validating already configured values (e.g. testing a database connection) - - For listing remote resources such as endpoints, schemas or tables - """ - client = KeboolaClient.from_state(ctx.session.state) - - config_detail = await client.storage_client.configuration_detail(component_id, configuration_id) - config_response = ConfigurationAPIResponse.model_validate({**config_detail, 'componentId': component_id}) - root_configuration = config_response.configuration - parameters = root_configuration.get('parameters') or {} - storage = root_configuration.get('storage') or {} - # `runtime` and `authorization` live only on the root configuration in the docker-runner's - # contract — rows do not override them. `authorization.oauth_api.id` is a broker reference - # that the sync-actions service resolves and decrypts before invoking the component; without - # it, OAuth/Service-Account components reject the call with "Missing authorization data". - runtime = root_configuration.get('runtime') or {} - authorization = root_configuration.get('authorization') or {} - - if configuration_row_id: - row_detail = await client.storage_client.configuration_row_detail( - component_id, configuration_id, configuration_row_id - ) - row_configuration = row_detail.get('configuration') or {} - row_parameters = row_configuration.get('parameters') or {} - row_storage = row_configuration.get('storage') or {} - parameters = {**parameters, **row_parameters} - storage = {**storage, **row_storage} - - config_data: dict[str, Any] = { - 'parameters': parameters, - 'storage': storage, - } - if runtime: - config_data['runtime'] = runtime - if authorization: - config_data['authorization'] = authorization - - result = await client.sync_actions_client.execute_action( - component_id=component_id, - action=action_name, - config_data=config_data, - ) - return result diff --git a/src/keboola_mcp_server/tools/components/utils.py b/src/keboola_mcp_server/tools/components/utils.py deleted file mode 100644 index fdeea5909..000000000 --- a/src/keboola_mcp_server/tools/components/utils.py +++ /dev/null @@ -1,1096 +0,0 @@ -""" -Utility functions for Keboola component and configuration management. - -This module contains helper functions and utilities used across the component tools: - -## Component Retrieval -- fetch_component: Fetches component details with AI Service/Storage API fallback -- handle_component_types: Normalizes component type filtering - -## Configuration Listing -- list_configs_by_types: Retrieves components+configs filtered by type -- list_configs_by_ids: Retrieves components+configs filtered by ID - -## SQL Transformation Utilities -- get_sql_transformation_id_from_sql_dialect: Maps SQL dialect to component ID -- get_transformation_configuration: Builds transformation config payloads -- clean_bucket_name: Sanitizes bucket names for transformations - -## Data Models -- TransformationConfiguration: Pydantic model for SQL transformation structure -""" - -import copy -import logging -import re -import unicodedata -from typing import Any, Mapping, Optional, Sequence, TypeVar, cast - -import jsonpath_ng -from httpx import HTTPStatusError -from jsonpath_ng.exceptions import JSONPathError - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import ( - CONDITIONAL_FLOW_COMPONENT_ID, - DATA_APP_COMPONENT_ID, - ORCHESTRATOR_COMPONENT_ID, - KeboolaClient, -) -from keboola_mcp_server.clients.storage import ComponentAPIResponse, ConfigurationAPIResponse -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.links import ProjectLinksManager -from keboola_mcp_server.tools.components import tf_update -from keboola_mcp_server.tools.components.model import ( - ALL_COMPONENT_TYPES, - ComponentSummary, - ComponentType, - ComponentWithConfigs, - ConfigParamUpdate, - ConfigSummary, - SimplifiedTfBlocks, - TfParamUpdate, - TransformationConfiguration, - VariableDefinition, -) - -LOG = logging.getLogger(__name__) -T = TypeVar('T') - - -# ============================================================================ -# CONSTANTS -# ============================================================================ - -SNOWFLAKE_TRANSFORMATION_ID = 'keboola.snowflake-transformation' -BIGQUERY_TRANSFORMATION_ID = 'keboola.google-bigquery-transformation' -PYTHON_TRANSFORMATION_ID = 'keboola.python-transformation-v2' -R_TRANSFORMATION_ID = 'keboola.r-transformation-v2' -VARIABLES_COMPONENT_ID = 'keboola.variables' - -# Component IDs for which update_config actively manages folder metadata (set/clear/hint). -# For all other components the folder parameter is accepted but silently skipped to avoid -# unnecessary API calls on components where folder organisation is not expected. -FOLDER_SUPPORTING_COMPONENT_IDS: frozenset[str] = frozenset({PYTHON_TRANSFORMATION_ID, R_TRANSFORMATION_ID}) - - -# ============================================================================ -# CONFIGURATION LISTING UTILITIES -# ============================================================================ - - -def expand_component_types(component_types: Sequence[ComponentType]) -> tuple[ComponentType, ...]: - """ - Expand empty component types list to all component types. - - :param component_types: Sequence of component types to expand - :return: Tuple of component types, or all component types if input is empty - """ - if not component_types: - return ALL_COMPONENT_TYPES - - out_component_types = set(component_types) - - return tuple(sorted(out_component_types)) - - -async def list_configs_by_types( - client: KeboolaClient, component_types: Sequence[ComponentType], links_manager: ProjectLinksManager -) -> list[ComponentWithConfigs]: - """ - Retrieves components with their configurations filtered by component types. - - Used by: - - get_configs tool (when component types are requested) - - :param client: Authenticated Keboola client instance - :param component_types: Types of components to retrieve (extractor, writer, application, transformation) - :return: List of components paired with their configuration summaries - """ - components_with_configurations = [] - - for comp_type in component_types: - # Fetch raw components with configurations included - raw_components_with_configurations_by_type = await client.storage_client.component_list( - component_type=comp_type, include=['configuration'] - ) - - # Process each component and its configurations - for raw_component in raw_components_with_configurations_by_type: - raw_configuration_responses = [ - ConfigurationAPIResponse.model_validate(raw_configuration | {'component_id': raw_component['id']}) - for raw_configuration in cast(list[JsonDict], raw_component.get('configurations', [])) - ] - - # Convert to domain models add links - configuration_summaries = [] - for api_config in raw_configuration_responses: - cfg_summary = ConfigSummary.from_api_response(api_config) - cfg_root = cfg_summary.configuration_root - cfg_summary.links.append( - links_manager.get_component_config_link( - component_id=cfg_root.component_id, - configuration_id=cfg_root.configuration_id, - configuration_name=cfg_root.name, - ) - ) - configuration_summaries.append(cfg_summary) - - # Process component - api_component = ComponentAPIResponse.model_validate(raw_component) - domain_component = ComponentSummary.from_api_response(api_component) - domain_component.links.append( - links_manager.get_config_dashboard_link( - component_id=domain_component.component_id, component_name=domain_component.component_name - ) - ) - components_with_configurations.append( - ComponentWithConfigs( - component=domain_component, - configs=configuration_summaries, - ) - ) - - total_configurations = sum(len(component.configs) for component in components_with_configurations) - LOG.info( - f'Found {len(components_with_configurations)} components with total of {total_configurations} configurations ' - f'for types {component_types}.' - ) - return components_with_configurations - - -async def list_configs_by_ids( - client: KeboolaClient, component_ids: Sequence[str], links_manager: ProjectLinksManager -) -> list[ComponentWithConfigs]: - """ - Retrieves components with their configurations filtered by specific component IDs. - - Used by: - - get_configs tool (when specific component IDs are requested) - - :param client: Authenticated Keboola client instance - :param component_ids: Specific component IDs to retrieve - :return: List of components paired with their configuration summaries - """ - components_with_configurations = [] - - for component_id in component_ids: - # Fetch configurations and component details - raw_configurations = await client.storage_client.configuration_list(component_id=component_id) - raw_component = await client.storage_client.component_detail(component_id=component_id) - - # Process component - api_component = ComponentAPIResponse.model_validate(raw_component) - domain_component = ComponentSummary.from_api_response(api_component) - domain_component.links.append( - links_manager.get_config_dashboard_link( - component_id=domain_component.component_id, component_name=domain_component.component_name - ) - ) - # Process configurations - raw_configuration_responses = [ - ConfigurationAPIResponse.model_validate({**raw_configuration, 'component_id': raw_component['id']}) - for raw_configuration in raw_configurations - ] - configuration_summaries = [] - for api_config in raw_configuration_responses: - cfg_summary = ConfigSummary.from_api_response(api_config) - cfg_summary.links.append( - links_manager.get_component_config_link( - component_id=cfg_summary.configuration_root.component_id, - configuration_id=cfg_summary.configuration_root.configuration_id, - configuration_name=cfg_summary.configuration_root.name, - ) - ) - configuration_summaries.append(cfg_summary) - - components_with_configurations.append( - ComponentWithConfigs( - component=domain_component, - configs=configuration_summaries, - ) - ) - - total_configurations = sum(len(component.configs) for component in components_with_configurations) - LOG.info( - f'Found {len(components_with_configurations)} components with total of {total_configurations} configurations ' - f'for ids {component_ids}.' - ) - return components_with_configurations - - -# ============================================================================ -# COMPONENT FETCHING -# ============================================================================ - - -async def fetch_component( - client: KeboolaClient, - component_id: str, -) -> ComponentAPIResponse: - """ - Fetches a component by ID, returning the raw API response. - - First tries to get component from the AI service catalog. If the component - is not found (404) or returns empty data (private components), falls back to using the - Storage API endpoint. - - Used by: - - get_components tool - - Configuration creation/update operations that need component schemas - - :param client: Authenticated Keboola client instance - :param component_id: Unique identifier of the component to fetch - :return: Unified API component response with available metadata - :raises HTTPStatusError: If component is not found in either API - """ - try: - # First attempt: AI Service catalog (includes documentation & schemas) - raw_component = await client.ai_service_client.get_component_detail(component_id=component_id) - LOG.info(f'Retrieved component {component_id} from AI service catalog.') - # Get sync actions until they are present in the AI service catalog response - # TODO: Consider adding the entire `data` section into the AI service catalog response - # to avoid this in the future - component_detail_raw = await client.storage_client.component_detail(component_id=component_id) - raw_component['data'] = component_detail_raw.get('data', {}) - - return ComponentAPIResponse.model_validate(raw_component) - - except HTTPStatusError as e: - if e.response.status_code == 404: - # Fallback: Storage API (basic component info only) - LOG.info( - f'Component {component_id} not found in AI service catalog (possibly private). ' - f'Falling back to Storage API.' - ) - - raw_component = await client.storage_client.component_detail(component_id=component_id) - LOG.info(f'Retrieved component {component_id} from Storage API.') - - return ComponentAPIResponse.model_validate(raw_component) - else: - # If it's not a 404, re-raise the error - raise - - -# ============================================================================ -# SQL TRANSFORMATION UTILITIES -# ============================================================================ - - -def get_sql_transformation_id_from_sql_dialect( - sql_dialect: str, -) -> str: - """ - Map SQL dialect to the appropriate transformation component ID. - - Keboola has different transformation components for different SQL dialects. - This function maps the workspace SQL dialect to the correct component ID. - - :param sql_dialect: SQL dialect from workspace configuration (e.g., 'snowflake', 'bigquery') - :return: Component ID for the appropriate SQL transformation - :raises ValueError: If the SQL dialect is not supported - """ - if sql_dialect.lower() == 'snowflake': - return SNOWFLAKE_TRANSFORMATION_ID - elif sql_dialect.lower() == 'bigquery': - return BIGQUERY_TRANSFORMATION_ID - else: - raise ValueError(f'Unsupported SQL dialect: {sql_dialect}') - - -def clean_bucket_name(bucket_name: str) -> str: - """ - Cleans the bucket name: - - Converts the bucket name to ASCII. (Handle diacritics like český -> cesky) - - Converts spaces to dashes. - - Removes leading underscores, dashes, and whitespace. - - Removes any character that is not alphanumeric, dash, or underscore. - - :param bucket_name: Raw bucket name to clean - :return: Cleaned bucket name suitable for Keboola storage - """ - max_bucket_length = 96 - bucket_name = bucket_name.strip() - # Convert the bucket name to ASCII - bucket_name = unicodedata.normalize('NFKD', bucket_name) - bucket_name = bucket_name.encode('ascii', 'ignore').decode('ascii') # český -> cesky - # Replace all whitespace (including tabs, newlines) with dashes - bucket_name = re.sub(r'\s+', '-', bucket_name) - # Remove any character that is not alphanumeric, dash, or underscore - bucket_name = re.sub(r'[^a-zA-Z0-9_-]', '', bucket_name) - # Remove leading underscores if present - bucket_name = re.sub(r'^_+', '', bucket_name) - bucket_name = bucket_name[:max_bucket_length] - return bucket_name - - -async def create_transformation_configuration( - codes: Sequence[SimplifiedTfBlocks.Block.Code], - transformation_name: str, - output_tables: Sequence[str], - sql_dialect: str, -) -> TransformationConfiguration: - """ - Creates transformation configuration from simplified code blocks and output tables. - Handles splitting the SQL `script`s into arrays of statements and creating the storage configuration. - - :param codes: The code blocks - :param transformation_name: The name of the transformation from which the bucket name is derived as in the UI - :param output_tables: The output tables of the transformation, created by the code statements - :param sql_dialect: The SQL dialect of the transformation - :return: TransformationConfiguration with parameters and storage - """ - storage = TransformationConfiguration.Storage() - # for simplicity, we create a single block with the name 'Blocks' - block = SimplifiedTfBlocks.Block( - name='Blocks', - codes=list(codes), - ) - parameters = SimplifiedTfBlocks(blocks=[block]) - raw_parameters = await parameters.to_raw_parameters() - - if output_tables: - # if the query creates new tables, output_table_mappings should contain the table names (llm generated) - # we create bucket name from the sql query name adding `out.c-` prefix as in the UI and use it as destination - # expected output table name format is `out.c-.` - bucket_name = clean_bucket_name(transformation_name) - destination = f'out.c-{bucket_name}' - storage.output.tables = [ - TransformationConfiguration.Storage.Destination.Table( - # here the source refers to the table name from the sql statement - # and the destination to the full bucket table name - # WARNING: when implementing input.tables, source and destination are swapped. - source=out_table, - destination=f'{destination}.{out_table}', - ) - for out_table in output_tables - ] - - return TransformationConfiguration(parameters=raw_parameters, storage=storage) - - -async def set_cfg_creation_metadata(client: KeboolaClient, component_id: str, configuration_id: str) -> None: - """ - Sets the configuration metadata to indicate it was created by MCP. - - :param client: KeboolaClient instance - :param component_id: ID of the component - :param configuration_id: ID of the configuration - """ - try: - await client.storage_client.configuration_metadata_update( - component_id=component_id, - configuration_id=configuration_id, - metadata={MetadataField.CREATED_BY_MCP: 'true'}, - ) - except HTTPStatusError as e: - logging.exception( - f'Failed to set "{MetadataField.CREATED_BY_MCP}" metadata for configuration {configuration_id}: {e}' - ) - - -async def set_cfg_update_metadata( - client: KeboolaClient, - component_id: str, - configuration_id: str, - configuration_version: int, -) -> None: - """ - Sets the configuration metadata to indicate it was updated by MCP. - - :param client: KeboolaClient instance - :param component_id: ID of the component - :param configuration_id: ID of the configuration - :param configuration_version: Version of the configuration - """ - updated_by_md_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{configuration_version}' - try: - await client.storage_client.configuration_metadata_update( - component_id=component_id, - configuration_id=configuration_id, - metadata={updated_by_md_key: 'true'}, - ) - except HTTPStatusError as e: - logging.exception(f'Failed to set "{updated_by_md_key}" metadata for configuration {configuration_id}: {e}') - - -async def get_config_folders(client: KeboolaClient, component_id: str) -> tuple[int, list[str], bool]: - """ - Returns the total number of existing configurations, the distinct folder names already in use, - and a flag indicating whether the count is a lower bound. - - When ≥20 configs already carry folder metadata, the full configuration list is skipped for - performance. In that case the returned count equals the number of folder-bearing configs, which - is a lower bound on the real total (``lower_bound=True``). Callers that surface this count to - the user should phrase it as "at least N" to avoid misrepresenting the actual total. - - :param client: KeboolaClient instance - :param component_id: ID of the component (e.g. keboola.snowflake-transformation) - :return: Tuple of (count, distinct_folder_names, lower_bound) - """ - # Fetch folder-bearing configs first — lighter than a full configuration_list for large projects. - folder_configs = await client.storage_client.component_configurations_search( - component_id=component_id, - metadata_keys=[MetadataField.CONFIGURATION_FOLDER_NAME], - ) - seen: set[str] = set() - folders: list[str] = [] - for cfg in folder_configs: - for meta in cfg.get('metadata', []): - if meta.get('key') == MetadataField.CONFIGURATION_FOLDER_NAME: - folder_name = meta.get('value', '').strip() - if folder_name and folder_name not in seen: - seen.add(folder_name) - folders.append(folder_name) - - # If ≥20 configs already have folders, total must be at least that many — skip configuration_list. - if len(folder_configs) >= 20: - return len(folder_configs), folders, True - - # Fewer than 20 folder-bearing configs — check actual total to decide whether to hint. - raw_configs = await client.storage_client.configuration_list(component_id=component_id) - total = len(raw_configs) - if total < 20: - return total, [], False - return total, folders, False - - -async def set_configuration_folder_metadata( - client: KeboolaClient, component_id: str, configuration_id: str, folder: str -) -> None: - """ - Sets the KBC.configuration.folderName metadata for a configuration. - Strips whitespace from the folder name; does nothing if the result is empty. - - :param client: KeboolaClient instance - :param component_id: ID of the component - :param configuration_id: ID of the configuration - :param folder: Folder name to assign - """ - normalized = folder.strip() - if not normalized: - return - await client.storage_client.configuration_metadata_update( - component_id=component_id, - configuration_id=configuration_id, - metadata={MetadataField.CONFIGURATION_FOLDER_NAME: normalized}, - ) - - -async def clear_configuration_folder_metadata(client: KeboolaClient, component_id: str, configuration_id: str) -> None: - """Removes the KBC.configuration.folderName metadata entry if present.""" - try: - metadata = await client.storage_client.configuration_metadata_get( - component_id=component_id, configuration_id=configuration_id - ) - for entry in metadata: - if entry.get('key') == MetadataField.CONFIGURATION_FOLDER_NAME: - metadata_id = entry.get('id') - if metadata_id is None: - LOG.warning( - 'Unable to clear folder metadata for component "%s", configuration "%s": ' - 'metadata entry is missing "id".', - component_id, - configuration_id, - ) - continue - await client.storage_client.configuration_metadata_delete( - component_id=component_id, - configuration_id=configuration_id, - metadata_id=metadata_id, - ) - except Exception: - LOG.warning( - 'Unable to clear folder metadata for component "%s", configuration "%s".', - component_id, - configuration_id, - ) - - -def _variables_config_name(component_id: str, config_id: str) -> str: - return f'Variables definition for {component_id}/{config_id}' - - -async def _find_vars_config( - client: KeboolaClient, - component_id: str, - config_id: str, - existing_vars_id: str | None, -) -> dict[str, Any] | None: - """Resolves the existing keboola.variables config for a parent configuration. - - Prefers the linked variables_id; falls back to a name search. After a name-based - match, re-fetches via configuration_detail to guarantee the rows field is present. - """ - if existing_vars_id: - try: - return await client.storage_client.configuration_detail(VARIABLES_COMPONENT_ID, existing_vars_id) - except HTTPStatusError as e: - if e.response.status_code != 404: - raise - vars_name = _variables_config_name(component_id, config_id) - all_vars_configs = await client.storage_client.configuration_list(VARIABLES_COMPONENT_ID) - found = next((c for c in all_vars_configs if c.get('name') == vars_name), None) - if found is None: - return None - return await client.storage_client.configuration_detail(VARIABLES_COMPONENT_ID, str(found['id'])) - - -async def _apply_vars_to_parent_cfg( - client: KeboolaClient, - component_id: str, - config_id: str, - variables: list[VariableDefinition], - parent_cfg: dict[str, Any], -) -> tuple[bool, str | None]: - """Manages the keboola.variables config for a parent and mutates parent_cfg with link fields. - - Returns ``(changed, vars_config_id_to_delete)``: - - ``changed``: True if parent_cfg was modified. - - ``vars_config_id_to_delete``: ID of the keboola.variables config that the caller must - delete AFTER successfully writing parent_cfg to Storage. Deleting before the parent - update risks leaving a stale ``variables_id`` reference if the update then fails. - - Does NOT write parent_cfg to Storage and does NOT delete any config — both are the - caller's responsibility. - """ - existing = await _find_vars_config(client, component_id, config_id, parent_cfg.get('variables_id')) - - if not variables: - vars_config_id_to_delete: str | None = str(existing['id']) if existing is not None else None - changed = False - for key in ('variables_id', 'variables_values_id'): - if key in parent_cfg: - parent_cfg.pop(key) - changed = True - return changed, vars_config_id_to_delete - - # Set path — create or update variables config. - var_defs = [{'name': v.name, 'type': v.type} for v in variables] - vars_configuration = {'variables': var_defs} - if existing is None: - created = await client.storage_client.configuration_create( - component_id=VARIABLES_COMPONENT_ID, - name=_variables_config_name(component_id, config_id), - description='', - configuration=vars_configuration, - ) - vars_config_id = str(created['id']) - else: - vars_config_id = str(existing['id']) - await client.storage_client.configuration_update( - component_id=VARIABLES_COMPONENT_ID, - configuration_id=vars_config_id, - configuration=vars_configuration, - change_description='Update variable definitions', - ) - - defaults = [{'name': v.name, 'value': v.default_value} for v in variables if v.default_value is not None] - existing_rows = (existing or {}).get('rows') or [] - default_row = next((r for r in existing_rows if r.get('name') == 'Default Values'), None) - default_values_row_id: str | None = None - if defaults: - row_cfg = {'values': defaults} - if default_row is None: - created_row = await client.storage_client.configuration_row_create( - component_id=VARIABLES_COMPONENT_ID, - config_id=vars_config_id, - name='Default Values', - description='', - configuration=row_cfg, - ) - default_values_row_id = str(created_row['id']) - else: - default_values_row_id = str(default_row['id']) - await client.storage_client.configuration_row_update( - component_id=VARIABLES_COMPONENT_ID, - config_id=vars_config_id, - configuration_row_id=default_values_row_id, - configuration=row_cfg, - change_description='Update default variable values', - ) - elif default_row is not None: - await client.storage_client.configuration_row_update( - component_id=VARIABLES_COMPONENT_ID, - config_id=vars_config_id, - configuration_row_id=str(default_row['id']), - configuration={'values': []}, - change_description='Clear default variable values', - ) - - parent_cfg['variables_id'] = vars_config_id - if default_values_row_id is not None: - parent_cfg['variables_values_id'] = default_values_row_id - else: - parent_cfg.pop('variables_values_id', None) - return True, None - - -async def apply_configuration_variables( - client: KeboolaClient, - component_id: str, - config_id: str, - variables: list[VariableDefinition], -) -> dict[str, Any] | None: - """ - Creates, updates, or clears the keboola.variables config linked to a parent configuration. - - Resolves an existing variables config by the parent's variables_id first; falls back to - a name-based search so renames do not cause duplicate configs to be created. - - - Non-empty list: creates or updates variable definitions, creates/updates a - "Default Values" row for any variable with a default_value, and patches - variables_id onto the parent config. - - Empty list: deletes the vars config (if found) and removes variables_id from the parent. - - Returns the parent config update response if the parent was updated, None otherwise. - """ - parent = await client.storage_client.configuration_detail(component_id, config_id) - parent_cfg = dict(parent.get('configuration') or {}) - changed, vars_config_id_to_delete = await _apply_vars_to_parent_cfg( - client, component_id, config_id, variables, parent_cfg - ) - if not changed and not vars_config_id_to_delete: - return None - change_description = 'Link variables' if variables else 'Unlink variables' - result = None - if changed: - result = await client.storage_client.configuration_update( - component_id=component_id, - configuration_id=config_id, - configuration=parent_cfg, - change_description=change_description, - ) - if vars_config_id_to_delete: - await client.storage_client.configuration_delete( - component_id=VARIABLES_COMPONENT_ID, - configuration_id=vars_config_id_to_delete, - skip_trash=True, - ) - return result - - -async def apply_folder_metadata( - client: KeboolaClient, - component_id: str, - configuration_id: str, - folder: Optional[str], - kind: str, - tool_name: str, - *, - is_new: bool = False, -) -> str | None: - """ - Sets or clears folder metadata for a configuration, or returns a hint when many exist. - - :param kind: Human-readable plural noun for the hint (e.g. 'configurations', 'data apps'). - :param tool_name: Tool name for the hint (e.g. 'update_config', 'modify_streamlit_data_app'). - :param is_new: When True, an empty folder string is a no-op (no folder to remove on a new item). - :return: Folder hint string if applicable, else None. - """ - if folder is None: - try: - total, existing_folders, lower_bound = await get_config_folders(client, component_id) - return build_folder_hint(total, existing_folders, kind, tool_name, lower_bound=lower_bound) - except Exception: - LOG.warning( - 'Unable to fetch %s folders for component "%s" when processing configuration "%s".', - kind, - component_id, - configuration_id, - ) - return None - normalized = folder.strip() - if normalized: - try: - await set_configuration_folder_metadata(client, component_id, configuration_id, normalized) - except Exception: - LOG.warning( - 'Unable to set folder metadata for component "%s", configuration "%s".', - component_id, - configuration_id, - ) - elif not is_new: - await clear_configuration_folder_metadata(client, component_id, configuration_id) - return None - - -def folder_field_description(singular: str, plural: str) -> str: - """Returns the standard Field description for a `folder` parameter. - - :param singular: Singular resource name, e.g. "transformation", "flow", "data app" - :param plural: Plural resource name, e.g. "transformations", "flows", "data apps" - """ - return ( - f'Folder name to organize this {singular} in the Keboola UI. ' - f'Pass an empty string to remove an existing folder assignment. ' - f'Existing folder names are returned in the response change_summary when no folder is provided ' - f'and there are 20 or more {plural} in the project. ' - f'If there are 20 or more {plural}, you should assign one of the existing folders or ' - f'create a new one that clearly reflects the {singular} purpose.' - ) - - -def build_folder_hint( - total: int, - existing_folders: list[str], - config_label: str, - update_tool: str, - *, - lower_bound: bool = False, -) -> str | None: - """Returns a folder-organization hint for the LLM when a project has ≥20 configurations of the given type. - - :param total: Total (or lower-bound) number of existing configurations for this component type - :param existing_folders: List of folder names already in use - :param config_label: Human-readable label for the config type (e.g. "SQL transformations", "flows") - :param update_tool: Name of the tool to call to assign a folder (e.g. "update_sql_transformation") - :param lower_bound: When True, ``total`` is a lower bound; the hint says "at least N" instead of "N" - :return: Hint string, or None if not enough configurations to warrant organizing - """ - if total < 20: - return None - count_str = f'at least {total}' if lower_bound else str(total) - hint = f'Note: This project already has {count_str} {config_label}. Consider organizing them with folders. ' - if existing_folders: - hint += ( - f'Existing folders: {", ".join(existing_folders)}. ' - f'Call {update_tool} with a folder= parameter to assign this to one.' - ) - else: - hint += f'No folders have been created yet. Call {update_tool} with a folder= parameter to start organizing.' - return hint - - -# ============================================================================ -# PARAMETER UPDATE UTILITIES -# ============================================================================ - - -def get_nested(obj: Mapping[str, Any] | None, key: str, *, default: T | None = None) -> T | None: - """ - Gets a value from a nested mapping associated with the key. - - :param obj: Mapping (dictionary) object to search in - :param key: Dot-separated key path (e.g., 'database.host') - :param default: Default value to return if key is not found - :return: Value associated with the key, or default if not found - """ - d = obj - for k in key.split('.'): - d = d.get(k) if isinstance(d, Mapping) else None - if d is None: - return default - return d - - -# Regex matching valid unquoted JSONPath field names (letters, digits, underscores, starting with letter/underscore) -_VALID_JSONPATH_FIELD = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$') - - -def _normalize_jsonpath(path: str) -> str: - """Normalize a dot-notation path by quoting segments that contain special characters. - - jsonpath_ng cannot parse field names with characters like '#' unless they are quoted. - This function auto-quotes such segments so that e.g. '#anthropic_api_key' becomes - '"#anthropic_api_key"' and 'parameters.#key' becomes 'parameters."#key"'. - - :param path: Dot-separated path (e.g., 'parameters.#anthropic_api_key') - :return: Path with special-character segments quoted for jsonpath_ng - """ - segments = [] - for segment in path.split('.'): - if segment.startswith('"') or segment.startswith("'") or '[' in segment or segment == '$': - segments.append(segment) - elif not _VALID_JSONPATH_FIELD.match(segment): - segments.append(f'"{segment}"') - else: - segments.append(segment) - return '.'.join(segments) - - -def set_nested_value(data: dict[str, Any], path: str, value: Any) -> None: - """ - Sets a value in a nested dictionary using a dot-separated path. - - :param data: The dictionary to modify - :param path: Dot-separated path (e.g., 'database.host') - :param value: The value to set - :raises ValueError: If a non-dict value is encountered in the path - """ - keys = path.split('.') - current = data - - for i, key in enumerate(keys[:-1]): - if key not in current: - current[key] = {} - current = current[key] - if not isinstance(current, dict): - path_so_far = '.'.join(keys[: i + 1]) - raise ValueError( - f'Cannot set nested value at path "{path}": ' - f'encountered non-dict value at "{path_so_far}" (type: {type(current).__name__})' - ) - - current[keys[-1]] = value - - -def _apply_param_update(params: dict[str, Any], update: ConfigParamUpdate) -> dict[str, Any]: - """ - Applies a single parameter update to the given parameters dictionary. - - Note: This function modifies the input dictionary in place for efficiency. - The caller (update_params) is responsible for creating a copy if needed. - - :param params: Current parameter values (will be modified in place) - :param update: Parameter update operation to apply - :return: The modified parameters dictionary - :raises ValueError: If trying to set a nested value through a non-dict value in the path - """ - normalized_path = _normalize_jsonpath(update.path) - try: - jsonpath_expr = jsonpath_ng.parse(normalized_path) - except (JSONPathError, TypeError) as e: - raise ValueError( - f'Invalid JSONPath expression "{update.path}": {e}. ' - f'Ensure the path contains valid field names separated by dots.' - ) from e - - if update.op == 'set': - try: - matches = jsonpath_expr.find(params) - if not matches: - # path doesn't exist, create it manually - set_nested_value(params, update.path, update.value) - else: - params = jsonpath_expr.update(params, update.value) - except Exception as e: - raise ValueError(f'Failed to set nested value at path "{update.path}": {e}') - return params - - elif update.op == 'str_replace': - - if not update.search_for: - raise ValueError('Search string is empty') - - if update.search_for == update.replace_with: - raise ValueError(f'Search string and replace string are the same: "{update.search_for}"') - - matches = jsonpath_expr.find(params) - - if not matches: - raise ValueError(f'Path "{update.path}" does not exist') - - replace_cnt = 0 - for match in matches: - current_value = match.value - if isinstance(current_value, str): - occurrences = current_value.count(update.search_for) - if occurrences: - new_value = current_value.replace(update.search_for, update.replace_with) - replace_cnt += occurrences - params = match.full_path.update(params, new_value) - elif isinstance(current_value, list): - if not all(isinstance(item, str) for item in current_value): - raise ValueError(f'Path "{match.full_path}" is not a string or list of strings') - - occurrences = 0 - new_value = [] - for item in current_value: - item_occurrences = item.count(update.search_for) - occurrences += item_occurrences - new_item = item.replace(update.search_for, update.replace_with) if item_occurrences else item - new_value.append(new_item) - - if occurrences: - replace_cnt += occurrences - params = match.full_path.update(params, new_value) - else: - raise ValueError(f'Path "{match.full_path}" is not a string or list of strings') - - if replace_cnt == 0: - raise ValueError(f'Search string "{update.search_for}" not found in path "{update.path}"') - - return params - - elif update.op == 'remove': - matches = jsonpath_expr.find(params) - - if not matches: - raise ValueError(f'Path "{update.path}" does not exist') - - return jsonpath_expr.filter(lambda x: True, params) - - elif update.op == 'list_append': - matches = jsonpath_expr.find(params) - - if not matches: - raise ValueError(f'Path "{update.path}" does not exist') - - for match in matches: - current_value = match.value - if not isinstance(current_value, list): - raise ValueError(f'Path "{match.full_path}" is not a list') - - current_value.append(update.value) - - return params - - -def update_params(params: dict[str, Any], updates: Sequence[ConfigParamUpdate]) -> dict[str, Any]: - """ - Applies a list of parameter updates to the given parameters dictionary. - The original dictionary is not modified. - - :param params: Current parameter values - :param updates: Sequence of parameter update operations - :return: New dictionary with all updates applied - """ - # Create a deep copy to avoid mutating the original - params = copy.deepcopy(params) - for update in updates: - params = _apply_param_update(params, update) - return params - - -def _apply_tf_param_update( - parameters: dict[str, Any], update: TfParamUpdate, sql_dialect: str -) -> tuple[dict[str, Any], str]: - """ - Applies a single parameter update to the given transformation parameters. - - Note: This function modifies the input dictionary in place for efficiency. - The caller (update_transformation_parameters) is responsible for creating a copy if needed. - - :param parameters: The transformation parameters - :param update: Parameter update operation to apply - :param sql_dialect: The SQL dialect of the transformation - :return: Tuple of (updated transformation parameters, change summary message) - """ - operation = update.op - tf_update_func = getattr(tf_update, operation) - return tf_update_func(params=parameters, op=update, sql_dialect=sql_dialect) - - -def add_ids(parameters: dict[str, Any]) -> dict[str, Any]: - """ - Adds IDs to the parameters dictionary. - Blocks are numbered sequentially from 0. - Codes are numbered sequentially from 0 within each block and prefixed with the block ID. - - :param parameters: Transformation parameters dictionary - :return: Parameters dictionary with IDs added to blocks and codes - """ - for bidx, block in enumerate(parameters['blocks']): - block['id'] = f'b{bidx}' - for cidx, code in enumerate(block['codes']): - code['id'] = f'b{bidx}.c{cidx}' - return parameters - - -def structure_summary(parameters: dict[str, Any]) -> str: - """ - Generate a markdown summary of transformation structure showing block IDs, code IDs, and SQL snippets. - - :param parameters: Transformation parameters dictionary with blocks containing IDs - :return: Markdown formatted summary of the transformation structure - """ - lines = ['## Updated Transformation Structure', ''] - - blocks = parameters.get('blocks', []) - - if not blocks: - return '## Updated Transformation Structure\n\nNo blocks found in transformation.\n' - - for block in blocks: - block_id = block['id'] - block_name = block.get('name', '') - - lines.append(f'### Block id: `{block_id}`, name: `{block_name}`') - lines.append('') - - codes = block.get('codes', []) - - if not codes: - lines.append('*No code blocks*') - lines.append('') - continue - - for code in codes: - code_id = code['id'] - code_name = code.get('name', '') - script = code.get('script', '') - - lines.append(f'- **Code id: `{code_id}`, name: `{code_name}`** SQL snippet:') - lines.append('') - - # SQL snippet (first 150 characters) - if script: - snippet = script.strip() - if len(snippet) > 150: - truncated_chars = len(snippet) - 150 - snippet = snippet[:150] + f'... ({truncated_chars} chars truncated)' - lines.append(' ```sql') - lines.append(f' {snippet}') - lines.append(' ```') - else: - lines.append(' *Empty script*') - - lines.append('') - - return '\n'.join(lines) - - -def update_transformation_parameters( - parameters: SimplifiedTfBlocks, updates: Sequence[TfParamUpdate], sql_dialect: str -) -> tuple[SimplifiedTfBlocks, str]: - """ - Applies a list of parameter updates to the given transformation parameters. - The original parameters are not modified. - - :param parameters: The transformation parameters - :param updates: Sequence of parameter update operations - :param sql_dialect: The SQL dialect of the transformation - :return: The updated transformation parameters and a summary of the changes. - """ - is_structure_change = any(update.op in tf_update.STRUCTURAL_OPS for update in updates) - parameters_dict = add_ids(parameters.model_dump()) - messages = [] - for update in updates: - parameters_dict, message = _apply_tf_param_update( - parameters=parameters_dict, update=update, sql_dialect=sql_dialect - ) - - if message: - messages.append(message) - - if is_structure_change: - # re-assign IDs to reflect changes in the structure - parameters_dict = add_ids(parameters_dict) - messages.append(structure_summary(parameters_dict)) - - change_summary = '\n'.join(messages) - return SimplifiedTfBlocks.model_validate(parameters_dict, extra='ignore'), change_summary - - -# ============================================================================ -# OTHER -# ============================================================================ - -_UNSUITABLE_COMPONENTS_MESSAGES: Mapping[str, str] = { - DATA_APP_COMPONENT_ID: 'Use the data applications tools.', - CONDITIONAL_FLOW_COMPONENT_ID: 'Use the flows tools.', - ORCHESTRATOR_COMPONENT_ID: 'Use the flows tools.', - BIGQUERY_TRANSFORMATION_ID: 'Use the SQL transformation tools.', - SNOWFLAKE_TRANSFORMATION_ID: 'Use the SQL transformation tools.', -} - - -def check_suitable(tool_name: str, component_id: str) -> None: - """ - Checks if the general components tooling can be used with the given component. - :raises ValueError: If the component needs to be handled by special tools. - """ - if message := _UNSUITABLE_COMPONENTS_MESSAGES.get(component_id): - raise ValueError(f'The "{tool_name}" tool cannot be used with {component_id} component. {message}') diff --git a/src/keboola_mcp_server/tools/constants.py b/src/keboola_mcp_server/tools/constants.py deleted file mode 100644 index 26ce74581..000000000 --- a/src/keboola_mcp_server/tools/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -FLOW_TOOLS_TAG = 'flows' -UPDATE_FLOW_TOOL_NAME = 'update_flow' -MODIFY_FLOW_TOOL_NAME = 'modify_flow' - -# Tag for tools supporting config diff preview feature -CONFIG_DIFF_PREVIEW_TAG = 'config-diff-preview' - -# Tag for semantic layer tools -SEMANTIC_TOOLS_TAG = 'semantic' diff --git a/src/keboola_mcp_server/tools/data_apps.py b/src/keboola_mcp_server/tools/data_apps.py deleted file mode 100644 index 72f49d2ff..000000000 --- a/src/keboola_mcp_server/tools/data_apps.py +++ /dev/null @@ -1,2078 +0,0 @@ -import copy -import importlib.resources as resources -import logging -import re -from typing import Annotated, Any, Literal, Mapping, Optional, Sequence, Union, cast -from urllib.parse import quote, urlsplit, urlunsplit - -import httpx -from fastmcp import Context, FastMCP -from fastmcp.tools import FunctionTool -from mcp.types import ToolAnnotations -from pydantic import BaseModel, Field - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import DATA_APP_COMPONENT_ID, KeboolaClient, get_metadata_property -from keboola_mcp_server.clients.data_science import ( - AppRunResponse, - CodeDataAppConfig, - DataAppConfig, - DataAppResponse, -) -from keboola_mcp_server.clients.storage import ConfigurationAPIResponse -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.links import Link, ProjectLinksManager -from keboola_mcp_server.mcp import process_concurrently, toon_serializer_compact -from keboola_mcp_server.tools.components.utils import ( - apply_folder_metadata, - folder_field_description, - set_cfg_creation_metadata, - set_cfg_update_metadata, -) -from keboola_mcp_server.tools.constants import CONFIG_DIFF_PREVIEW_TAG -from keboola_mcp_server.tools.validation import ValidationContext, validate_storage_configuration_against_schema -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - -DATA_APP_TOOLS_TAG = 'data-apps' - - -def add_data_app_tools(mcp: FastMCP) -> None: - """Add tools to the MCP server.""" - - mcp.add_tool( - FunctionTool.from_function( - modify_streamlit_data_app, - tags={DATA_APP_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}, - annotations=ToolAnnotations(destructiveHint=True), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - modify_python_js_data_app, - tags={DATA_APP_TOOLS_TAG}, - annotations=ToolAnnotations(destructiveHint=True), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - create_python_js_data_app_git_credential, - tags={DATA_APP_TOOLS_TAG}, - annotations=ToolAnnotations(destructiveHint=False), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - get_data_apps, - tags={DATA_APP_TOOLS_TAG}, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - deploy_data_app, - tags={DATA_APP_TOOLS_TAG}, - annotations=ToolAnnotations(destructiveHint=False), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - delete_python_js_data_app_draft, - tags={DATA_APP_TOOLS_TAG}, - annotations=ToolAnnotations(destructiveHint=True), - ) - ) - LOG.info('Data app tools initialized.') - - -# State of the data app -State = Literal['created', 'running', 'stopped', 'starting', 'stopping', 'restarting'] -# Accepts known states or any string preventing from validation errors when receiving unknown states from the API -# LLM agent can still understand the state of the data app even if it is different from the known states -SafeState = Union[State, str] -# Type of the data app -Type = Literal['streamlit', 'python-js'] -# Accepts known types or any string preventing from validation errors when receiving unknown types from the API -# LLM agent can still understand the type of the data app even if it is different from the known types -SafeType = Union[Type, str] - -_DATA_APP_RESOURCES = resources.files('keboola_mcp_server.resources.data_app') -_QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE = _DATA_APP_RESOURCES.joinpath('qsapi_query_data_code.py').read_text( - encoding='utf-8' -) -_STORAGE_QUERY_DATA_FUNCTION_CODE = _DATA_APP_RESOURCES.joinpath('sapi_query_data_code.py').read_text(encoding='utf-8') - -_DEFAULT_STREAMLIT_THEME = ( - '[theme]\nfont = "sans serif"\ntextColor = "#222529"\nbackgroundColor = "#FFFFFF"\nsecondaryBackgroundColor = ' - '"#E6F2FF"\nprimaryColor = "#1F8FFF"' -) -_DEFAULT_PACKAGES = ['pandas', 'httpx'] - -# Username embedded in the HTTPS clone URL alongside the one-time token returned by the -# managed git-repo credentials endpoint. The git-service ignores the username portion of -# basic auth — only the password (token) is checked — but a non-empty username is required -# for `git clone` to accept the URL without prompting. -_MANAGED_GIT_REPO_USERNAME = 'kai' - -# Default branch name used for the very first draft of a brand-new prod app, when the agent -# doesn't supply a descriptive branch via `branch=`. Uniqueness across drafts is the agent's -# responsibility — if `init` collides with an existing branch on the prod's repo, the agent -# will see the error from its own `git push` or from `deploy_data_app`. -_DEFAULT_DRAFT_BRANCH = 'init' - -# How much of an AppRun's diagnostics is surfaced in tool output. `failure_message` can embed the -# entire startup log (StartupProbeFailed duplicates it verbatim), so both are trimmed to keep the -# tool output bounded while preserving the error tail, which is where the actionable line lives. -_APP_RUN_LOG_LINES = 30 -_APP_RUN_MESSAGE_LIMIT = 3000 - - -INJECTED_BLOCK_RE = re.compile( - r'(?P.*?)#\s###\sINJECTED_CODE\s####.*?#\s###\sEND_OF_INJECTED_CODE\s####(?P.*)', - re.DOTALL, -) - -# Type of the authentication used in the data app -AuthenticationType = Literal['no-auth', 'basic-auth', 'default'] - -SECRET_WORKSPACE_ID = 'WORKSPACE_ID' -SECRET_BRANCH_ID = 'BRANCH_ID' - -# Project feature that opts python-js data apps into platform-managed per-app workspaces. -# When enabled, the platform auto-provisions a workspace and injects WORKSPACE_ID at runtime. -# When disabled, the MCP falls back to passing WORKSPACE_ID via parameters.dataApp.secrets. -DATA_APPS_STORAGE_WORKSPACE_FEATURE = 'data-apps-storage-workspace' - - -class DataAppSummary(BaseModel): - """A summary of a data app used for sync operations.""" - - component_id: str = Field(description='The ID of the data app component.') - configuration_id: str = Field(description='The ID of the data app config.') - data_app_id: str = Field(description='The ID of the data app.') - project_id: str = Field(description='The ID of the project.') - branch_id: str = Field(description='The ID of the branch.') - config_version: str = Field(description='The version of the data app config.') - state: SafeState = Field(description='The state of the data app.') - type: SafeType = Field( - description=( - 'The type of the data app. Currently, only "streamlit" is supported in the MCP. However, Keboola DSAPI ' - 'supports additional types, which can be retrieved from the API.' - ) - ) - deployment_url: Optional[str] = Field(description='The URL of the running data app.', default=None) - auto_suspend_after_seconds: Optional[int] = Field( - description='The number of seconds after which the running data app is automatically suspended.', - default=None, - ) - repo_url: Optional[str] = Field( - default=None, - description=( - 'HTTPS clone URL of the managed git repo (without embedded credentials). ' - 'Only set for python-js data apps, and only populated by detail-style fetches ' - '(`get_data_apps(configuration_ids=[...])`) and `modify_python_js_data_app` ' - 'responses. The inventory list path (`get_data_apps` without `configuration_ids`) ' - 'always leaves this `None` to keep the listing cheap — call the detail path to ' - 'retrieve the URL. Mint a token via `create_python_js_data_app_git_credential` ' - 'to authenticate.' - ), - ) - - @classmethod - def from_api_response(cls, api_response: DataAppResponse) -> 'DataAppSummary': - return cls( - component_id=api_response.component_id, - configuration_id=api_response.config_id, - data_app_id=api_response.id, - project_id=api_response.project_id, - branch_id=api_response.branch_id or '', - config_version=api_response.config_version, - state=api_response.state, - type=api_response.type, - deployment_url=api_response.url, - auto_suspend_after_seconds=api_response.auto_suspend_after_seconds, - ) - - -class AppRunInfo(BaseModel): - """Outcome of a single deployment attempt (AppRun) of a data app.""" - - state: str = Field(description='The state of the run: "starting", "running", "finished" or "failed".') - created_at: Optional[str] = Field(description='The timestamp when the run was created.', default=None) - stopped_at: Optional[str] = Field( - description='The timestamp when the run stopped, or `null` while it is still active.', default=None - ) - failure_reason: Optional[str] = Field( - description=( - 'Machine-readable code identifying why the run failed, e.g. "ConfigDecryptionFailed" ' - 'or "StartupProbeFailed". `null` for successful runs.' - ), - default=None, - ) - failure_message: Optional[str] = Field( - description='Detailed explanation of the failure, when the platform provides one.', default=None - ) - startup_logs: list[str] = Field( - description="The last lines of the run's startup (entrypoint) log, when available.", - default_factory=list, - ) - - @classmethod - def from_api_response(cls, run: AppRunResponse) -> 'AppRunInfo': - startup_logs = (run.startup_logs or '').strip().rsplit('\n', _APP_RUN_LOG_LINES)[-_APP_RUN_LOG_LINES:] - failure_message = run.failure_reason.message if run.failure_reason else None - if failure_message and len(failure_message) > _APP_RUN_MESSAGE_LIMIT: - failure_message = '…' + failure_message[-(_APP_RUN_MESSAGE_LIMIT - 1) :] - return cls( - state=run.state, - created_at=run.created_at, - stopped_at=run.stopped_at, - failure_reason=run.failure_reason.reason if run.failure_reason else None, - failure_message=failure_message, - startup_logs=[line for line in startup_logs if line], - ) - - -class DeploymentInfo(BaseModel): - """Deployment information of a data app.""" - - version: str = Field(description='The version of the data app deployment.') - state: str = Field(description='The state of the data app deployment.') - url: Optional[str] = Field(description='The URL of the running data app deployment.', default=None) - last_request_timestamp: Optional[str] = Field( - description='The last request timestamp of the data app deployment.', default=None - ) - last_start_timestamp: Optional[str] = Field( - description='The last start timestamp of the data app deployment.', default=None - ) - logs: list[str] = Field( - description='The latest 20 log lines reported in the data app deployment.', default_factory=list - ) - last_run: Optional[AppRunInfo] = Field( - description=( - 'The most recent deployment attempt (AppRun). When the app failed to start, its ' - '`failure_reason`/`failure_message` explain why — including setup-phase failures ' - '(e.g. invalid secrets) that happen before the container starts and so produce no ' - 'container logs at all. Check this FIRST when diagnosing an app that does not start.' - ), - default=None, - ) - - -class DataApp(BaseModel): - """A data app used for detail views.""" - - name: str = Field(description='The name of the data app.') - description: Optional[str] = Field(description='The description of the data app.', default=None) - component_id: str = Field(description='The ID of the data app component.') - configuration_id: str = Field(description='The ID of the data app configuration.') - data_app_id: str = Field(description='The ID of the data app.') - project_id: str = Field(description='The ID of the project.') - branch_id: str = Field(description='The ID of the branch.') - config_version: str = Field(description='The version of the data app config.') - state: SafeState = Field(description='The state of the data app.') - type: SafeType = Field( - description=( - 'The type of the data app. Currently, only "streamlit" is supported in the MCP. However, Keboola DSAPI ' - 'supports additional types, which can be retrieved from the API.' - ) - ) - deployment_url: Optional[str] = Field(description='The URL of the running data app.', default=None) - auto_suspend_after_seconds: Optional[int] = Field( - description='The number of seconds after which the running data app is automatically suspended.', - default=None, - ) - repo_url: Optional[str] = Field( - default=None, - description=( - 'HTTPS clone URL of the managed git repo (without embedded credentials). ' - 'Only set for python-js data apps. Mint a token via ' - '`create_python_js_data_app_git_credential` to authenticate.' - ), - ) - configuration: dict[str, Any] = Field( - description='The nested configuration object containing parameters, storage and authorization' - ) - folder: str = Field(default='', description='The UI folder this data app is organized into') - deployment_info: Optional[DeploymentInfo] = Field( - description='Deployment info of the data app including a url of the app and logs to diagnose in-app errors.', - default=None, - ) - drafts: list[DataAppSummary] = Field( - default_factory=list, - description=( - 'Draft python-js data apps that iterate against this prod app (each carries ' - '`parameters.dataApp.parentConfigurationId == this.configuration_id`). Populated only ' - 'when the detail path is used for a python-js **prod** app — empty for drafts ' - 'themselves and for non-python-js apps.' - ), - ) - drafts_unavailable: int = Field( - default=0, - description=( - 'Count of drafts that exist for this prod app but whose details could not be fetched on ' - 'this call (transient DSAPI failure — expired token, timeout, 5xx). A non-zero value means ' - '`drafts` is INCOMPLETE: those drafts still exist and were NOT deleted, so do not treat ' - 'their absence as "no drafts" before a teardown decision. Retry to get the full list.' - ), - ) - links: list[Link] = Field(description='Navigation links for the web interface.', default_factory=list) - - @classmethod - def from_api_responses( - cls, - api_response: DataAppResponse, - api_configuration: ConfigurationAPIResponse, - ) -> 'DataApp': - return cls( - component_id=api_configuration.component_id, - configuration_id=api_configuration.configuration_id, - data_app_id=api_response.id, - project_id=api_response.project_id, - branch_id=api_response.branch_id or '', - config_version=str(api_configuration.version), - state=api_response.state, - type=api_response.type, - deployment_url=api_response.url, - auto_suspend_after_seconds=api_response.auto_suspend_after_seconds, - name=api_configuration.name, - description=api_configuration.description, - folder=get_metadata_property(api_configuration.metadata, MetadataField.CONFIGURATION_FOLDER_NAME) or '', - configuration=api_configuration.configuration, - deployment_info=None, - links=[], - ) - - def with_links(self, links: list[Link]) -> 'DataApp': - self.links = links - return self - - def with_deployment_info(self, logs: list[str], last_run: Optional[AppRunInfo] = None) -> 'DataApp': - """Adds deployment info to the data app. - - :param logs: The logs of the data app deployment. - :param last_run: The most recent deployment attempt (AppRun), when available. - :return: The data app with the deployment info. - """ - self.deployment_info = DeploymentInfo( - version=self.config_version, - state=self.state, - url=self.deployment_url or 'deployment link not available yet', - logs=logs, - last_run=last_run, - ) - return self - - -class ModifiedDataAppOutput(BaseModel): - """Modified data app output containing the response of the action performed and the data app and links to the web - interface.""" - - response: str = Field(description='The response of the action performed with potential additional information.') - change_summary: Optional[str] = Field(default=None, description='Additional notes or hints about the operation.') - data_app: DataAppSummary = Field(description='The data app.') - links: list[Link] = Field(description='Navigation links for the web interface.') - - -class ModifiedPythonJsDataAppOutput(BaseModel): - """Output for `modify_python_js_data_app`. Includes git repo URL on create.""" - - response: str = Field(description='The response of the action performed with potential additional information.') - change_summary: Optional[str] = Field(default=None, description='Additional notes or hints about the operation.') - data_app: DataAppSummary = Field(description='The data app.') - repo_url: Optional[str] = Field( - default=None, - description=( - 'HTTPS clone URL of the managed git repo (without embedded credentials). Returned on create so the ' - 'caller can clone the repo and push initial source code. On update, populated when the repo info can ' - "be fetched. On the draft create path this is the **parent prod app's** managed repo URL — that " - 'is the repo the agent should clone, branch, and push to. Mint a token via ' - '`create_python_js_data_app_git_credential` to authenticate (or use `git_clone_url` returned by this ' - 'tool on the draft create path).' - ), - ) - git_clone_url: Optional[str] = Field( - default=None, - description=( - 'Ready-to-use authenticated HTTPS clone URL embedding the freshly-minted prod-app token (format: ' - '`https://kai:@/.git`). Only populated on the **draft create path** — the ' - 'token was minted on the parent prod app and is one-time, so it is surfaced here so the agent can ' - 'clone immediately without a separate `create_python_js_data_app_git_credential` call. None on prod ' - 'create and on update.' - ), - ) - branch: Optional[str] = Field( - default=None, - description=( - 'Draft branch the new draft is pinned to (set in `parameters.dataApp.git.branch`). Only populated ' - 'on the **draft create path** — defaults to `init` when the caller does not pass `branch`. The ' - 'agent should `git checkout ` (creating it if needed) and push code on this branch before ' - 'calling `deploy_data_app(mode="dev")`. None on prod create and on update.' - ), - ) - links: list[Link] = Field(description='Navigation links for the web interface.') - - -class CreatedGitCredentialOutput(BaseModel): - """Output for `create_python_js_data_app_git_credential`.""" - - response: str = Field(description='The response of the action performed.') - configuration_id: str = Field(description='The Storage configuration ID of the python-js data app.') - data_app_id: str = Field(description='The ID of the data app the credential was created on.') - credential_id: str = Field(description='The ID of the created credential.') - git_clone_url: str = Field( - description=( - 'Ready-to-use HTTPS clone URL with the one-time token embedded (format: ' - '`https://kai:@/.git`). Pass directly to `git clone`.' - ), - ) - secret: str = Field( - description=( - 'One-time HTTPS token. Also embedded in `git_clone_url`. Surfaced separately so it can be plugged into ' - 'a `git credential` helper. **The platform does not return this value again** — store it if you need ' - 'to reuse it outside of `git_clone_url`.' - ), - ) - permissions: str = Field(description='The permissions of the credential, e.g. "readWrite".') - links: list[Link] = Field(description='Navigation links for the web interface.') - - -class DeploymentDataAppOutput(BaseModel): - """Deployment data app output containing the action performed, links and deployment info.""" - - state: SafeState = Field(description='The state of the data app deployment.') - deployment_info: DeploymentInfo | None = Field( - description='Deployment info with a link to the app and logs to diagnose in-app errors.', default=None - ) - links: list[Link] = Field(description='Navigation links for the web interface.') - - -class DeletedDraftOutput(BaseModel): - """Output for `delete_python_js_data_app_draft`.""" - - response: str = Field(description='Status of the delete operation, e.g. "deleted".') - configuration_id: str = Field(description='Storage configuration ID of the deleted draft.') - data_app_id: str = Field(description='Data-science API ID of the deleted draft data app.') - parent_configuration_id: Optional[str] = Field( - default=None, - description=( - 'Storage configuration ID of the parent prod app the draft was iterating against. ' - 'Surfaced so the agent can pivot back to the prod app after cleanup. May be None if ' - 'the draft was orphaned (parent already deleted).' - ), - ) - links: list[Link] = Field(description='Navigation links for the web interface.', default_factory=list) - - -class GetDataAppsOutput(BaseModel): - """Output of the get_data_apps tool. Serves for both DataAppSummary and DataApp outputs.""" - - data_apps: Sequence[DataAppSummary | DataApp] = Field(description='The data apps in the project.') - links: list[Link] = Field(description='Navigation links for the web interface.', default_factory=list) - - -@tool_errors() -async def modify_streamlit_data_app( - ctx: Context, - name: Annotated[str, Field(description='Name of the data app (max ~50 chars to fit DNS label limit).')], - description: Annotated[str, Field(description='Description of the data app.')], - source_code: Annotated[str, Field(description='Complete Python/Streamlit source code for the data app.')], - packages: Annotated[ - list[str], - Field( - description='Python packages used in the source code that will be installed by `pip install` ' - 'into the environment before the code runs. For example: ["pandas", "requests~=2.32"].' - ), - ], - authentication_type: Annotated[ - AuthenticationType, - Field( - description=( - 'Authentication type, "no-auth" removes authentication completely, "basic-auth" sets the data ' - 'app to be secured using the HTTP basic authentication, and "default" keeps the existing ' - 'authentication type when updating.' - ) - ), - ], - configuration_id: Annotated[ - str, Field(description='The ID of existing data app configuration when updating, otherwise empty string.') - ] = '', - change_description: Annotated[ - str, - Field(description='The description of the change when updating (e.g. "Update Code"), otherwise empty string.'), - ] = '', - folder: Annotated[ - Optional[str], - Field(description=folder_field_description('data app', 'data apps')), - ] = None, -) -> ModifiedDataAppOutput: - """Creates or updates a Streamlit data app. - - Considerations: - - The `source_code` parameter must be a complete and runnable Streamlit app. It must include a placeholder - `{QUERY_DATA_FUNCTION}` where a `query_data` function will be injected. This function queries the workspace to get - data, it accepts a string of SQL query following current sql dialect and returns a pandas DataFrame with the results - from the workspace. - - Write SQL queries so they are compatible with the current workspace backend, you can ensure this by using the - `query_data` tool to inspect the data in the workspace before using it in the data app. - - If you're updating an existing data app, provide the `configuration_id` parameter and the `change_description` - parameter. To keep existing data app values during an update, leave them as empty strings, lists, or None - appropriately based on the parameter type. - - After creating or updating a data app with this tool, ALWAYS call - `deploy_data_app(action="deploy", configuration_id=...)` to start a new app or restart an existing app so - changes take effect. Without this step, a newly created app will not start, and an existing app will keep - running the previous deployment without the latest changes. - - New apps use the HTTP basic authentication by default for security unless explicitly specified otherwise; when - updating, set `authentication_type` to `default` to keep the existing authentication type configuration - (including OIDC setups) unless explicitly specified otherwise. - - SQL & DATA TYPE RULES: - - Use delimited identifiers for the current SQL dialect for all column names and aliases in SQL. - Match the exact identifier case used in SQL when referencing columns in Python code. - - `query_data` RETURNS ALL COLUMNS AS STRINGS regardless of SQL CAST. Always convert types in Python after loading: - `df["col"] = pd.to_numeric(df["col"], errors="coerce").fillna(0)` and - `df["date"] = pd.to_datetime(df["date"], errors="coerce")`. - """ - client = KeboolaClient.from_state(ctx.session.state) - workspace_manager = WorkspaceManager.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - project_id = await client.storage_client.project_id() - workspace_id = await workspace_manager.get_workspace_id() - sql_dialect = await workspace_manager.get_sql_dialect() - branch_id = await workspace_manager.get_branch_id() - - secrets = _get_secrets( - workspace_id=str(workspace_id), - branch_id=str(branch_id), - ) - - if configuration_id: - # Update existing data app. - # - # `configuration_update` below is the ONLY step allowed to fail the whole tool: if it raises, nothing - # was committed and a ToolError truthfully reports the failure. Everything after it merely enriches the - # response (re-fetch, metadata, folder, links) and must NEVER turn a committed write into a reported - # failure -- otherwise the agent, seeing an error, retries and re-applies the change, producing the - # v27->v29 double-write that deployed broken code (AJDA-2852). Hence the best-effort block below. - data_app_pre, updated_config, _ = await modify_streamlit_data_app_internal( - client=client, - workspace_manager=workspace_manager, - name=name, - description=description, - source_code=source_code, - packages=packages, - authentication_type=authentication_type, - configuration_id=configuration_id, - change_description=change_description, - ) - update_resp = await client.storage_client.configuration_update( - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - configuration=updated_config, - change_description=change_description or 'Change Data App', - updated_name=name or data_app_pre.name, - updated_description=description or data_app_pre.description, - ) - # --- write committed past this point; response building is strictly best-effort --- - # The new version comes straight from the PUT response, so it is known even if the re-fetch below fails. - # Read it defensively: this runs after the committing write and must not raise (a non-dict response would - # otherwise turn the committed write into a ToolError -- the very failure mode this block guards against). - new_version = str(update_resp.get('version') or '') if isinstance(update_resp, dict) else '' - try: - # Only stamp the UPDATED_BY_MCP version when the new version is actually known. Falling back to the - # pre-update version would record a misleading version (the write did increment it), so skip the - # metadata write when the response carried no numeric version. - if new_version.isdigit(): - await set_cfg_update_metadata( - client=client, - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - configuration_version=int(new_version), - ) - folder_hint = await apply_folder_metadata( - client, DATA_APP_COMPONENT_ID, configuration_id, folder, 'data apps', 'modify_streamlit_data_app' - ) - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - links = links_manager.get_data_app_links( - configuration_id=data_app.configuration_id, - configuration_name=name, - deployment_link=data_app.deployment_url, - uses_basic_authentication=_uses_basic_authentication(data_app.configuration.get('authorization') or {}), - ) - response = ( - 'updated (redeploy required to apply changes in the running app)' - if data_app.state in ('running', 'starting') - else 'updated' - ) - return ModifiedDataAppOutput( - response=response, - change_summary=folder_hint, - data_app=DataAppSummary.model_validate(data_app.model_dump()), - links=links, - ) - except Exception: - LOG.exception( - 'Data app configuration %s was updated (version %s) but building the response failed; ' - 'returning a partial success so the change is not retried.', - configuration_id, - new_version or '?', - ) - return _partial_update_output( - data_app_pre=data_app_pre, - links_manager=links_manager, - configuration_id=configuration_id, - name=name, - new_version=new_version, - ) - else: - # Create new data app. As with the update path, `create_data_app` is the committing write; the - # metadata/folder/links steps after it are best-effort and must not fail the tool once the app exists, - # so a failed retry cannot create a duplicate app (AJDA-2852). - config = _build_data_app_config(name, source_code, packages, authentication_type, secrets, sql_dialect) - config = await client.encryption_client.encrypt( - config, component_id=DATA_APP_COMPONENT_ID, project_id=project_id - ) - validated_config = DataAppConfig.model_validate(config) - data_app_resp = await client.data_science_client.create_data_app( - name, description, configuration=validated_config - ) - # --- app created past this point; response building is strictly best-effort --- - try: - await set_cfg_creation_metadata( - client=client, - component_id=DATA_APP_COMPONENT_ID, - configuration_id=data_app_resp.config_id, - ) - folder_hint = await apply_folder_metadata( - client, - DATA_APP_COMPONENT_ID, - data_app_resp.config_id, - folder, - 'data apps', - 'modify_streamlit_data_app', - is_new=True, - ) - links = links_manager.get_data_app_links( - configuration_id=data_app_resp.config_id, - configuration_name=name, - deployment_link=data_app_resp.url, - uses_basic_authentication=_uses_basic_authentication(validated_config.authorization), - ) - return ModifiedDataAppOutput( - response='created', - change_summary=folder_hint, - data_app=DataAppSummary.from_api_response(data_app_resp), - links=links, - ) - except Exception: - LOG.exception( - 'Data app %s was created (configuration %s) but building the response failed; ' - 'returning a partial success so creation is not retried.', - data_app_resp.id, - data_app_resp.config_id, - ) - return _partial_create_output( - data_app_resp=data_app_resp, - links_manager=links_manager, - validated_config=validated_config, - name=name, - ) - - -async def modify_streamlit_data_app_internal( - *, - client: KeboolaClient, - workspace_manager: WorkspaceManager, - name: str, - description: str = '', - source_code: str, - packages: list[str], - authentication_type: AuthenticationType, - configuration_id: str, - change_description: str = '', - folder: Optional[str] = None, -) -> tuple[DataApp, JsonDict, dict | None]: - secrets = _get_secrets( - workspace_id=str(await workspace_manager.get_workspace_id()), - branch_id=str(await workspace_manager.get_branch_id()), - ) - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - existing_config = data_app.configuration - updated_config = _update_existing_data_app_config( - existing_config, - name, - source_code, - packages, - authentication_type, - secrets, - await workspace_manager.get_sql_dialect(), - ) - updated_config = cast( - JsonDict, - await client.encryption_client.encrypt( - updated_config, component_id=DATA_APP_COMPONENT_ID, project_id=await client.storage_client.project_id() - ), - ) - - folder_preview: dict | None = None - if folder is not None: - normalized_folder = folder.strip() - try: - current_metadata = await client.storage_client.configuration_metadata_get( - component_id=DATA_APP_COMPONENT_ID, configuration_id=configuration_id - ) - current_folder = next( - ( - m.get('value', '') - for m in current_metadata - if m.get('key') == MetadataField.CONFIGURATION_FOLDER_NAME - ), - '', - ) - if normalized_folder != current_folder: - folder_preview = {'original_folder': current_folder, 'updated_folder': normalized_folder} - except Exception as e: - LOG.warning( - 'Failed to fetch configuration metadata for folder preview ' - '(component_id=%s, configuration_id=%s): %s. Proceeding without folder preview.', - DATA_APP_COMPONENT_ID, - configuration_id, - e, - ) - - return data_app, updated_config, folder_preview - - -def _partial_update_output( - *, - data_app_pre: DataApp, - links_manager: ProjectLinksManager, - configuration_id: str, - name: str, - new_version: str, -) -> ModifiedDataAppOutput: - """Build a truthful partial-success response after a committed Streamlit update whose response building failed. - - The configuration write already landed, so this MUST NOT raise -- it is built entirely from the pre-update - data app (fetched before the write) plus the new version from the update response. The ``change_summary`` - tells the agent the change IS applied and must not be retried, preventing the duplicate-write loop. The - underlying exception is intentionally not surfaced to the caller (it is logged at the call site) -- the - agent-facing message stays stable and free of internal detail. - """ - try: - summary = DataAppSummary.model_validate(data_app_pre.model_dump()) - summary.config_version = new_version or summary.config_version - except Exception: - # Defensive: this helper must never raise (the write already committed). SafeState/SafeType accept any - # string, so building from the pre-update primitives cannot fail even if the schema is later tightened. - summary = DataAppSummary( - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - data_app_id=getattr(data_app_pre, 'data_app_id', '') or '', - project_id=getattr(data_app_pre, 'project_id', '') or '', - branch_id=getattr(data_app_pre, 'branch_id', '') or '', - config_version=new_version or getattr(data_app_pre, 'config_version', '') or '', - state=getattr(data_app_pre, 'state', 'unknown') or 'unknown', - type=getattr(data_app_pre, 'type', 'streamlit') or 'streamlit', - ) - try: - links = links_manager.get_data_app_links( - configuration_id=configuration_id, - configuration_name=name or data_app_pre.name, - deployment_link=data_app_pre.deployment_url, - uses_basic_authentication=_uses_basic_authentication(data_app_pre.configuration.get('authorization') or {}), - ) - except Exception: - links = [] - # Mirror the success-path wording: the redeploy hint only applies to a running/starting app. The pre-update - # state is a faithful proxy here -- a config write does not change deployment state on its own. - response = ( - 'updated (redeploy required to apply changes in the running app)' - if data_app_pre.state in ('running', 'starting') - else 'updated' - ) - return ModifiedDataAppOutput( - response=response, - change_summary=( - f'The configuration WAS updated (version {new_version or "unknown"}), but loading the full app ' - f'details failed, so this response is partial. Do NOT retry the update -- the change is already ' - f'applied. Call deploy_data_app to apply it to the running app.' - ), - data_app=summary, - links=links, - ) - - -def _partial_create_output( - *, - data_app_resp: DataAppResponse, - links_manager: ProjectLinksManager, - validated_config: DataAppConfig, - name: str, -) -> ModifiedDataAppOutput: - """Build a truthful partial-success response after a committed Streamlit create whose response building failed. - - The app already exists, so this MUST NOT raise -- it is built from the create response the API already - returned. The ``change_summary`` tells the agent the app IS created and must not be retried, preventing a - duplicate app. The underlying exception is intentionally not surfaced to the caller (it is logged at the - call site) -- the agent-facing message stays stable and free of internal detail. - """ - try: - summary = DataAppSummary.from_api_response(data_app_resp) - except Exception: - # Defensive: this helper must never raise (the app already exists). SafeState/SafeType accept any - # string, so building from the create-response primitives cannot fail even if the schema is tightened. - summary = DataAppSummary( - component_id=getattr(data_app_resp, 'component_id', DATA_APP_COMPONENT_ID) or DATA_APP_COMPONENT_ID, - configuration_id=getattr(data_app_resp, 'config_id', '') or '', - data_app_id=getattr(data_app_resp, 'id', '') or '', - project_id=getattr(data_app_resp, 'project_id', '') or '', - branch_id=getattr(data_app_resp, 'branch_id', '') or '', - config_version=getattr(data_app_resp, 'config_version', '') or '', - state=getattr(data_app_resp, 'state', 'unknown') or 'unknown', - type=getattr(data_app_resp, 'type', 'streamlit') or 'streamlit', - ) - try: - links = links_manager.get_data_app_links( - configuration_id=data_app_resp.config_id, - configuration_name=name, - deployment_link=data_app_resp.url, - uses_basic_authentication=_uses_basic_authentication(validated_config.authorization), - ) - except Exception: - links = [] - return ModifiedDataAppOutput( - response='created', - change_summary=( - f'The data app WAS created (configuration {data_app_resp.config_id}), but building the full response ' - f'failed, so this response is partial. Do NOT retry creation -- it would create a duplicate. ' - f'Call deploy_data_app to start the app.' - ), - data_app=summary, - links=links, - ) - - -@tool_errors() -async def modify_python_js_data_app( - ctx: Context, - name: Annotated[str, Field(description='Name of the data app (max ~50 chars to fit DNS label limit).')], - description: Annotated[str, Field(description='Description of the data app.')], - configuration_id: Annotated[ - str, Field(description='The ID of existing data app configuration when updating, otherwise empty string.') - ] = '', - change_description: Annotated[ - str, - Field(description='The description of the change when updating (e.g. "Bump image"), otherwise empty string.'), - ] = '', - slug: Annotated[ - Optional[str], - Field( - description=( - 'URL-safe slug for the data app (used as a subdomain). Required when creating; immutable after.' - ), - ), - ] = None, - parent_configuration_id: Annotated[ - Optional[str], - Field( - description=( - 'Storage configuration ID of the prod python-js data app this draft will iterate against. ' - 'When set on create, the new app is created as a **draft**: no managed repo is provisioned ' - "for it; instead its `parameters.dataApp.git` block is populated to point at the prod app's " - 'managed repo, with a freshly-minted prod-app HTTPS token and the chosen draft branch. ' - 'Leave None on create to make a **prod app** (which gets its own managed repo). Rejected on ' - 'update.' - ), - ), - ] = None, - branch: Annotated[ - Optional[str], - Field( - description=( - 'Draft branch to pin the new draft to. Only valid on the draft create path ' - '(when `parent_configuration_id` is set). Defaults to `init` when unset (a sensible ' - 'name for the first draft of a brand-new prod app). For subsequent edit-existing ' - "drafts, pass a descriptive branch name like 'add-revenue-filter'. Must not be `main` " - '(reserved for the prod app). Rejected on prod create and on update.' - ), - ), - ] = None, - authentication_type: Annotated[ - AuthenticationType, - Field( - description=( - 'Authentication type. "no-auth" removes authentication completely, "basic-auth" secures the ' - 'data app via HTTP basic authentication, and "default" means: on create, apply basic auth ' - '(safe default for new apps); on update, keep the existing authentication configuration ' - '(including OIDC setups configured outside the MCP).' - ), - ), - ] = 'default', - auto_suspend_after_seconds: Annotated[ - int, - Field( - description='Number of seconds after which the running data app is automatically suspended.', - ), - ] = 900, - storage: Annotated[ - Optional[dict[str, Any]], - Field( - description=( - 'Complete storage configuration for the data app (input/output table mappings). ' - 'Validated against the storage JSON schema. Replaces the ENTIRE storage block when ' - 'updating an existing app. For data apps with Storage Access, declare output tables ' - 'with `unload_strategy: "direct-grant"` (in that case `source` is not required and ' - 'the workspace is granted direct SELECT/INSERT/UPDATE/DELETE/TRUNCATE on the destination ' - 'Storage table). Leave unset (None) to preserve the existing storage configuration; ' - 'pass an empty dict to explicitly clear it.' - ), - ), - ] = None, - folder: Annotated[ - Optional[str], - Field(description=folder_field_description('data app', 'data apps')), - ] = None, -) -> ModifiedPythonJsDataAppOutput: - """Creates or updates a python-js data app. - - Two-app project model. Every python-js project has a persistent **prod app** that owns the - only managed git repository for the project, and zero or more **drafts** parented to that - prod app. A draft is a Storage configuration with `parameters.dataApp.isDraft=true` and - `parameters.dataApp.parentConfigurationId=`; it's an *external-git* app that - clones the parent prod's repo at a pinned branch on every deploy. Drafts are surfaced in the - Keboola UI under their parent prod app. Use `deploy_data_app(mode='dev')` to deploy a draft - as a dev version of the data app (hot reload + auto-auth for iframe preview); use - `delete_python_js_data_app_draft` to tear a draft down after its branch has been promoted. - - **MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, - branch-delete — is yours. MCP gives you authenticated clone URLs and manages configs/deploys; - it never invokes git. - - **The draft flow below is mandatory — never edit prod source directly.** Every source-code - change goes through a draft branch that the user previews and explicitly approves first. NEVER - push directly to `main`: `main` only ever advances by merging an approved draft branch, and - only after the user has approved that draft's preview. - - Three scenarios the agent has to distinguish: - - ## Scenario A — Create a brand-new data app - - 1. `modify_python_js_data_app(slug='demo')` → `(configuration_id=PROD, repo_url=R)`. - PROD owns the only managed repo for this app. - 2. `modify_python_js_data_app(slug='demo-draft', parent_configuration_id=PROD)` - → `(configuration_id=DRAFT, repo_url=R, git_clone_url=U, branch='init')`. - Default draft branch is `'init'`. Override with `branch=` for a descriptive name. - 3. YOU: `git clone U`; `git checkout init` (creating it if the repo is empty); write source; - `git push origin init`. - 4. `deploy_data_app(action='deploy', configuration_id=DRAFT, mode='dev')` - → preview URL serving the `init` branch as a dev version. Iterate with the user. - 5. Once approved — YOU: `git checkout main`; `git merge init`; `git push origin main`; - `git push origin --delete init`. - 6. `deploy_data_app(action='deploy', configuration_id=PROD)` - → prod URL now serves the merged `main`. - 7. `delete_python_js_data_app_draft(configuration_id=DRAFT)` - → tears down the draft's config + data-app instance. Always run this once promoted. - - ## Scenario B — Edit an existing data app - - You already have PROD's `configuration_id` (from `get_data_apps` or earlier conversation). - - 1. `create_python_js_data_app_git_credential(configuration_id=PROD)` - → fresh `git_clone_url U` with an embedded one-time token. - 2. `modify_python_js_data_app( - slug='demo-draft-', - parent_configuration_id=PROD, - branch='', # e.g. 'add-revenue-filter' - )` → `(DRAFT, R, U2, branch)`. Use U2 (it has its own fresh token). - 3. YOU: `git clone U2`; `git checkout ` (creating it from `main`); edit source; - `git push origin `. - 4–7. Same as Scenario A steps 4–7. - - ## Scenario C — Continue an unfinished draft - - The previous sandbox is gone. You have PROD's `configuration_id` but no working clone and no - draft handle. - - 1. `get_data_apps(configuration_ids=[PROD])` → returns PROD's detail including `drafts: [...]`. - Pick the draft the user means (ask if multiple and unclear). Each entry exposes its - `configuration_id`, slug, and pinned branch. - 2. `create_python_js_data_app_git_credential(configuration_id=PROD)` - → fresh `git_clone_url U` (the previous one was minted in a wiped sandbox and is lost). - Drafts have no managed repo of their own — always mint against PROD. - 3. YOU: `git clone U`; `git checkout `; resume work; `git push`. - 4. `deploy_data_app(action='deploy', configuration_id=, mode='dev')` → preview URL. - The draft's branch is already pinned in its config. - 5–7. Same promote/cleanup sequence as Scenario A steps 5–7. - - ## Argument rules - - - `parent_configuration_id` is **create-only**. Rejected on update. - - `branch` is **create-only** and only valid when `parent_configuration_id` is set. - Defaults to `'init'`. Must not be `'main'`. Rejected on prod create and on update. - - `slug` is required on create and immutable after. - - The **update path** (passing `configuration_id`) is for changing `name`, `description`, - `authentication_type`, `auto_suspend_after_seconds`, `storage` on either a prod app or - a draft. Source code changes go through the git flow above, not this tool. - - ## Authentication - - New apps default to HTTP basic authentication for safety. Pass `authentication_type='no-auth'` - to expose publicly. On update, `authentication_type='default'` preserves the existing - `authorization` block (including OIDC setups configured outside the MCP); `'basic-auth'` / - `'no-auth'` overwrite it. - - ## Slug constraint - - Must be DNS-label-safe (lowercase letters, digits, hyphens, ≤63 chars). For drafts, append a - short suffix (e.g. `-draft-abc123`) to keep slugs unique across the prod and its drafts. - """ - if configuration_id: - if slug: - raise ValueError('slug cannot be changed after the data app is created.') - if parent_configuration_id: - raise ValueError('parent_configuration_id is only valid when creating a draft (no configuration_id).') - if branch: - raise ValueError('branch is only valid when creating a draft (no configuration_id).') - else: - if not slug: - raise ValueError('slug is required when creating a python-js data app.') - if branch is not None and not parent_configuration_id: - raise ValueError('branch is only valid on the draft create path (pair it with parent_configuration_id).') - - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - validated_storage = _validate_data_app_storage(storage, configuration_id=configuration_id or None) - - # When the platform-managed workspace feature is off, the data app cannot rely on the - # platform to inject WORKSPACE_ID; fall back to passing it via parameters.dataApp.secrets. - has_storage_workspace = await client.has_feature(DATA_APPS_STORAGE_WORKSPACE_FEATURE) - legacy_secrets: Optional[dict[str, Any]] = None - if not has_storage_workspace: - workspace_manager = WorkspaceManager.from_state(ctx.session.state) - legacy_secrets = {SECRET_WORKSPACE_ID: str(await workspace_manager.get_workspace_id())} - - if configuration_id: - # Update existing python-js data app - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - updated_config = _update_existing_code_data_app_config( - existing_config=data_app.configuration, - auto_suspend_after_seconds=auto_suspend_after_seconds, - authentication_type=authentication_type, - secrets=legacy_secrets, - storage=validated_storage, - ) - await client.storage_client.configuration_update( - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - configuration=updated_config, - change_description=change_description or 'Update python-js data app', - updated_name=name or data_app.name, - updated_description=description or data_app.description, - ) - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - await set_cfg_update_metadata( - client=client, - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - configuration_version=int(data_app.config_version), - ) - folder_hint = await apply_folder_metadata( - client, DATA_APP_COMPONENT_ID, configuration_id, folder, 'data apps', 'modify_python_js_data_app' - ) - repo_url = data_app.repo_url - links = links_manager.get_data_app_links( - configuration_id=data_app.configuration_id, - configuration_name=name or data_app.name, - deployment_link=data_app.deployment_url, - uses_basic_authentication=_uses_basic_authentication(data_app.configuration.get('authorization') or {}), - ) - response = ( - 'updated (redeploy required to apply changes in the running app)' - if data_app.state in ('running', 'starting') - else 'updated' - ) - data_app_summary = DataAppSummary.model_validate(data_app.model_dump()) - data_app_summary.repo_url = repo_url - return ModifiedPythonJsDataAppOutput( - response=response, - change_summary=folder_hint, - data_app=data_app_summary, - repo_url=repo_url, - links=links, - ) - else: - # Create new python-js data app — either a prod app (own managed repo) or a draft - # (external-git binding pointing at the parent prod app's managed repo). - # Narrowed by the validation block at the top of this function. - assert slug is not None - # On create, treat 'default' as 'basic-auth' (safe-by-default) to match modify_streamlit_data_app. - uses_basic_auth = authentication_type in ('basic-auth', 'default') - authorization_model = DataAppConfig.Authorization.model_validate(_get_authorization(uses_basic_auth)) - - git_clone_url: Optional[str] = None - draft_branch: Optional[str] = None - git_block: Optional[CodeDataAppConfig.Parameters.DataApp.Git] = None - if parent_configuration_id: - # Draft create path: resolve the parent's repo + mint a parent-side credential, then - # serialize an external-git block into the draft's config. - parent = await _fetch_data_app(client, configuration_id=parent_configuration_id, data_app_id=None) - if parent.type != 'python-js': - raise ValueError( - f'parent_configuration_id "{parent_configuration_id}" is type "{parent.type}", but only ' - f'python-js prod apps can parent a draft.' - ) - if _is_draft_config(parent.configuration): - # A draft has no managed repo of its own and cannot parent another draft. Reject it - # explicitly instead of falling through to the misleading "no repo URL" error below. - raise ValueError( - f'parent_configuration_id "{parent_configuration_id}" is itself a python-js **draft**, ' - "not a prod app. Drafts iterate against the prod app's repo and cannot parent another " - "draft — pass the prod app's configuration_id (a draft's parentConfigurationId points to it)." - ) - if not parent.repo_url: - raise ValueError( - f'Parent python-js data app "{parent_configuration_id}" has no managed git repo URL. ' - 'This indicates a platform-side bug — retry or contact support.' - ) - draft_branch = (branch or _DEFAULT_DRAFT_BRANCH).strip() - if not draft_branch or any(c.isspace() for c in draft_branch): - raise ValueError(f'branch "{branch}" is not a valid git branch name.') - if draft_branch == 'main': - raise ValueError('branch "main" is reserved for the prod app — pick a different draft branch.') - cred = await client.data_science_client.create_app_git_credential(parent.data_app_id) - if not cred.secret: - raise ValueError( - f'Parent data app {parent.data_app_id} credentials endpoint returned no `secret` for an ' - f'http_token credential. This indicates a platform-side bug — retry or contact support.' - ) - git_block = CodeDataAppConfig.Parameters.DataApp.Git( - repository=parent.repo_url, - username=_MANAGED_GIT_REPO_USERNAME, - password=cred.secret, - branch=draft_branch, - ) - git_clone_url = _build_authenticated_clone_url(parent.repo_url, cred.secret) - - config = CodeDataAppConfig( - parameters=CodeDataAppConfig.Parameters( - auto_suspend_after_seconds=auto_suspend_after_seconds, - data_app=CodeDataAppConfig.Parameters.DataApp( - slug=slug, - secrets=legacy_secrets, - git=git_block, - is_draft=True if parent_configuration_id is not None else None, - parent_configuration_id=parent_configuration_id, - ), - ), - runtime=( - CodeDataAppConfig.Runtime(workspace=CodeDataAppConfig.Runtime.Workspace(enabled=True)) - if has_storage_workspace - else None - ), - authorization=authorization_model, - # An empty (or all-empty) storage block prunes to `{}`; omit it entirely rather than - # persisting an empty object that the backend would store as `[]` (AI-3135). - storage=validated_storage or None, - ) - if git_block is not None: - # The git block's `#password` is plaintext at this point; the encryption service walks - # the dict and only encrypts keys starting with `#`, so everything else is untouched. - project_id = await client.storage_client.project_id() - config_payload = cast(dict[str, Any], config.model_dump(by_alias=True, exclude_none=True)) - encrypted_payload = await client.encryption_client.encrypt( - config_payload, - component_id=DATA_APP_COMPONENT_ID, - project_id=project_id, - ) - config = CodeDataAppConfig.model_validate(encrypted_payload) - data_app_resp = await client.data_science_client.create_data_app( - name=name, - description=description, - configuration=config, - app_type='python-js', - # Dev twins bring their own external-git binding; only prod apps get a managed repo. - use_managed_git_repo=parent_configuration_id is None, - ) - if parent_configuration_id: - # Dev twin: the repo the agent must clone is the parent prod's managed repo. - assert git_block is not None - repo_url = git_block.repository - else: - repo_resp = await client.data_science_client.get_app_git_repo(data_app_resp.id) - if repo_resp.https_url is None: - raise ValueError( - f'Data app {data_app_resp.id} reports no HTTPS clone URL despite having a managed git repo. ' - 'This indicates a platform-side bug — retry or contact support.' - ) - repo_url = repo_resp.https_url - await set_cfg_creation_metadata( - client=client, - component_id=DATA_APP_COMPONENT_ID, - configuration_id=data_app_resp.config_id, - ) - folder_hint = await apply_folder_metadata( - client, - DATA_APP_COMPONENT_ID, - data_app_resp.config_id, - folder, - 'data apps', - 'modify_python_js_data_app', - is_new=True, - ) - links = links_manager.get_data_app_links( - configuration_id=data_app_resp.config_id, - configuration_name=name, - deployment_link=data_app_resp.url, - uses_basic_authentication=uses_basic_auth, - ) - data_app_summary = DataAppSummary.from_api_response(data_app_resp) - data_app_summary.repo_url = repo_url - return ModifiedPythonJsDataAppOutput( - response='created', - change_summary=folder_hint, - data_app=data_app_summary, - repo_url=repo_url, - git_clone_url=git_clone_url, - branch=draft_branch, - links=links, - ) - - -@tool_errors() -async def create_python_js_data_app_git_credential( - ctx: Context, - configuration_id: Annotated[str, Field(description='Storage configuration ID of the python-js data app.')], -) -> CreatedGitCredentialOutput: - """Mints a one-time HTTPS token on a python-js **prod** data app so the caller can clone, pull, - and push to the app's managed git repo over HTTPS. - - **Always call against the prod app's configuration_id** — drafts have no managed repo of their - own, so calling this on a draft fails. The prod app is the canonical repo owner; drafts - iterate against branches of that same repo. - - **MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, - branch-delete — is yours. This tool only mints credentials. - - Returns a ready-to-use `git_clone_url` of the form `https://kai:@/.git` - plus the raw `secret`. The token is returned **only** at creation — the platform cannot return - it again on any subsequent read. Stash the URL (or the secret) somewhere the LLM can reuse for - the rest of the session. - - The data-science API accepts multiple credentials per app, so calling this again mints an - additional token without invalidating any tokens already held by other clients. - - ## When to call - - 1. **Right after `modify_python_js_data_app` create of a prod app** — the new prod has a - managed repo but no credentials yet. Call this tool with the new app's `configuration_id` - to enable git access. (Note: when creating a **draft**, the prod-side token is minted and - embedded into the returned `git_clone_url` automatically — no separate call needed.) - - 2. **Recovery when the cached token is gone / continuing an unfinished draft** — e.g., a fresh - sandbox continuing yesterday's work, with the previous sandbox's filesystem wiped. The - cached `git_clone_url` is lost; the configuration ID for the prod app is all you have. - Call this tool with the **prod app's** `configuration_id` to mint a fresh token (drafts - have no managed repo, so always mint against prod). Existing credentials remain valid, so - other clients are not disrupted. - - ## Constraints - - Only python-js prod data apps have a managed git repo. Streamlit apps reject the call with - a clear error. - - Permissions are always `readWrite` — the LLM virtually always needs push access. The - data-science API supports read-only credentials, but the tool does not expose that knob; - revisit once a real use case appears. - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - if data_app.type != 'python-js': - raise ValueError( - f'create_python_js_data_app_git_credential only supports python-js data apps, but configuration ' - f'"{configuration_id}" is type "{data_app.type}".' - ) - if _is_draft_config(data_app.configuration): - # Drafts have no managed repo of their own — they iterate against branches of the parent - # prod's repo. Reject early with an actionable message instead of letting get_app_git_repo - # return https_url=None below and raising a misleading "platform-side bug" error. - data_app_block = cast(Mapping[str, Any], data_app.configuration.get('parameters') or {}).get('dataApp') or {} - parent_cfg_id = data_app_block.get('parentConfigurationId') - hint = f' (parentConfigurationId="{parent_cfg_id}")' if isinstance(parent_cfg_id, str) else '' - raise ValueError( - f'Configuration "{configuration_id}" is a python-js **draft**, which has no managed git repo ' - f'of its own. Mint credentials against the parent prod app instead{hint}.' - ) - - repo_resp = await client.data_science_client.get_app_git_repo(data_app.data_app_id) - if repo_resp.https_url is None: - raise ValueError( - f'Data app {data_app.data_app_id} reports no HTTPS clone URL despite being a python-js managed-repo ' - f'app. This indicates a platform-side bug — retry or contact support.' - ) - - credential_resp = await client.data_science_client.create_app_git_credential( - data_app_id=data_app.data_app_id, - ) - if not credential_resp.secret: - raise ValueError( - f'Data app {data_app.data_app_id} credentials endpoint returned no `secret` for an http_token ' - f'credential. This indicates a platform-side bug — retry or contact support.' - ) - - git_clone_url = _build_authenticated_clone_url(repo_resp.https_url, credential_resp.secret) - links = links_manager.get_data_app_links( - configuration_id=data_app.configuration_id, - configuration_name=data_app.name, - deployment_link=data_app.deployment_url, - uses_basic_authentication=False, - ) - return CreatedGitCredentialOutput( - response='created', - configuration_id=data_app.configuration_id, - data_app_id=data_app.data_app_id, - credential_id=credential_resp.id, - git_clone_url=git_clone_url, - secret=credential_resp.secret, - permissions=credential_resp.permissions, - links=links, - ) - - -def _build_authenticated_clone_url(https_url: str, secret: str) -> str: - """Embed the hardcoded git-service username and the one-time `secret` into the bare HTTPS URL - so the LLM can pass it straight to `git clone`. - """ - parts = urlsplit(https_url) - if not parts.scheme or not parts.netloc: - raise ValueError(f'Could not parse HTTPS clone URL: {https_url!r}') - # Strip any pre-existing userinfo (the GET /git-repo endpoint already strips credentials, - # but be defensive). - host = parts.hostname or '' - if parts.port is not None: - host = f'{host}:{parts.port}' - netloc = f'{_MANAGED_GIT_REPO_USERNAME}:{quote(secret, safe="")}@{host}' - return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) - - -def _prune_empty_storage_objects(value: Any) -> Any: - """Recursively drop empty-object (``{}``) values from a data-app ``storage`` block. - - The Keboola backend (PHP/SAPI) cannot tell an empty JSON object from an empty array and - serializes ``{}`` as ``[]``. The UI's Input/Output Mapping editor (Writable Tables) then - silently refuses to add or save entries when ``storage`` — or one of its ``input``/``output`` - containers — is an array instead of an object (AI-3135). We therefore never persist an empty - object inside the storage block; the platform recreates a correctly shaped one when the user - adds the first mapping. Empty *arrays* (e.g. the canonical ``{"output": {"tables": []}}``) are - intentionally preserved. - """ - if isinstance(value, dict): - pruned: dict[str, Any] = {} - for key, sub_value in value.items(): - pruned_sub = _prune_empty_storage_objects(sub_value) - if isinstance(pruned_sub, dict) and not pruned_sub: - continue - pruned[key] = pruned_sub - return pruned - if isinstance(value, list): - return [_prune_empty_storage_objects(item) for item in value] - return value - - -def _normalize_config_storage(config: dict[str, Any]) -> None: - """Normalize the ``storage`` block of a data-app config dict in place before persisting. - - Drops empty mapping containers and removes the ``storage`` key entirely when it carries no - mappings (or is a non-object leftover such as ``[]``), so the broken array shape that breaks - the Writable Tables editor is never written or left behind (AI-3135). - """ - if 'storage' not in config: - return - storage = config['storage'] - pruned = _prune_empty_storage_objects(storage) if isinstance(storage, dict) else None - if pruned: - config['storage'] = pruned - else: - config.pop('storage', None) - - -def _validate_data_app_storage( - storage: Optional[dict[str, Any]], - *, - configuration_id: Optional[str] = None, -) -> Optional[dict[str, Any]]: - """Validate a caller-provided storage block for a data app. - - Returns the validated `storage` dict, or None when no storage was provided (caller - should preserve the existing storage configuration). - - The storage component-type rules in `validate_root_storage_configuration` (writer / SQL - transformation special cases) don't apply to data apps — we just run the JSON-schema check. - """ - if storage is None: - return None - # Accept both raw `storage` dict and pre-wrapped {'storage': storage}, mirroring the - # behavior of validate_root_storage_configuration. - storage_cfg = cast(dict[str, Any], storage.get('storage', storage)) if storage else {} - normalized = cast(dict[str, Any], {'storage': storage_cfg}) - validation_context = ValidationContext( - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - scope='storage', - ) - validated = validate_storage_configuration_against_schema( - normalized, - initial_message='The "storage" field is not valid.', - validation_context=validation_context, - ) - # Strip empty mapping containers so we never persist an empty object that the backend would - # serialize as `[]`, which silently breaks the Writable Tables editor (AI-3135). An all-empty - # block prunes down to `{}` (treated as a wipe by the callers). - return cast(dict[str, Any], _prune_empty_storage_objects(validated['storage'])) - - -def _update_existing_code_data_app_config( - existing_config: Mapping[str, Any], - auto_suspend_after_seconds: int, - authentication_type: AuthenticationType = 'default', - secrets: Optional[dict[str, Any]] = None, - storage: Optional[dict[str, Any]] = None, -) -> dict[str, Any]: - """Apply requested updates to the existing python-js data app storage configuration. - - Slug is intentionally not updated here (immutable post-create). `runtime.image.version` is - not touched either — the platform now picks a default for python-js apps, and any legacy - `image.version` pin already in the stored config is preserved verbatim via deepcopy. - `authentication_type='default'` preserves the existing `authorization` block (including OIDC - setups configured outside the MCP); 'no-auth' / 'basic-auth' overwrite it. - `secrets` are merged into the existing `parameters.dataApp.secrets` map without overwriting - keys already present. Used on projects without the `data-apps-storage-workspace` feature to - inject WORKSPACE_ID; on projects with the feature, pass None. - `storage` replaces the entire `storage` block when provided (None preserves the existing one; - an empty dict — or one that prunes down to nothing — is an explicit wipe that removes the - `storage` key entirely). Whatever storage ends up in the config is normalized so empty mapping - containers never persist as `[]` and break the Writable Tables editor (AI-3135). - """ - new_config = cast(dict[str, Any], copy.deepcopy(existing_config)) - new_config.setdefault('parameters', {}) - new_config['parameters']['autoSuspendAfterSeconds'] = auto_suspend_after_seconds - if authentication_type != 'default': - new_config['authorization'] = _get_authorization(authentication_type == 'basic-auth') - if secrets: - data_app = new_config['parameters'].setdefault('dataApp', {}) - updated_secrets = dict(data_app.get('secrets') or {}) - for key, value in secrets.items(): - if key not in updated_secrets: - updated_secrets[key] = value - data_app['secrets'] = updated_secrets - if storage is not None: - new_config['storage'] = storage - _normalize_config_storage(new_config) - return new_config - - -@tool_errors() -async def get_data_apps( - ctx: Context, - configuration_ids: Annotated[Sequence[str], Field(description='The IDs of the data app configurations.')] = tuple(), - limit: Annotated[int, Field(description='The limit of the data apps to fetch.')] = 100, - offset: Annotated[int, Field(description='The offset of the data apps to fetch.')] = 0, -) -> GetDataAppsOutput: - """Lists summaries of data apps in the project given the limit and offset or gets details of a data apps by - providing their configuration IDs. - - WHEN NOT TO USE: - - Do NOT list all data apps just to find one by name. Use `search` with - item_types=["data-app"] instead. - - Only list all data apps when you need a complete inventory. - - Considerations: - - If configuration_ids are provided, the tool will return details of the data apps by their configuration IDs. - - If no configuration_ids are provided, the tool will list all data apps in the project given the limit and offset. - - Data App detail contains configuration, metadata, source code, links, and deployment info along with the latest - data app logs to investigate in-app errors. The logs may be updated after opening the data app URL. - - `deployment_info.last_run` carries the outcome of the most recent deployment attempt. For an app - that fails to start, check its `failure_reason`/`failure_message` FIRST — they cover setup-phase - failures (e.g. invalid secrets, git clone errors, failing setup scripts) that happen before the - container starts and therefore never appear in the regular logs. - - `repo_url` (managed git repo URL for python-js apps) is ONLY populated on the detail path - (when `configuration_ids` is provided). The inventory list always returns `repo_url=None`, - even for python-js apps with a managed repo — to retrieve the URL, call this tool again - with the target `configuration_ids`. - - When called with `configuration_ids=[]` for a python-js **prod** app, the response - includes a `drafts: [...]` array of every draft (configs with `isDraft=true` and - `parentConfigurationId == `) currently in the project. Drafts in trash are not - included. Use this to discover existing drafts when continuing a previously abandoned - iteration (Scenario C in `modify_python_js_data_app`). The array is empty for drafts - themselves and for Streamlit apps. - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - if configuration_ids: - # Get details of the data apps by their configuration IDs using 10 parallel requests at a time to not overload - # the API - async def fetch_data_app_detail(configuration_id: str) -> DataApp | str: - return await _fetch_data_app_details_task(client, links_manager, configuration_id) - - data_app_details = await process_concurrently(configuration_ids, fetch_data_app_detail, max_concurrency=10) - found_data_apps: list[DataApp] = [dap for dap in data_app_details if isinstance(dap, DataApp)] - not_found_ids: list[str] = [dap for dap in data_app_details if isinstance(dap, str)] - if not_found_ids: - LOG.error(f'Could not find Data Apps Configurations for IDs: {not_found_ids}') - return GetDataAppsOutput(data_apps=found_data_apps) - else: - # List all data apps in the project - data_apps: list[DataAppResponse] = await client.data_science_client.list_data_apps(limit=limit, offset=offset) - # Filter to only include keboola.data-apps component - data_apps = [app for app in data_apps if app.component_id == DATA_APP_COMPONENT_ID] - links = [links_manager.get_data_app_dashboard_link()] - return GetDataAppsOutput( - data_apps=[DataAppSummary.from_api_response(data_app) for data_app in data_apps], - links=links, - ) - - -@tool_errors() -async def deploy_data_app( - ctx: Context, - action: Annotated[Literal['deploy', 'stop'], Field(description='The action to perform.')], - configuration_id: Annotated[str, Field(description='The ID of the data app configuration.')], - mode: Annotated[ - Optional[Literal['dev', 'production']], - Field( - description=( - 'Deployment mode. Set to "dev" to deploy a python-js draft as a **dev version of the data ' - 'app** — the runtime uses a development `setup.sh` (hot reload), and the data-app proxy ' - 'enables an auto-auth path so an iframe preview can render without a manual login. ' - 'Only meaningful on **draft** configs (python-js apps with `isDraft=true`). Leave None ' - '(default) for prod redeploys and for Streamlit apps.' - ), - ), - ] = None, -) -> DeploymentDataAppOutput: - """Deploys/redeploys a data app or stops a running data app in the Keboola environment asynchronously, given the - action and the configuration ID. - - **MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, - branch-delete — is yours. This tool only triggers deploys against existing git state. - - ## Mode (python-js apps) - - `mode='dev'` deploys the target as a **dev version of the data app** — the runtime uses a - development `setup.sh` (hot reload) and the data-app proxy enables an auto-auth path so an - iframe preview can render without a manual login. Only meaningful on **draft** configs - (python-js apps with `isDraft=true`). - - For prod redeploys (including after merging a draft's branch into `main`), use no `mode` — - the prod app picks up the current `main`. - - The branch a draft deploys from is pinned in `parameters.dataApp.git.branch` at create time; - there is no deploy-time override. - - python-js apps do NOT fetch a Storage `configVersion` for deployment (their source lives in - git, not in the Storage configuration); this is handled automatically. - - ## Streamlit apps - Streamlit apps have no managed git repo, so `mode` has no effect on the deployed app. - `mode=None` is the expected call shape. - - ## General considerations - - Redeploying a data app takes some time, and the app may temporarily report status "stopped" during the - restart. - - After deployment, the deployment info includes the app URL and the latest logs to help diagnose in-app - errors. - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - if action == 'deploy': - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - if data_app.state == 'stopping': - raise ValueError('Data app is currently "stopping", could not be started at the moment.') - # python-js apps don't carry a Storage configVersion in the deploy payload; only Streamlit apps do. - if data_app.type == 'python-js': - config_version_arg: str | None = None - else: - config_version = await client.storage_client.configuration_version_latest( - DATA_APP_COMPONENT_ID, data_app.configuration_id - ) - config_version_arg = str(config_version) - _ = await client.data_science_client.deploy_data_app( - data_app.data_app_id, - config_version_arg, - mode=mode, - ) - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - data_app = data_app.with_deployment_info( - await _fetch_logs(client, data_app.data_app_id), - last_run=await _fetch_latest_run(client, data_app.data_app_id), - ) - links = links_manager.get_data_app_links( - configuration_id=data_app.configuration_id, - configuration_name=data_app.name, - deployment_link=data_app.deployment_url, - uses_basic_authentication=_uses_basic_authentication(data_app.configuration.get('authorization') or {}), - ) - return DeploymentDataAppOutput(state=data_app.state, links=links, deployment_info=data_app.deployment_info) - elif action == 'stop': - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - if data_app.state in ('starting', 'restarting'): - raise ValueError('Data app is currently "starting", could not be stopped at the moment.') - _ = await client.data_science_client.suspend_data_app(data_app.data_app_id) - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - links = links_manager.get_data_app_links( - configuration_id=data_app.configuration_id, - configuration_name=data_app.name, - deployment_link=None, - uses_basic_authentication=_uses_basic_authentication(data_app.configuration.get('authorization') or {}), - ) - return DeploymentDataAppOutput(state=data_app.state, links=links, deployment_info=None) - else: - raise ValueError(f'Invalid action: {action}') - - -@tool_errors() -async def delete_python_js_data_app_draft( - ctx: Context, - configuration_id: Annotated[ - str, Field(description='Storage configuration ID of the python-js draft data app to delete.') - ], -) -> DeletedDraftOutput: - """Deletes a python-js DRAFT data app — both the data-app instance (DSAPI) and its Storage - configuration. - - **MCP never runs git on your behalf.** Deleting the feature branch on the remote is your job; - this tool only tears down the draft config and its data-app instance. - - WHEN TO CALL: at the end of a promote-to-prod sequence, after you have merged the draft's - branch into `main`, pushed, deleted the feature branch from the remote, and redeployed the - prod app. The Keboola UI lists drafts under their parent prod app; once you call this tool, - the draft disappears from that list. - - WHAT THIS TOOL REFUSES: - - prod apps (no `isDraft` flag) — protects against accidental prod deletion; - - Streamlit apps — they have no draft concept. - - WHAT THIS TOOL DOES NOT DO: - - Run git. Deleting the feature branch on the remote is your job. - - Revoke the prod-side git credential minted when the draft was created. Credential - rotation is the user's job via the Keboola UI. - - After a successful call, pivot back to the parent prod app (its configuration_id is returned - in the response) or to `get_data_apps` for further work. - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - if data_app.type != 'python-js': - raise ValueError( - f'delete_python_js_data_app_draft only supports python-js data apps, but configuration ' - f'"{configuration_id}" is type "{data_app.type}".' - ) - if not _is_draft_config(data_app.configuration): - raise ValueError( - f'Configuration "{configuration_id}" is a python-js **prod** app, not a draft ' - '(parameters.dataApp.isDraft is not true). This tool only deletes drafts — ' - 'prod apps must be deleted from the Keboola UI.' - ) - - data_app_block = cast(Mapping[str, Any], data_app.configuration.get('parameters') or {}).get('dataApp') or {} - parent_cfg_id = data_app_block.get('parentConfigurationId') - parent_configuration_id: Optional[str] = parent_cfg_id if isinstance(parent_cfg_id, str) else None - - # DSAPI deletes the data app and moves its Storage config to the trash. Don't delete the config - # via Storage API on top of that — deleting an already-trashed config purges it from the trash - # (even with skip_trash=False), making the draft unrestorable. - await client.data_science_client.delete_data_app(data_app.data_app_id) - - # When a parent prod app is known, the links pivot to it. We don't have the parent's name here, - # so label it explicitly as the parent rather than reusing the (now-deleted) draft's name, which - # would mislabel a link that points at a different configuration. - links = links_manager.get_data_app_links( - configuration_id=parent_configuration_id or configuration_id, - configuration_name='parent prod app' if parent_configuration_id else data_app.name, - deployment_link=None, - uses_basic_authentication=False, - ) - return DeletedDraftOutput( - response='deleted', - configuration_id=configuration_id, - data_app_id=data_app.data_app_id, - parent_configuration_id=parent_configuration_id, - links=links, - ) - - -def _build_data_app_config( - name: str, - source_code: str, - packages: list[str], - authentication_type: AuthenticationType, - secrets: dict[str, Any], - sql_dialect: str, -) -> dict[str, Any]: - packages = sorted(list(set(packages + _DEFAULT_PACKAGES))) - slug = _get_data_app_slug(name) or 'Data-App' - parameters = { - 'size': 'tiny', - 'autoSuspendAfterSeconds': 900, - 'dataApp': { - 'slug': slug, - 'streamlit': { - 'config.toml': _DEFAULT_STREAMLIT_THEME, - }, - 'secrets': secrets, - }, - 'script': [_inject_query_to_source_code(source_code, sql_dialect)], - 'packages': packages, - } - # By default secure with basic authorization - authorization = _get_authorization(authentication_type in ['basic-auth', 'default']) - return {'parameters': parameters, 'authorization': authorization} - - -def _update_existing_data_app_config( - existing_config: Mapping[str, Any], - name: str, - source_code: str, - packages: list[str], - authentication_type: AuthenticationType, - secrets: dict[str, Any], - sql_dialect: str, -) -> dict[str, Any]: - new_config = cast(dict[str, Any], copy.deepcopy(existing_config)) - new_config['parameters']['dataApp']['slug'] = ( - _get_data_app_slug(name) or existing_config['parameters']['dataApp']['slug'] - ) - if source_code: - new_config['parameters']['script'] = [_inject_query_to_source_code(source_code, sql_dialect)] - new_config['parameters']['packages'] = ( - sorted(list[str](set[str](packages + _DEFAULT_PACKAGES))) - if packages - else sorted(list[str](set[str](existing_config['parameters'].get('packages', []) + _DEFAULT_PACKAGES))) - ) - - updated_secrets = existing_config['parameters']['dataApp'].get('secrets', {}).copy() - # Add new secrets, do not overwrite existing secrets - for key in secrets: - if key not in updated_secrets: - updated_secrets[key] = secrets[key] - - new_config['parameters']['dataApp']['secrets'] = updated_secrets - - if authentication_type != 'default': - new_config['authorization'] = _get_authorization(authentication_type == 'basic-auth') - # Clean up any empty/array-shaped storage left behind by earlier MCP/CLI/KAI writes so the - # Writable Tables editor keeps working after a re-save (AI-3135). - _normalize_config_storage(new_config) - return new_config - - -async def _fetch_data_app( - client: KeboolaClient, - *, - data_app_id: Optional[str], - configuration_id: Optional[str], -) -> DataApp: - """ - Fetches data app from both data-science API and storage API based on the provided data_app_id or - configuration_id. - - :param client: The Keboola client - :param data_app_id: The ID of the data app - :param configuration_id: The ID of the configuration - :return: The data app - """ - - if data_app_id: - # Fetch data app from science API to get the configuration ID - data_app_science = await client.data_science_client.get_data_app(data_app_id) - if data_app_science.component_id != DATA_APP_COMPONENT_ID: - raise ValueError( - f'Data app tools only support {DATA_APP_COMPONENT_ID} component, but the data app ' - f'"{data_app_id}" has component_id "{data_app_science.component_id}".' - ) - raw_data_app_config = await client.storage_client.configuration_detail( - component_id=DATA_APP_COMPONENT_ID, configuration_id=data_app_science.config_id - ) - api_config = ConfigurationAPIResponse.model_validate( - raw_data_app_config | {'component_id': DATA_APP_COMPONENT_ID} - ) - return await _build_data_app_with_repo(client, data_app_science, api_config) - elif configuration_id: - raw_configuration = await client.storage_client.configuration_detail( - component_id=DATA_APP_COMPONENT_ID, configuration_id=configuration_id - ) - api_config = ConfigurationAPIResponse.model_validate( - raw_configuration | {'component_id': DATA_APP_COMPONENT_ID} - ) - data_app_id = cast(str, api_config.configuration['parameters']['id']) - data_app_science = await client.data_science_client.get_data_app(data_app_id) - if data_app_science.component_id != DATA_APP_COMPONENT_ID: - raise ValueError( - f'Data app tools only support {DATA_APP_COMPONENT_ID} component, but the data app ' - f'"{data_app_id}" has component_id "{data_app_science.component_id}".' - ) - return await _build_data_app_with_repo(client, data_app_science, api_config) - else: - raise ValueError('Either data_app_id or configuration_id must be provided.') - - -async def _build_data_app_with_repo( - client: KeboolaClient, - data_app_science: DataAppResponse, - api_config: ConfigurationAPIResponse, -) -> DataApp: - """Build a `DataApp` and, for python-js apps, attach the managed git repo URL.""" - data_app = DataApp.from_api_responses(data_app_science, api_config) - if data_app_science.type == 'python-js': - try: - repo_resp = await client.data_science_client.get_app_git_repo(data_app_science.id) - data_app.repo_url = repo_resp.https_url - except Exception as exc: - LOG.warning(f'Could not fetch git repo URL for python-js app {data_app_science.id}: {exc}') - return data_app - - -async def _fetch_data_app_details_task( - client: KeboolaClient, links_manager: ProjectLinksManager, configuration_id: str -) -> DataApp | str: - """Task fetching data app details with logs and links by configuration ID. - :param client: The Keboola client - :param configuration_id: The ID of the data app configuration - :return: The data app details or the configuration ID if the data app is not found - """ - try: - data_app = await _fetch_data_app(client, configuration_id=configuration_id, data_app_id=None) - links = links_manager.get_data_app_links( - configuration_id=data_app.configuration_id, - configuration_name=data_app.name, - deployment_link=data_app.deployment_url, - uses_basic_authentication=_uses_basic_authentication(data_app.configuration.get('authorization') or {}), - ) - logs = await _fetch_logs(client, data_app.data_app_id) - last_run = await _fetch_latest_run(client, data_app.data_app_id) - data_app = data_app.with_links(links).with_deployment_info(logs, last_run=last_run) - # Drafts of a python-js prod are surfaced inline so the agent can find them in one round-trip - # — see Scenario C in `modify_python_js_data_app`. Skip for drafts themselves and for Streamlit - # (neither has children). - if data_app.type == 'python-js' and not _is_draft_config(data_app.configuration): - data_app.drafts, data_app.drafts_unavailable = await _fetch_prod_drafts( - client, prod_configuration_id=data_app.configuration_id - ) - return data_app - except Exception: - LOG.exception(f'Failed to fetch data app by configuration ID: {configuration_id}') - return configuration_id - - -def _is_draft_config(configuration: Mapping[str, Any]) -> bool: - """True iff the data app's stored configuration carries `parameters.dataApp.isDraft = true`. - - Shape-safe: a malformed/corrupted config whose `parameters` or `dataApp` is not a mapping is - simply "not a draft" rather than an `AttributeError` (this helper runs in the detail-fetch path). - """ - parameters = configuration.get('parameters') - if not isinstance(parameters, Mapping): - return False - data_app = parameters.get('dataApp') - if not isinstance(data_app, Mapping): - return False - return data_app.get('isDraft') is True - - -async def _fetch_prod_drafts(client: KeboolaClient, *, prod_configuration_id: str) -> tuple[list[DataAppSummary], int]: - """List the drafts (configs with `parentConfigurationId == prod_configuration_id`) of a python-js - prod app. Returns full `DataAppSummary` entries (one extra DSAPI fetch per draft, capped at - 10 parallel) plus the count of drafts whose detail fetch transiently failed and were omitted — - so the caller can tell "temporarily unreachable" from "deleted". Drafts in trash are not returned - by `configuration_list` and so do not appear here. - """ - configs = await client.storage_client.configuration_list(DATA_APP_COMPONENT_ID) - draft_cfg_ids: list[str] = [] - for cfg in configs: - cfg_body = cast(Mapping[str, Any], cfg.get('configuration') or {}) - # A draft must satisfy BOTH halves of the contract: `isDraft=true` AND `parentConfigurationId` - # pointing at this prod. Checking only the parent pointer would surface a misconfigured - # non-draft (e.g. a clone that kept the pointer but lost the flag) as a draft. - if not _is_draft_config(cfg_body): - continue - data_app_block = cast(Mapping[str, Any], cfg_body.get('parameters') or {}).get('dataApp') or {} - if data_app_block.get('parentConfigurationId') == prod_configuration_id: - cfg_id = cfg.get('id') - if isinstance(cfg_id, str): - draft_cfg_ids.append(cfg_id) - - if not draft_cfg_ids: - return [], 0 - - async def fetch_summary(cfg_id: str) -> DataAppSummary | None: - try: - draft = await _fetch_data_app(client, configuration_id=cfg_id, data_app_id=None) - except Exception: - LOG.exception(f'Failed to fetch draft data app by configuration ID: {cfg_id}') - return None - summary = DataAppSummary.model_validate(draft.model_dump()) - summary.repo_url = draft.repo_url - return summary - - results = await process_concurrently(draft_cfg_ids, fetch_summary, max_concurrency=10) - drafts = [s for s in results if isinstance(s, DataAppSummary)] - return drafts, len(draft_cfg_ids) - len(drafts) - - -async def _fetch_logs(client: KeboolaClient, data_app_id: str) -> list[str]: - """Fetches the logs of a data app if it is running otherwise returns empty list.""" - try: - str_logs = await client.data_science_client.tail_app_logs(data_app_id, since=None, lines=20) - logs = str_logs.split('\n') - return logs - except httpx.HTTPStatusError: - # The data app is not running, return empty list - return [] - - -async def _fetch_latest_run(client: KeboolaClient, data_app_id: str) -> Optional[AppRunInfo]: - """Fetches the most recent run (deployment attempt) of a data app, or None when there is none. - - Diagnostics must not break the detail fetch: any error (e.g. an older DSAPI without the - runs endpoint) is logged and reported as "no run info" rather than raised. - """ - try: - runs = await client.data_science_client.list_app_runs(data_app_id, limit=1) - if not runs: - return None - return AppRunInfo.from_api_response(runs[0]) - except Exception: - LOG.exception(f'Failed to fetch app runs for data app: {data_app_id}') - return None - - -def _get_authorization(auth_with_password: bool) -> dict[str, Any]: - if auth_with_password: - return { - 'app_proxy': { - 'auth_providers': [{'id': 'simpleAuth', 'type': 'password'}], - 'auth_rules': [{'type': 'pathPrefix', 'value': '/', 'auth_required': True, 'auth': ['simpleAuth']}], - }, - } - else: - return { - 'app_proxy': { - 'auth_providers': [], - 'auth_rules': [{'type': 'pathPrefix', 'value': '/', 'auth_required': False}], - } - } - - -# Maximum length for DNS labels per RFC 1035 -MAX_DNS_LABEL_LENGTH = 63 - - -class DataAppSlugTooLongError(ValueError): - """Raised when the generated data app slug exceeds the DNS label length limit.""" - - pass - - -def _get_data_app_slug(name: str) -> str: - """Generate a URL-safe slug from the data app name. - - The slug is used as part of the data app URL prefix, which is a DNS label. - DNS labels have a maximum length of 63 characters per RFC 1035. - - :param name: The name of the data app - :return: A URL-safe slug - :raises DataAppSlugTooLongError: If the generated slug exceeds 63 characters - """ - slug = re.sub(r'[^a-z0-9\-]', '', name.strip().lower().replace(' ', '-')) - if len(slug) > MAX_DNS_LABEL_LENGTH: - raise DataAppSlugTooLongError( - f'Data app name "{name}" generates a URL slug that is {len(slug)} characters long, ' - f'which exceeds the maximum DNS label length of {MAX_DNS_LABEL_LENGTH} characters. ' - f'Please use a shorter name (the slug "{slug[:20]}..." is too long). ' - f'The name should generate a slug of at most {MAX_DNS_LABEL_LENGTH} characters after ' - f'converting to lowercase, replacing spaces with hyphens, and removing special characters.' - ) - return slug - - -def _uses_basic_authentication(authorization: dict[str, Any]) -> bool: - try: - return any( - auth_rule['auth_required'] and 'simpleAuth' in auth_rule.get('auth', []) - for auth_rule in authorization['app_proxy']['auth_rules'] - ) - except Exception: - return False - - -def _get_query_function_code(sql_dialect: str) -> str: - """ - Selects the appropriate query function code for the given SQL dialect. - - Snowflake: uses Query Service API - - BigQuery: uses Storage API (Query Service API is not supported for BigQuery yet) - """ - sql_dialect = sql_dialect.lower() - if sql_dialect == 'snowflake': - return _QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE - elif sql_dialect == 'bigquery': - return _STORAGE_QUERY_DATA_FUNCTION_CODE - else: - raise ValueError(f'Unsupported SQL dialect: {sql_dialect}') - - -def _strip_injected_query_code(source_code: str) -> str: - """ - Removes injected query_data function code to keep the generated source consistent when reinjecting the code. - - :param source_code: The source code of the data app - :return: The source code with the injected query_data function code removed - """ - for snippet in (_QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE, _STORAGE_QUERY_DATA_FUNCTION_CODE): - source_code = source_code.replace(snippet, '') - return source_code - - -def _inject_query_to_source_code(source_code: str, sql_dialect: str) -> str: - """ - Injects the query_data function into the source code based on the SQL dialect, while removing the - existing injected code for consistency. - - :param source_code: The source code of the data app - :param sql_dialect: The SQL dialect of the workspace - :return: The source code with the query_data function injected - """ - if not source_code: - return '' - - query_function_code = _get_query_function_code(sql_dialect) - if query_function_code in source_code: - return source_code - - # remove existing injected code to keep the code in sync with the current SQL dialect - source_code = _strip_injected_query_code(source_code) - - if '{QUERY_DATA_FUNCTION}' in source_code: - return source_code.replace('{QUERY_DATA_FUNCTION}', query_function_code) - - match = INJECTED_BLOCK_RE.match(source_code) - if match: - before = match.group('before').rstrip() - after = match.group('after').lstrip() - return f'{before}\n\n{query_function_code}\n\n{after}' - else: - return f'{query_function_code}\n\n{source_code.lstrip()}' - - -def _get_secrets(workspace_id: str, branch_id: str) -> dict[str, Any]: - """ - Generates secrets for the data app for querying the tables in the given workspace QS or SAPI. - """ - secrets: dict[str, Any] = { - SECRET_WORKSPACE_ID: workspace_id, - SECRET_BRANCH_ID: branch_id, - } - return secrets diff --git a/src/keboola_mcp_server/tools/doc.py b/src/keboola_mcp_server/tools/doc.py deleted file mode 100644 index 8e91ec8d7..000000000 --- a/src/keboola_mcp_server/tools/doc.py +++ /dev/null @@ -1,49 +0,0 @@ -import logging -from typing import Annotated - -from fastmcp import Context, FastMCP -from fastmcp.tools import FunctionTool -from mcp.types import ToolAnnotations -from pydantic import BaseModel, Field - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.errors import tool_errors - -LOG = logging.getLogger(__name__) - -DOC_TOOLS_TAG = 'docs' - - -def add_doc_tools(mcp: FastMCP) -> None: - """Add tools to the MCP server.""" - LOG.info(f'Adding tool {docs_query.__name__} to the MCP server.') - mcp.add_tool( - FunctionTool.from_function( - docs_query, - annotations=ToolAnnotations(readOnlyHint=True), - tags={DOC_TOOLS_TAG}, - ) - ) - - LOG.info('Doc tools initialized.') - - -class DocsAnswer(BaseModel): - """An answer to a documentation query.""" - - text: str = Field(description='Text of the answer to a documentation query.') - source_urls: list[str] = Field(description='List of URLs to the sources of the answer.') - - -@tool_errors() -async def docs_query( - ctx: Context, - query: Annotated[str, Field(description='Natural language query to search for in the documentation.')], -) -> DocsAnswer: - """ - Answers a question using the Keboola documentation as a source. - """ - client = KeboolaClient.from_state(ctx.session.state) - answer = await client.ai_service_client.docs_question(query) - - return DocsAnswer(text=answer.text, source_urls=answer.source_urls) diff --git a/src/keboola_mcp_server/tools/flow/__init__.py b/src/keboola_mcp_server/tools/flow/__init__.py deleted file mode 100644 index 196626951..000000000 --- a/src/keboola_mcp_server/tools/flow/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -from keboola_mcp_server.tools.flow.model import ( - Flow, - FlowConfiguration, - FlowPhase, - FlowTask, - FlowToolOutput, - GetFlowsDetailOutput, - GetFlowsListOutput, - GetFlowsOutput, -) -from keboola_mcp_server.tools.flow.tools import ( - add_flow_tools, - create_flow, - get_flow_schema, - get_flows, - modify_flow, - update_flow, -) diff --git a/src/keboola_mcp_server/tools/flow/model.py b/src/keboola_mcp_server/tools/flow/model.py deleted file mode 100644 index 38aff8777..000000000 --- a/src/keboola_mcp_server/tools/flow/model.py +++ /dev/null @@ -1,520 +0,0 @@ -""" -Flow models for Keboola MCP server. -""" - -import logging -from datetime import datetime -from typing import Annotated, Any, Literal, Optional, Union - -from pydantic import AliasChoices, BaseModel, ConfigDict, Field, ValidationError - -from keboola_mcp_server.clients.client import ORCHESTRATOR_COMPONENT_ID, FlowType, get_metadata_property -from keboola_mcp_server.clients.storage import APIFlowResponse -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.flow.scheduler_model import SchedulesOutput - -LOG = logging.getLogger(__name__) - -# ============================================================================= -# RESPONSE MODELS -# ============================================================================= - - -class GetFlowsListOutput(BaseModel, frozen=True): - """Output of get_flows tool when listing all flows (no flow_ids specified).""" - - flows: list['FlowSummary'] = Field(description='The retrieved flow configurations.') - links: list[Link] = Field(description='The list of links relevant to the flows.') - - -class GetFlowsDetailOutput(BaseModel, frozen=True): - """Output of get_flows tool when retrieving specific flow_ids.""" - - flows: list['Flow'] = Field(description='The retrieved flow configurations with full details.') - - -GetFlowsOutput = Union[GetFlowsListOutput, GetFlowsDetailOutput] - - -class FlowToolOutput(BaseModel): - """ - Standard response model for flow tool operations. - - :param configuration_id: The configuration ID of the flow. - :param component_id: The component ID of the flow. - :param description: The description of the Flow. - :param timestamp: The timestamp of the operation. - :param success: Indicates if the operation succeeded. - :param links: The links relevant to the flow. - :param version: The version number of the flow configuration. - :param response: The response messages from the operation. - """ - - configuration_id: str = Field(description='The configuration ID of the flow.') - component_id: str = Field(description='The ID of the component.') - description: str = Field(description='The description of the Flow.') - version: int = Field(description='The version number of the flow configuration.') - timestamp: datetime = Field(description='The timestamp of the operation.') - response: Optional[str] = Field(default=None, description='The response message from the operation.') - change_summary: Optional[str] = Field(default=None, description='Additional notes or hints about the operation.') - success: bool = Field(default=True, description='Indicates if the operation succeeded.') - links: list[Link] = Field(description='The links relevant to the flow.') - - -# ============================================================================= -# LEGACY ORCHESTRATOR FLOW MODELS -# ============================================================================= - - -class FlowPhase(BaseModel): - """Represents a phase in a legacy flow configuration.""" - - id: int | str = Field(description='Unique identifier of the phase') - name: str = Field(description='Name of the phase', min_length=1) - description: str = Field(default_factory=str, description='Description of the phase') - depends_on: list[int | str] = Field( - default_factory=list, - description='List of phase IDs this phase depends on', - validation_alias=AliasChoices('depends_on', 'dependsOn', 'depends-on'), - serialization_alias='dependsOn', - ) - - -class FlowTask(BaseModel): - """Represents a task in a legacy flow configuration.""" - - id: int | str = Field(description='Unique identifier of the task') - name: str = Field(description='Name of the task') - phase: int | str = Field(description='ID of the phase this task belongs to') - enabled: Optional[bool] = Field(default=True, description='Whether the task is enabled') - continue_on_failure: Optional[bool] = Field( - default=False, - description='Whether to continue if task fails', - validation_alias=AliasChoices('continue_on_failure', 'continueOnFailure', 'continue-on-failure'), - serialization_alias='continueOnFailure', - ) - task: dict[str, Any] = Field(description='Task configuration containing componentId, configId, etc.') - - -class FlowConfiguration(BaseModel): - """Represents a complete legacy flow configuration.""" - - phases: list[FlowPhase] = Field(description='List of phases in the flow') - tasks: list[FlowTask] = Field(description='List of tasks in the flow') - - -# ============================================================================= -# CONDITIONAL FLOW MODELS - RETRY CONFIGURATION -# NOTE: These will be removed in future iterations once we fetch the shcema from AI service -# ============================================================================= - - -class BaseExtraModel(BaseModel): - """Base for conditional-flow configuration nodes (conditions, tasks, phases, transitions, retry). - - Uses ``extra='allow'`` so fields the live ``keboola.flow`` schema may add — and that we do not - model yet — pass through the model round-trip (``get_flow_configuration`` on the write path, - ``Flow.from_api_response`` on the read path) instead of being silently dropped. Validation - against the live schema remains the authoritative gate; this only prevents lossy serialization. - """ - - model_config = ConfigDict(extra='allow') - - -class RetryStrategyParams(BaseExtraModel): - """Retry strategy parameters configuration.""" - - max_retries: int = Field(default=3, description='Maximum number of retry attempts', alias='maxRetries') - delay: int = Field(default=10, description='Delay in seconds between retry attempts') - - -class RetryOnCondition(BaseExtraModel): - """Retry condition configuration.""" - - type: Literal['errorMessageContains', 'errorMessageExact'] = Field(description='Type of retry condition') - value: str = Field(description='Value to match for retry condition') - - -class RetryConfiguration(BaseExtraModel): - """Retry configuration for tasks and phases.""" - - strategy: Literal['linear'] = Field(default='linear', description='Retry strategy') - strategy_params: RetryStrategyParams = Field( - default_factory=RetryStrategyParams, description='Strategy parameters', alias='strategyParams' - ) - retry_on: Optional[list[RetryOnCondition]] = Field( - default=None, description='Conditions that trigger retry', alias='retryOn' - ) - - -# ============================================================================= -# CONDITIONAL FLOW MODELS - CONDITIONS -# ============================================================================= - - -class TaskCondition(BaseExtraModel): - """Task-based condition for flow transitions.""" - - type: Literal['task'] = Field(description='Condition type') - task: str = Field(description='ID of the task to evaluate, or "*" when used with phase operators') - value: str = Field( - description=( - 'Property path or JMESPath expression to retrieve from the task context. Common simple ' - "paths: 'taskId', 'phaseId', 'status', 'job.id', 'job.componentId', 'job.configId', " - "'job.status', 'job.result', 'job.startTime', 'job.endTime', 'job.duration', " - "'job.result.output.tables', 'job.result.message'. JMESPath expressions over the job " - 'result are also supported, for example: ' - "sum(job.result.output.tables[].metrics[?name=='importedRowsCount'][].value)" - ) - ) - - -class PhaseCondition(BaseExtraModel): - """Phase-based condition for flow transitions.""" - - type: Literal['phase'] = Field(description='Condition type') - phase: str = Field(description='ID of the phase to evaluate') - value: Literal['phaseId', 'status'] = Field(description='Property to retrieve from the phase') - - -class ConstantCondition(BaseExtraModel): - """Constant value condition.""" - - type: Literal['const', 'constant'] = Field(description='Condition type') - value: Union[str, int, bool, list] = Field(description='Constant value') - - -class VariableCondition(BaseExtraModel): - """Variable-based condition.""" - - type: Literal['variable'] = Field(description='Condition type') - value: str = Field(description='The name of the variable to evaluate') - - -class OperatorCondition(BaseExtraModel): - """Operator-based condition with operands.""" - - type: Literal['operator'] = Field(description='Condition type') - operator: Literal['AND', 'OR', 'EQUALS', 'NOT_EQUALS', 'GREATER_THAN', 'LESS_THAN', 'INCLUDES', 'CONTAINS'] = Field( - description='Operator type' - ) - operands: list['ConditionObject'] = Field(description='List of operand conditions') - - -class PhaseOperatorCondition(BaseExtraModel): - """Phase-specific operator condition.""" - - type: Literal['operator'] = Field(description='Condition type') - operator: Literal['ALL_TASKS_IN_PHASE', 'ANY_TASKS_IN_PHASE'] = Field(description='Phase operator type') - phase: str = Field(description='ID of the phase to check') - operands: list['OperatorCondition'] = Field(description='List of operand conditions') - - -class FunctionCondition(BaseExtraModel): - """Function-based condition.""" - - type: Literal['function'] = Field(description='Condition type') - function: str = Field( - description=( - "Function name. Supported values: 'COUNT' (returns the number of elements in its single " - "array operand) and 'DATE' (returns the current date/time formatted via PHP " - "DateTime::format; takes one operand that resolves to a format string such as 'Y', 'm', " - "'d', 'H', 'i', 's', or 'U'). Typed as str rather than Literal so unknown server-side " - 'values do not break flow listing; only the two values above are accepted at execution time.' - ) - ) - operands: list['VariableSourceObject'] = Field(description='List of operand conditions') - - -class ArrayCondition(BaseExtraModel): - """Array-based condition.""" - - type: Literal['array'] = Field(description='Condition type') - operands: list['VariableSourceObject'] = Field(description='List of operand conditions') - - -# Union type for all condition types -ConditionObject = Union[ - TaskCondition, - PhaseCondition, - ConstantCondition, - VariableCondition, - OperatorCondition, - PhaseOperatorCondition, - FunctionCondition, - ArrayCondition, -] - - -# ============================================================================= -# CONDITIONAL FLOW MODELS - TASK CONFIGURATIONS -# ============================================================================= - - -class JobTaskConfiguration(BaseExtraModel): - """Job task configuration.""" - - type: Literal['job'] = Field(description='Task type') - component_id: str = Field(description='Component ID', alias='componentId') - config_id: Optional[str] = Field(default=None, description='Configuration ID', alias='configId') - config_data: Optional[dict[str, Any]] = Field(default=None, description='Configuration data', alias='configData') - mode: Literal['run'] = Field(description='Execution mode') - delay: Optional[Union[str, int]] = Field(default=None, description='Initial delay in seconds') - retry: Optional[RetryConfiguration] = Field(default=None, description='Retry configuration') - variable_overrides: Optional[list[str]] = Field( - default=None, - description='Names of flow variables to pass into this job as variable overrides', - alias='variableOverrides', - ) - - -class NotificationRecipient(BaseExtraModel): - """Notification recipient configuration.""" - - channel: Literal['email', 'webhook'] = Field(description='Channel type') - address: str = Field(description='Recipient address (email or webhook URL)') - - -class NotificationTaskConfiguration(BaseExtraModel): - """Notification task configuration.""" - - type: Literal['notification'] = Field(description='Task type') - recipients: list[NotificationRecipient] = Field(description='List of notification recipients', min_length=1) - title: str = Field(description='Notification title') - message: Optional[str] = Field(default=None, description='Notification message') - - -# Variable source object (limited subset of conditions). Each member has a unique `type` -# literal, so we discriminate on it: pydantic dispatches directly to the matching model and -# reports one targeted error instead of trying every member and producing a cascade. -VariableSourceObject = Annotated[ - Union[ConstantCondition, PhaseCondition, TaskCondition, VariableCondition, FunctionCondition, ArrayCondition], - Field(discriminator='type'), -] - - -class VariableTaskConfiguration(BaseExtraModel): - """Variable task configuration.""" - - type: Literal['variable'] = Field(description='Task type') - name: str = Field(description='Variable name') - value: Optional[str] = Field(default=None, description='Variable value') - source: Optional[VariableSourceObject] = Field(default=None, description='Variable source') - - -TaskConfiguration = Annotated[ - Union[JobTaskConfiguration, NotificationTaskConfiguration, VariableTaskConfiguration], - Field(discriminator='type'), -] - - -# ============================================================================= -# CONDITIONAL FLOW MODELS - CORE STRUCTURES -# ============================================================================= - - -class ConditionalFlowTransition(BaseExtraModel): - """Transition model with structured conditions.""" - - id: str = Field(description='Unique identifier of the transition') - name: Optional[str] = Field(default=None, description='Optional descriptive name for the transition') - condition: Optional[ConditionObject] = Field(default=None, description='Structured condition for this transition') - goto: str | None = Field(description='Target phase ID to transition to, or null to end the flow') - - -class ConditionalFlowTask(BaseExtraModel): - """Task model with structured configuration.""" - - id: str = Field(description='Unique identifier of the task (must be string)') - name: str = Field(description='Name of the task') - phase: str = Field(description='ID of the phase this task belongs to (must be string)') - enabled: Optional[bool] = Field(default=True, description='Whether the task is enabled') - task: TaskConfiguration = Field(description='Structured task configuration') - - -class ConditionalFlowPhase(BaseExtraModel): - """Phase model with structured retry configuration.""" - - id: str = Field(description='Unique identifier of the phase (must be string)') - name: str = Field(description='Name of the phase', min_length=1) - description: Optional[str] = Field(default=None, description='Description of the phase') - retry: Optional[RetryConfiguration] = Field( - default=None, description='Retry configuration for all tasks in this phase' - ) - next: Optional[list[ConditionalFlowTransition]] = Field( - default_factory=list, description='Array of transitions to other phases' - ) - - def model_dump(self, *, exclude_unset: bool = False, **kwargs): - # When exclude_unset=True, also exclude "next" if: - # 1. next is an empty list, OR - # 2. next contains only one transition with goto=null - # This allows us to modify the ending phases without specified transitions in the UI of conditional flows in - # Keboola Designer, and prevents UI damage from single null transitions. - data = super().model_dump(exclude_unset=exclude_unset, **kwargs) - if exclude_unset: - if 'next' in data and isinstance(data['next'], list): - if len(data['next']) == 0: - data.pop('next') - elif len(data['next']) == 1 and data['next'][0].get('goto') is None: - data.pop('next') - return data - - -class ConditionalFlowConfiguration(BaseExtraModel): - """Represents a complete legacy flow configuration.""" - - phases: list[ConditionalFlowPhase] = Field(description='List of phases in the flow') - tasks: list[ConditionalFlowTask] = Field(description='List of tasks in the flow') - - -_T = Union[ConditionalFlowPhase, ConditionalFlowTask] - - -def _safe_validate(model_cls: type[_T], raw: dict[str, Any], flow_id: str, kind: str) -> _T: - """Validate a flow element; on failure log and fall back to ``model_construct``. - - The READ path (``get_flows``) must keep returning data even when the backend ships a task - or phase shape the schema hasn't caught up to — silently dropping items would hide flows - from the agent. ``model_construct`` skips validation, so unknown variants pass through as - raw dicts on the typed field; this is safe for display/serialization but is intentionally - NOT used on the WRITE paths (``utils.get_flow_configuration``), which must remain strict. - """ - try: - return model_cls.model_validate(raw) - except ValidationError as exc: - LOG.warning( - 'Flow %s: %s %r failed strict validation, falling back to permissive parse: %s', - flow_id, - kind, - raw.get('id'), - exc, - ) - return model_cls.model_construct(**raw) - - -# ============================================================================= -# DOMAIN MODELS -# ============================================================================= - - -class Flow(BaseModel): - """Complete flow configuration with all data.""" - - component_id: FlowType = Field(description='The ID of the component (keboola.orchestrator/keboola.flow)') - configuration_id: str = Field(description='The ID of this flow configuration') - name: str = Field(description='The name of the flow configuration') - description: Optional[str] = Field(default=None, description='The description of the flow configuration') - version: int = Field(description='The version of the flow configuration') - is_disabled: bool = Field(default=False, description='Whether the flow configuration is disabled') - is_deleted: bool = Field(default=False, description='Whether the flow configuration is deleted') - configuration: FlowConfiguration | ConditionalFlowConfiguration = Field( - description='The flow configuration containing phases and tasks' - ) - change_description: Optional[str] = Field(default=None, description='The description of the latest changes') - configuration_metadata: list[dict[str, Any]] = Field( - default_factory=list, description='Flow configuration metadata including MCP tracking' - ) - folder: str = Field(default='', description='The UI folder this flow is organized into') - created: Optional[str] = Field(None, description='Creation timestamp') - updated: Optional[str] = Field(None, description='Last update timestamp') - schedules: Optional[SchedulesOutput] = Field(default=None, description='List of schedules for this flow') - links: list[Link] = Field(default_factory=list, description='MCP-specific links for UI navigation') - - @classmethod - def from_api_response( - cls, - api_config: APIFlowResponse, - flow_component_id: FlowType, - links: Optional[list[Link]] = None, - schedules: Optional[SchedulesOutput] = None, - ) -> 'Flow': - """ - Create a Flow domain model from an APIFlowResponse. - - :param api_config: The APIFlowResponse instance. - :param flow_component_id: The component ID of the flow. - :param links: Optional list of navigation links. - :return: Flow domain model. - """ - is_legacy = flow_component_id == ORCHESTRATOR_COMPONENT_ID - - if is_legacy: - phases = [FlowPhase.model_validate(p) for p in api_config.configuration.get('phases', [])] - tasks = [FlowTask.model_validate(t) for t in api_config.configuration.get('tasks', [])] - config = FlowConfiguration(phases=phases, tasks=tasks) - else: - phases = [ - _safe_validate(ConditionalFlowPhase, p, api_config.configuration_id, 'phase') - for p in api_config.configuration.get('phases', []) - ] - tasks = [ - _safe_validate(ConditionalFlowTask, p, api_config.configuration_id, 'task') - for p in api_config.configuration.get('tasks', []) - ] - config = ConditionalFlowConfiguration(phases=phases, tasks=tasks) - - return cls.model_construct( - component_id=flow_component_id, - configuration_id=api_config.configuration_id, - name=api_config.name, - description=api_config.description, - version=api_config.version, - is_disabled=api_config.is_disabled, - is_deleted=api_config.is_deleted, - configuration=config, - change_description=api_config.change_description, - configuration_metadata=api_config.metadata, - folder=get_metadata_property(api_config.metadata, MetadataField.CONFIGURATION_FOLDER_NAME) or '', - created=api_config.created, - updated=api_config.updated, - links=links or [], - schedules=schedules, - ) - - -class FlowSummary(BaseModel): - """Lightweight flow configuration for list operations.""" - - component_id: FlowType = Field(description='The ID of the component (keboola.orchestrator/keboola.flow)') - configuration_id: str = Field(description='The ID of this flow configuration') - name: str = Field(description='The name of the flow configuration') - description: Optional[str] = Field(default=None, description='The description of the flow configuration') - version: int = Field(description='The version of the flow configuration') - is_disabled: bool = Field(default=False, description='Whether the flow configuration is disabled') - is_deleted: bool = Field(default=False, description='Whether the flow configuration is deleted') - phases_count: int = Field(description='Number of phases in the flow') - tasks_count: int = Field(description='Number of tasks in the flow') - schedules_count: int = Field(default=0, description='Number of configured schedules for this flow') - folder: str = Field(default='', description='The UI folder this flow is organized into') - created: Optional[str] = Field(None, description='Creation timestamp') - updated: Optional[str] = Field(None, description='Last update timestamp') - - @classmethod - def from_api_response( - cls, api_config: APIFlowResponse, flow_component_id: FlowType, n_schedules: int = 0 - ) -> 'FlowSummary': - """ - Create a FlowSummary domain model from an APIFlowResponse. - - :param api_config: The APIFlowResponse instance. - :param flow_component_id: The component ID of the flow. - :return: FlowSummary domain model. - """ - config = getattr(api_config, 'configuration', {}) or {} - return cls.model_construct( - component_id=flow_component_id, - configuration_id=api_config.configuration_id, - name=api_config.name, - description=api_config.description, - version=api_config.version, - is_disabled=api_config.is_disabled, - is_deleted=api_config.is_deleted, - phases_count=len(config.get('phases', [])), - tasks_count=len(config.get('tasks', [])), - schedules_count=n_schedules, - folder=get_metadata_property(api_config.metadata, MetadataField.CONFIGURATION_FOLDER_NAME) or '', - created=api_config.created, - updated=api_config.updated, - ) diff --git a/src/keboola_mcp_server/tools/flow/scheduler.py b/src/keboola_mcp_server/tools/flow/scheduler.py deleted file mode 100644 index f1b39f147..000000000 --- a/src/keboola_mcp_server/tools/flow/scheduler.py +++ /dev/null @@ -1,525 +0,0 @@ -"""Scheduler management functions for creating, updating, and deleting schedulers.""" - -import logging -from typing import Any, Sequence - -from pydantic import AliasChoices, BaseModel, Field, field_validator - -from keboola_mcp_server.clients.client import FlowType, KeboolaClient -from keboola_mcp_server.clients.storage import CreateConfigurationAPIResponse -from keboola_mcp_server.links import ProjectLinksManager -from keboola_mcp_server.tools.components.utils import set_cfg_creation_metadata, set_cfg_update_metadata -from keboola_mcp_server.tools.flow.model import Flow, FlowSummary -from keboola_mcp_server.tools.flow.scheduler_model import ( - ScheduleDetail, - ScheduleRequest, - SchedulesOutput, -) - -LOG = logging.getLogger(__name__) - -SCHEDULER_COMPONENT_ID = 'keboola.scheduler' - - -CRON_TAB_INSTRUCTIONS = """ -Cron Tab Expression should be in the format: `* * * * *`. -Field order: -1. Minute (0-59) -2. Hour (0-23) -3. Day of month (1-31, or L for last day of month) -4. Month (1-12) -5. Day of week (0-6, where 0 = Sunday) - -Examples: -1. schedule daily at 1:00 PM and 1:00 AM would be `0 1,13 * * *` -2. schedule weekly on Monday at 9:00 AM would be `0 9 * * 1` -3. schedule monthly on the 1st and 20th day of the month at 10:00 AM would be `0 10 1,20 * *` -4. schedule yearly on the 1st of january and august at 11:00 AM would be `0 11 1 1,8 *` -5. schedule hourly every 15 minutes would be `0,15,30,45 * * * *` -6. schedule monthly on the last day of the month at 10:00 AM would be `0 10 L * *` -""" - - -def validate_cron_tab(cron_tab: str | None) -> None: - """Validate the cron tab expression.""" - try: - if cron_tab is None: - return None - split_cron_tab = cron_tab.strip().split() - if len(split_cron_tab) != 5: - raise ValueError( - f'Cron expression must have exactly 5 parts got: {cron_tab} which has {len(split_cron_tab)} parts.' - ) - - def to_int_list(field: str, allow_l: bool = False) -> tuple[list[int], bool]: - """Parse a cron field into a list of integers and a flag indicating if L was found.""" - if field == '*': - return [], False - has_l = False - parts = [] - for x in field.split(','): - x = x.strip() - if allow_l and x.upper() == 'L': - has_l = True - else: - try: - parts.append(int(x)) - except ValueError: - raise ValueError(f'Cron expression must have only digits got: {field} in "{cron_tab}".') - if allow_l and has_l and parts: - raise ValueError('Day of month must use either `L` or numeric values, not both.') - return parts, has_l - - minutes, _ = to_int_list(split_cron_tab[0].strip()) - hours, _ = to_int_list(split_cron_tab[1].strip()) - days, has_last_day = to_int_list(split_cron_tab[2].strip(), allow_l=True) - months, _ = to_int_list(split_cron_tab[3].strip()) - weekdays, _ = to_int_list(split_cron_tab[4].strip()) - - if any(x < 0 or x > 59 for x in minutes): - raise ValueError(f'Minutes of hour `M _ _ _ _` must be between 0 and 59, got: {split_cron_tab[0]}') - if any(x < 0 or x > 23 for x in hours): - raise ValueError(f'Hours of day `_ H _ _ _` must be between 0 and 23, got: {split_cron_tab[1]}') - if any(x < 1 or x > 31 for x in days): - raise ValueError(f'Days of month `_ _ D _ _`must be between 1 and 31, got: {split_cron_tab[2]}') - if any(x < 1 or x > 12 for x in months): - raise ValueError(f'Months of year `_ _ _ M _` must be between 1 and 12, got: {split_cron_tab[3]}') - if any(x < 0 or x > 6 for x in weekdays): - raise ValueError( - f'Days of week `_ _ _ _ W` must be between 0=Sunday and 6=Saturday, got: {split_cron_tab[4]}' - ) - if months and not days and not has_last_day: - raise ValueError('Months of year must be specified with days of month. Example: `35 12 31 1,3 *`') - if (days or has_last_day) and not hours: - raise ValueError('Days of month must be specified with hours of day. Example: `55 12 31 * *`') - if hours and not minutes: - raise ValueError('Hours of day must be specified with minutes of hour. Example: `55 12 * * *`') - if weekdays and not hours: - raise ValueError('Days of week must be specified with hours of day. Example: `55 12 * * 0`') - if weekdays and (days or months or has_last_day): - raise ValueError('Days of week must not be specified with days of month nor months of year.') - except ValueError as e: - raise ValueError(f'Invalid cron tab expression: {str(e)}.\n{CRON_TAB_INSTRUCTIONS}') from e - - -class SimplifiedSchedule(BaseModel): - """Simplified schedule dictionary.""" - - schedule_id: str | None = Field( - description='The schedule ID', - validation_alias=AliasChoices('scheduleId', 'schedule_id'), - serialization_alias='scheduleId', - default=None, - ) - cron_tab: str = Field( - description='The cron tab', - validation_alias=AliasChoices('cronTab', 'cron_tab'), - serialization_alias='cronTab', - ) - timezone: str = Field( - description='The timezone', - default='UTC', - ) - state: str = Field( - description='The state', - default='enabled', - ) - - @field_validator('cron_tab') - @classmethod - def _validate_cron_tab(cls, value: str) -> str: - validate_cron_tab(value) - return value - - def update_from_request(self, request: ScheduleRequest) -> 'SimplifiedSchedule': - """Return a new schedule with the updated fields from the request.""" - if self.schedule_id != request.schedule_id: - raise ValueError(f'Cannot update schedule with different ID: {self.schedule_id} != {request.schedule_id}') - return SimplifiedSchedule( - schedule_id=self.schedule_id, - cron_tab=self.cron_tab if request.cron_tab is None else request.cron_tab, - timezone=self.timezone if request.timezone is None else request.timezone, - state=self.state if request.state is None else request.state, - ) - - -async def _update_schedulers_internal( - *, - client: KeboolaClient, - configuration_id: str, - component_id: str, - schedules: Sequence[ScheduleRequest] = tuple(), -) -> tuple[dict[str, SimplifiedSchedule], dict[str, SimplifiedSchedule | None], list[SimplifiedSchedule]]: - """ - Compute original, updated and new schedulers for preview by adding/updating/removing schedules. - - :param client: KeboolaClient instance - :param configuration_id: The configuration ID to schedule - :param component_id: The component ID to schedule - :param schedules: The list of schedule requests to compute the preview for - :return: A tuple of the original, updated and new schedulers - """ - - current_schedulers = await list_schedules_for_config( - client=client, component_id=component_id, configuration_id=configuration_id - ) - - original_schedulers: dict[str, SimplifiedSchedule] = { - schedule.schedule_id: SimplifiedSchedule( - schedule_id=schedule.schedule_id, - cron_tab=schedule.cron_tab, - timezone=schedule.timezone, - state=schedule.state, - ) - for schedule in current_schedulers - } - new_schedulers: list[SimplifiedSchedule] = [] - updated_schedulers: dict[str, SimplifiedSchedule | None] = {} - for request in schedules: - if request.action == 'add': - new_schedulers.append( - SimplifiedSchedule.model_validate(request.model_dump(by_alias=True, exclude_none=True)) - ) - elif request.action == 'update': - if request.schedule_id not in original_schedulers: - raise ValueError( - f'Schedule (ID: {request.schedule_id}) cannot be updated because it was not found in the ' - 'existing schedulers.' - ) - updated_schedulers[request.schedule_id] = original_schedulers[request.schedule_id].update_from_request( - request - ) - elif request.action == 'remove': - if request.schedule_id not in original_schedulers: - raise ValueError( - f'Schedule (ID: {request.schedule_id}) cannot be removed because it was not found in the ' - 'existing schedulers.' - ) - updated_schedulers[request.schedule_id] = None - else: - raise ValueError(f'Invalid action for schedulers: {request.action}.') - return original_schedulers, updated_schedulers, new_schedulers - - -async def compute_schedulers_preview( - *, - client: KeboolaClient, - configuration_id: str, - flow_type: FlowType, - schedules: Sequence[ScheduleRequest], -) -> dict[str, list[dict[str, Any]]]: - """ - Compute the preview of the schedulers for a configuration. - - :param client: KeboolaClient instance - :param configuration_id: The configuration ID to schedule - :param flow_type: The type of flow to schedule - :param schedules: The list of schedule requests to compute the preview for - :return: A mutator preview payload with original and updated schedulers - """ - original_schedulers, updated_schedulers, new_schedulers = await _update_schedulers_internal( - client=client, configuration_id=configuration_id, component_id=flow_type, schedules=schedules - ) - - # Sync the updated schedulers with the original schedulers and sort them by schedule_id for diff preview - synced_updated_list = [] - original_list = [] - for prev in sorted(original_schedulers.values(), key=lambda x: x.schedule_id): - original_list.append(prev.model_dump(by_alias=True, exclude_none=False)) - if prev.schedule_id in updated_schedulers: - if updated_schedulers[prev.schedule_id] is None: - # Explicit remove action -> the schedule should not appear in the updated preview list. - continue - # Update schedule -> add the updated schedule. - synced_updated_list.append( - updated_schedulers[prev.schedule_id].model_dump(by_alias=True, exclude_none=False) - ) - else: - # No update -> sync the original schedule as it is. - synced_updated_list.append(prev.model_dump(by_alias=True, exclude_none=False)) - - new_list = [s.model_dump(by_alias=True, exclude_none=False) for s in new_schedulers] - return { - 'original_schedulers': original_list, - 'updated_schedulers': synced_updated_list + new_list, - } - - -async def process_schedule_request( - client: KeboolaClient, - target_component_id: str, - target_configuration_id: str, - requests: Sequence[ScheduleRequest], -) -> list[str]: - """ - Process a schedule request and perform the appropriate action. - - :param client: KeboolaClient instance - :param target_component_id: The component ID to schedule (e.g., 'keboola.flow') - :param target_configuration_id: The configuration ID to schedule - :param request: ScheduleUpdateRequest object - :param flow_name: Optional name of the flow (used for generating schedule names) - :return: ScheduleDetail for the created/modified schedule - """ - - _, updated_schedulers, new_schedulers = await _update_schedulers_internal( - client=client, configuration_id=target_configuration_id, component_id=target_component_id, schedules=requests - ) - responses: list[str] = [] - try: - for schedule_id, schedule in updated_schedulers.items(): - if schedule is None: - # Remove schedule if schedule is None - await remove_schedule(client=client, schedule_config_id=schedule_id) - responses.append(f'Removed schedule: {schedule_id}') - else: - # Update schedule if schedule is not None - await update_schedule( - client=client, - schedule_config_id=schedule_id, - cron_tab=schedule.cron_tab, - timezone=schedule.timezone, - state=schedule.state, - change_description='Schedule Updated', - ) - responses.append(f'Updated schedule: {schedule_id}') - for new_scheduler in new_schedulers: - response = await create_schedule( - client=client, - target_component_id=target_component_id, - target_configuration_id=target_configuration_id, - cron_tab=new_scheduler.cron_tab, - timezone=new_scheduler.timezone, - state=new_scheduler.state, - schedule_name=f'Schedule for {target_configuration_id}', - schedule_description=f'Automated schedule for {target_configuration_id}', - target_mode='run', - ) - responses.append(f'Created schedule: {response.schedule_id}') - except Exception as e: - raise ValueError(f'Error processing schedule requests: {str(e)}') from e - return responses - - -async def create_schedule( - client: KeboolaClient, - target_component_id: str, - target_configuration_id: str, - cron_tab: str, - timezone: str, - state: str, - schedule_name: str | None = None, - schedule_description: str = '', - target_mode: str = 'run', - target_tag: str | None = None, -) -> ScheduleDetail: - """ - Create a scheduler for a component configuration. - - This is a two-step process: - 1. Create a scheduler configuration in Storage API (keboola.scheduler component) - 2. Activate the scheduler in the Scheduler API - - :param client: KeboolaClient instance - :param target_component_id: The component ID to schedule (e.g., 'keboola.flow') - :param target_configuration_id: The configuration ID to schedule - :param schedule: SimplifiedCronSchedule with schedule details - :param schedule_name: Name for the scheduler configuration (defaults to 'Scheduler for {config_id}') - :param schedule_description: Description for the scheduler configuration - :param timezone: Timezone for the scheduler - :param target_mode: Execution mode (default: 'run') - :param target_tag: Optional tag for the target configuration - :return: ScheduleDetail with the activated scheduler details - """ - if schedule_name is None: - schedule_name = f'Schedule for {target_configuration_id}' - - # Step 1: Create scheduler configuration in Storage API - scheduler_config = { - 'schedule': { - 'cronTab': cron_tab, - 'timezone': timezone, - 'state': state, - }, - 'target': { - 'componentId': target_component_id, - 'configurationId': target_configuration_id, - 'mode': target_mode, - }, - } - - if target_tag: - scheduler_config['target']['tag'] = target_tag - - # Storage API expects configuration as a dict, will be converted appropriately - storage_response = CreateConfigurationAPIResponse.model_validate( - await client.storage_client.configuration_create( - component_id=SCHEDULER_COMPONENT_ID, - name=schedule_name, - description=schedule_description, - configuration=scheduler_config, - ) - ) - schedule_config_id = storage_response.id - LOG.info(f'Created schedule configuration in Storage API: {schedule_config_id}') - - # Step 2: Activate scheduler in Scheduler API - schedule_response = await client.scheduler_client.activate_schedule(schedule_config_id) - LOG.info(f'Activated schedule in Scheduler API: {schedule_response.id}') - await set_cfg_creation_metadata( - client, - component_id=SCHEDULER_COMPONENT_ID, - configuration_id=schedule_config_id, - ) - - return ScheduleDetail.from_api_response(schedule_response) - - -async def update_schedule( - client: KeboolaClient, - schedule_config_id: str, - cron_tab: str | None, - timezone: str | None, - state: str | None, - scheduler_name: str | None = None, - scheduler_description: str | None = None, - change_description: str = 'Scheduler updated', -) -> ScheduleDetail: - """ - Update an existing scheduler. - - This is a two-step process: - 1. Update the scheduler configuration in Storage API - 2. Reactivate the scheduler in the Scheduler API (posts the updated config) - - :param client: KeboolaClient instance - :param schedule_config_id: The schedule configuration ID in Storage API - :param cron_tab: Optional cron tab to update schedule details - :param timezone: Optional timezone to update schedule details - :param state: Optional state to update schedule details - :param scheduler_name: Optional new name for the scheduler - :param scheduler_description: Optional new description - :param change_description: Description of the change - :return: ScheduleDetail with updated scheduler details - """ - - # Get current configuration to merge with updates - current_config = CreateConfigurationAPIResponse.model_validate( - await client.storage_client.configuration_detail( - component_id=SCHEDULER_COMPONENT_ID, configuration_id=schedule_config_id - ) - ) - - current_scheduler_config = current_config.configuration - - if cron_tab is not None: - current_scheduler_config['schedule']['cronTab'] = cron_tab - if timezone is not None: - current_scheduler_config['schedule']['timezone'] = timezone - if state is not None: - current_scheduler_config['schedule']['state'] = state - - # Step 1: Update configuration in Storage API - updated_confg = CreateConfigurationAPIResponse.model_validate( - await client.storage_client.configuration_update( - component_id=SCHEDULER_COMPONENT_ID, - configuration_id=schedule_config_id, - configuration=current_scheduler_config, - change_description=change_description, - updated_name=scheduler_name, - updated_description=scheduler_description, - ) - ) - LOG.info(f'Updated schedule configuration in Storage API: {schedule_config_id}') - - # Step 2: Reactivate in Scheduler API to apply changes - scheduler_response = await client.scheduler_client.activate_schedule(schedule_config_id) - LOG.info(f'Reactivated scheduler in Scheduler API: {scheduler_response.id}') - - await set_cfg_update_metadata( - client, - component_id=SCHEDULER_COMPONENT_ID, - configuration_id=schedule_config_id, - configuration_version=updated_confg.version, - ) - - return ScheduleDetail.from_api_response(scheduler_response) - - -async def remove_schedule(client: KeboolaClient, schedule_config_id: str) -> None: - """ - Remove a schedule completely. - - This is a two-step process: - 1. Remove the schedule from Scheduler API - 2. Remove the configuration from Storage API - - :param client: KeboolaClient instance - :param schedule_config_id: The schedule configuration ID in Storage API - """ - LOG.info(f'Deleting schedule: {schedule_config_id}') - - # Step 1: Delete from Scheduler API - await client.scheduler_client.delete_schedule(schedule_config_id) - LOG.info(f'Deleted schedule from Scheduler API: {schedule_config_id}') - - # Step 2: Delete from Storage API - await client.storage_client.configuration_delete( - component_id=SCHEDULER_COMPONENT_ID, configuration_id=schedule_config_id - ) - LOG.info(f'Deleted schedule configuration from Storage API: {schedule_config_id}') - - -async def list_schedules_for_config( - client: KeboolaClient, component_id: str, configuration_id: str -) -> list[ScheduleDetail]: - """ - List all schedules for a configuration. - - :param client: KeboolaClient instance - :param component_id: The component ID - :param configuration_id: The configuration ID - :return: List of Schedules - """ - schedules_api = await client.scheduler_client.list_schedules_by_config_id( - component_id=component_id, configuration_id=configuration_id - ) - return [ScheduleDetail.from_api_response(schedule) for schedule in schedules_api] - - -async def fetch_schedules_for_flow_summaries( - client: KeboolaClient, flow_summaries: list[FlowSummary] -) -> list[FlowSummary]: - """ - Fetch schedules for a list of flow summaries. - - :param client: KeboolaClient instance - :param flow_summaries: The list of flow summaries to add the schedule to - :return: The list of flow summaries with the schedules added - """ - for flow_summary in flow_summaries: - schedules = await list_schedules_for_config( - client=client, component_id=flow_summary.component_id, configuration_id=flow_summary.configuration_id - ) - flow_summary.schedules_count = len(schedules) - return flow_summaries - - -async def fetch_schedules_for_flows( - client: KeboolaClient, links_manager: ProjectLinksManager, list_of_flows: list[Flow] -) -> list[Flow]: - """ - Fetch schedules for a list of flows. - - :param client: KeboolaClient instance - :param links_manager: The links manager to use - :param list_of_flows: The list of flows to fetch the schedules for - :return: The list of flows with the schedules added - """ - for flow in list_of_flows: - schedules = await list_schedules_for_config( - client=client, component_id=flow.component_id, configuration_id=flow.configuration_id - ) - link = links_manager.get_scheduler_detail_link(flow.configuration_id, flow.component_id) - flow.schedules = SchedulesOutput(schedules=schedules, n_schedules=len(schedules), links=[link]) - return list_of_flows diff --git a/src/keboola_mcp_server/tools/flow/scheduler_model.py b/src/keboola_mcp_server/tools/flow/scheduler_model.py deleted file mode 100644 index d0ee7806b..000000000 --- a/src/keboola_mcp_server/tools/flow/scheduler_model.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Pydantic models for representing scheduler details and requests for Agent tool output and inputs. - -These models represent the structure of schedulers used to automate flow execution. -""" - -from typing import Literal - -from pydantic import AliasChoices, BaseModel, Field - -from keboola_mcp_server.clients.scheduler import ScheduleApiResponse, TargetExecution -from keboola_mcp_server.links import Link - - -class ScheduleRequest(BaseModel): - action: Literal['add', 'update', 'remove'] = Field(description='Action to perform on the schedule.') - schedule_id: str | None = Field( - description='ID of the schedule configuration to update. None if creating a new schedule.', - default=None, - serialization_alias='scheduleId', - validation_alias=AliasChoices('scheduleId', 'schedule_id'), - ) - timezone: str | None = Field(description='Timezone for the schedule. Default UTC if None provided.', default=None) - cron_tab: str | None = Field( - description=( - 'Cron expression for the schedule following the format: `* * * * *`.' - 'Where 1. minutes, 2. hours, 3. days of month, 4. months, 5. days of week. Example: `15,45 1,13 * * 0`' - ), - default=None, - validation_alias=AliasChoices('cronTab', 'cron_tab'), - serialization_alias='cronTab', - ) - state: Literal['enabled', 'disabled'] | None = Field(description='Enable or disable the schedule.', default=None) - - -class SchedulesOutput(BaseModel): - """Schedules output used in the flow models when getting details.""" - - schedules: list['ScheduleDetail'] = Field(description='List of schedules', default_factory=list) - n_schedules: int = Field(description='Number of schedules', default=0) - links: list[Link] = Field(description='List of links', default_factory=list) - - -class ScheduleDetail(BaseModel): - """Schedule model for flow tools.""" - - schedule_id: str = Field( - description='Schedule configuration ID', - serialization_alias='scheduleId', - validation_alias=AliasChoices('id', 'schedule_id', 'scheduleId'), - ) - timezone: str = Field(description='Timezone') - state: Literal['enabled', 'disabled'] = Field(description='Schedule state') - timezone: str = Field(description='Timezone') - cron_tab: str = Field( - description=( - 'Cron Tab `* * * * *`. Where 1. minutes, 2. hours, 3. days of month, 4. months, 5. days of week.' - 'Example: `15,45 1,13 * * 0`' - ), - serialization_alias='cronTab', - validation_alias=AliasChoices('cronTab', 'cron_tab'), - ) - target_executions: list[TargetExecution] = Field(default_factory=list, description='List of recent target runs') - - @classmethod - def from_api_response(cls, schedule_api: ScheduleApiResponse) -> 'ScheduleDetail': - """Create a schedule detail from a schedule response.""" - return cls.model_construct( - schedule_id=schedule_api.configuration_id, - timezone=schedule_api.schedule.timezone, - state=schedule_api.schedule.state, - cron_tab=schedule_api.schedule.cron_tab, - target_executions=schedule_api.executions, - ) diff --git a/src/keboola_mcp_server/tools/flow/tools.py b/src/keboola_mcp_server/tools/flow/tools.py deleted file mode 100644 index 7e260e474..000000000 --- a/src/keboola_mcp_server/tools/flow/tools.py +++ /dev/null @@ -1,766 +0,0 @@ -"""Flow management tools for the MCP server (orchestrations/flows).""" - -import copy -import importlib.resources as pkg_resources -import json -import logging -from datetime import datetime, timezone -from typing import Annotated, Any, Optional, Sequence, cast - -from fastmcp import Context, FastMCP -from fastmcp.tools import FunctionTool -from mcp.types import ToolAnnotations -from pydantic import Field - -from keboola_mcp_server import resources -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import ( - CONDITIONAL_FLOW_COMPONENT_ID, - ORCHESTRATOR_COMPONENT_ID, - FlowType, - KeboolaClient, -) -from keboola_mcp_server.clients.storage import CreateConfigurationAPIResponse -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.links import ProjectLinksManager -from keboola_mcp_server.mcp import process_concurrently, toon_serializer_compact, unwrap_results -from keboola_mcp_server.tools.components.utils import ( - build_folder_hint, - clear_configuration_folder_metadata, - folder_field_description, - get_config_folders, - set_cfg_creation_metadata, - set_cfg_update_metadata, - set_configuration_folder_metadata, -) -from keboola_mcp_server.tools.constants import ( - CONFIG_DIFF_PREVIEW_TAG, - FLOW_TOOLS_TAG, -) -from keboola_mcp_server.tools.flow.model import ( - Flow, - FlowToolOutput, - GetFlowsDetailOutput, - GetFlowsListOutput, - GetFlowsOutput, -) -from keboola_mcp_server.tools.flow.scheduler import ( - compute_schedulers_preview, - fetch_schedules_for_flows, - process_schedule_request, -) -from keboola_mcp_server.tools.flow.scheduler_model import ScheduleRequest -from keboola_mcp_server.tools.flow.utils import ( - get_all_flows, - get_flow_configuration, - get_schema_as_markdown, - resolve_flow_by_id, - resolve_flow_schema, - validate_flow_structure, -) -from keboola_mcp_server.tools.project import get_project_info -from keboola_mcp_server.tools.validation import validate_flow_configuration_against_schema - -LOG = logging.getLogger(__name__) - - -def add_flow_tools(mcp: FastMCP) -> None: - """Add flow tools to the MCP server.""" - mcp.add_tool( - FunctionTool.from_function( - create_flow, - tags={FLOW_TOOLS_TAG}, - annotations=ToolAnnotations(destructiveHint=False), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - create_conditional_flow, - tags={FLOW_TOOLS_TAG}, - annotations=ToolAnnotations(destructiveHint=False), - ) - ) - mcp.add_tool( - FunctionTool.from_function( - get_flows, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={FLOW_TOOLS_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - update_flow, - annotations=ToolAnnotations(destructiveHint=True), - tags={FLOW_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - modify_flow, - annotations=ToolAnnotations(destructiveHint=True), - tags={FLOW_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - get_flow_examples, - annotations=ToolAnnotations(readOnlyHint=True), - tags={FLOW_TOOLS_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - get_flow_schema, - annotations=ToolAnnotations(readOnlyHint=True), - tags={FLOW_TOOLS_TAG}, - ) - ) - - LOG.info('Flow tools initialized.') - - -@tool_errors() -async def get_flow_schema( - ctx: Context, - flow_type: Annotated[FlowType, Field(description='The type of flow for which to fetch schema.')], -) -> Annotated[str, Field(description='The configuration schema of the specified flow type.')]: - """ - Returns the JSON schema for the given flow type (markdown). - - PRE-REQUISITES: - - Unknown schema for the target flow type: `keboola.flow` (conditional) or `keboola.orchestrator` (legacy) - - RULES: - - Projects without conditional flows enabled cannot request `keboola.flow` schema - - Use the returned schema to shape `phases` and `tasks` for `create_flow` / `create_conditional_flow` / - `update_flow` - """ - project_info = await get_project_info(ctx) - - if flow_type == CONDITIONAL_FLOW_COMPONENT_ID and not project_info.conditional_flows: - raise ValueError( - f'Conditional flows are not supported in this project. ' - f'Project "{project_info.project_name}" has conditional_flows=false. ' - f'If you want to use conditional flows, please enable them in your project settings. ' - f'Otherwise, use flow_type="{ORCHESTRATOR_COMPONENT_ID}" for legacy flows instead.' - ) - - LOG.info(f'Returning flow configuration schema for flow type: {flow_type}') - client = KeboolaClient.from_state(ctx.session.state) - return await get_schema_as_markdown(client, flow_type) - - -@tool_errors() -async def create_flow( - ctx: Context, - name: Annotated[str, Field(description='A short, descriptive name for the flow.')], - description: Annotated[str, Field(description='Detailed description of the flow purpose.')], - phases: Annotated[list[dict[str, Any]], Field(description='List of phase definitions.')], - tasks: Annotated[list[dict[str, Any]], Field(description='List of task definitions.')], - folder: Annotated[ - str, - Field(description=folder_field_description('flow', 'flows')), - ] = '', -) -> FlowToolOutput: - """ - Creates a new legacy (non-conditional) flow using `keboola.orchestrator`. - - PRE-REQUISITES: - - Always use `get_flow_schema` with flow_type="keboola.orchestrator" and review `get_flow_examples` if unknown - - Collect component configuration IDs for every task you include - - RULES: - - `phases` and `tasks` must follow the orchestrator schema; each entry must include `id` and `name` - - Phases run sequentially; tasks inside a phase run in parallel - - Use `dependsOn` on phases to sequence them; reference other phase ids - - Always share the returned links with the user - - WHEN TO USE: - - Simple/linear orchestrations without branching or conditions - - ETL/ELT pipelines where phases just need ordering and parallel task groups - """ - flow_type = ORCHESTRATOR_COMPONENT_ID - flow_configuration = get_flow_configuration(phases=phases, tasks=tasks, flow_type=flow_type) - - LOG.info(f'Creating new flow: {name} (type: {ORCHESTRATOR_COMPONENT_ID})') - client = KeboolaClient.from_state(ctx.session.state) - - # Validate flow structure before to catch semantic errors in the structure - validate_flow_structure(cast(JsonDict, flow_configuration), flow_type=flow_type) - # Validate flow configuration against schema to catch syntax errors in the configuration - schema = await resolve_flow_schema(client, flow_type) - validate_flow_configuration_against_schema(cast(JsonDict, flow_configuration), flow_type=flow_type, schema=schema) - - links_manager = await ProjectLinksManager.from_client(client) - new_raw_configuration = await client.storage_client.configuration_create( - component_id=flow_type, - name=name, - description=description, - configuration=flow_configuration, - ) - api_config = CreateConfigurationAPIResponse.model_validate(new_raw_configuration) - await set_cfg_creation_metadata( - client, - component_id=flow_type, - configuration_id=str(new_raw_configuration['id']), - ) - - folder = folder.strip() - change_summary = None - if folder: - try: - await set_configuration_folder_metadata(client, flow_type, api_config.id, folder) - except Exception: - LOG.warning( - 'Unable to set folder metadata for component "%s", configuration "%s".', - flow_type, - api_config.id, - ) - else: - try: - total, existing_folders, lower_bound = await get_config_folders(client, flow_type) - change_summary = build_folder_hint( - total, existing_folders, 'legacy flows', 'modify_flow', lower_bound=lower_bound - ) - except Exception: - LOG.warning( - 'Unable to fetch flow folders for component "%s" when creating flow "%s".', - flow_type, - api_config.id, - ) - - flow_links = links_manager.get_flow_links(flow_id=api_config.id, flow_name=api_config.name, flow_type=flow_type) - tool_response = FlowToolOutput( - configuration_id=api_config.id, - component_id=flow_type, - description=api_config.description or '', - version=api_config.version, - timestamp=datetime.now(timezone.utc), - success=True, - links=flow_links, - change_summary=change_summary, - ) - - LOG.info(f'Created legacy flow "{name}" with configuration ID "{api_config.id}" (type: {flow_type})') - return tool_response - - -@tool_errors() -async def create_conditional_flow( - ctx: Context, - name: Annotated[str, Field(description='A short, descriptive name for the flow.')], - description: Annotated[str, Field(description='Detailed description of the flow purpose.')], - phases: Annotated[list[dict[str, Any]], Field(description='List of phase definitions for conditional flows.')], - tasks: Annotated[list[dict[str, Any]], Field(description='List of task definitions for conditional flows.')], - folder: Annotated[ - str, - Field(description=folder_field_description('flow', 'flows')), - ] = '', -) -> FlowToolOutput: - """ - Creates a new conditional flow configuration using `keboola.flow`. - - PRE-REQUISITES: - - Always use `get_flow_schema` with flow_type="keboola.flow" and review `get_flow_examples` if unknown - - Gather component configuration IDs for all tasks you include - - RULES: - - `phases` and `tasks` must follow the keboola.flow schema; each entry needs `id` and `name` - - Exactly one entry phase (no incoming transitions); all phases must be reachable - - Connect phases via `next` transitions; no cycles or dangling phases; empty `next` means flow end - - Task/phase failures already stop the flow; add retries/conditions only if the user requests them - - Always share the returned links with the user - - WHEN TO USE: - - Flows needing branching, conditions, retries, or notifications - - Default choice when user simply says "create a flow," unless they explicitly want legacy orchestrator behavior - """ - flow_type = CONDITIONAL_FLOW_COMPONENT_ID - flow_configuration = get_flow_configuration(phases=phases, tasks=tasks, flow_type=flow_type) - - LOG.info(f'Creating flow: {name} (type: {flow_type})') - client = KeboolaClient.from_state(ctx.session.state) - - # Validate flow structure to catch semantic errors in the structure - validate_flow_structure(flow_configuration=flow_configuration, flow_type=flow_type) - # Validate flow configuration against schema to catch syntax errors in the configuration - schema = await resolve_flow_schema(client, flow_type) - validate_flow_configuration_against_schema(cast(JsonDict, flow_configuration), flow_type=flow_type, schema=schema) - - links_manager = await ProjectLinksManager.from_client(client) - new_raw_configuration = await client.storage_client.configuration_create( - component_id=flow_type, - name=name, - description=description, - configuration=flow_configuration, - ) - api_config = CreateConfigurationAPIResponse.model_validate(new_raw_configuration) - - await set_cfg_creation_metadata( - client, - component_id=flow_type, - configuration_id=str(new_raw_configuration['id']), - ) - - folder = folder.strip() - change_summary = None - if folder: - try: - await set_configuration_folder_metadata(client, flow_type, api_config.id, folder) - except Exception: - LOG.warning( - 'Unable to set folder metadata for component "%s", configuration "%s".', - flow_type, - api_config.id, - ) - else: - try: - total, existing_folders, lower_bound = await get_config_folders(client, flow_type) - change_summary = build_folder_hint( - total, existing_folders, 'conditional flows', 'modify_flow', lower_bound=lower_bound - ) - except Exception: - LOG.warning( - 'Unable to fetch flow folders for component "%s" when creating flow "%s".', - flow_type, - api_config.id, - ) - - flow_links = links_manager.get_flow_links(flow_id=api_config.id, flow_name=api_config.name, flow_type=flow_type) - tool_response = FlowToolOutput( - configuration_id=api_config.id, - component_id=flow_type, - description=api_config.description or '', - version=api_config.version, - timestamp=datetime.now(timezone.utc), - success=True, - links=flow_links, - change_summary=change_summary, - ) - - LOG.info(f'Created conditional flow "{name}" with configuration ID "{api_config.id}" (type: {flow_type})') - return tool_response - - -@tool_errors() -async def update_flow( - ctx: Context, - configuration_id: Annotated[str, Field(description='ID of the flow configuration.')], - flow_type: Annotated[ - FlowType, - Field( - description=( - 'The type of flow to update. Use "keboola.flow" for conditional flows or ' - '"keboola.orchestrator" for legacy flows. This MUST match the existing flow type.' - ) - ), - ], - change_description: Annotated[str, Field(description='Description of changes made.')], - phases: Annotated[list[dict[str, Any]], Field(description='Updated list of phase definitions.')] = None, - tasks: Annotated[list[dict[str, Any]], Field(description='Updated list of task definitions.')] = None, - name: Annotated[str, Field(description='Updated flow name. Only updated if provided.')] = '', - description: Annotated[str, Field(description='Updated flow description. Only updated if provided.')] = '', - is_disabled: Annotated[ - bool | None, - Field( - description=( - "Enable or disable the flow. Set to True to disable execution (flow won't run), " - 'False to enable execution (flow will run). Only provide if changing the status, ' - 'leave as null to preserve current state.' - ), - ), - ] = None, - folder: Annotated[ - Optional[str], - Field(description=folder_field_description('flow', 'flows')), - ] = None, -) -> FlowToolOutput: - """ - Updates an existing flow configuration (either legacy `keboola.orchestrator` or conditional `keboola.flow`). - - PRE-REQUISITES: - - Always use `get_flow_schema` (and `get_flow_examples`) for that flow type you want to update to follow the - required structure and see the examples if unknown - - Only pass `phases`/`tasks` when you want to replace them; omit to keep the existing ones unchanged - - RULES (ALL FLOWS): - - `flow_type` must match the stored component id of the flow; do not switch flow types during update - - `phases` and `tasks` must follow the schema for the selected flow type; include at least `id` and `name` - - Tasks must reference existing component configurations; keep dependencies consistent - - Always provide a clear `change_description` and surface any links returned in the response to the user - - CONDITIONAL FLOWS (`keboola.flow`): - - Maintain a single entry phase and ensure every phase is reachable; connect phases via `next` transitions - - No cycles or dangling phases; failed tasks already stop the flow, so only add retries/conditions if requested - - LEGACY FLOWS (`keboola.orchestrator`): - - Phases run sequentially; tasks inside a phase run in parallel; `dependsOn` references other phase ids - - Use `continueOnFailure` or best-effort patterns only when the user explicitly asks for them - - WHEN TO USE: - - Renaming a flow, updating descriptions, adding/removing phases or tasks, adjusting dependencies, - or enabling/disabling flow execution - """ - return await modify_flow( - ctx=ctx, - configuration_id=configuration_id, - flow_type=flow_type, - change_description=change_description, - phases=phases, - tasks=tasks, - name=name, - description=description, - schedules=tuple(), - is_disabled=is_disabled, - folder=folder, - ) - - -@tool_errors() -async def modify_flow( - ctx: Context, - configuration_id: Annotated[str, Field(description='ID of the flow configuration.')], - flow_type: Annotated[ - FlowType, - Field( - description=( - 'The type of flow to update. Use "keboola.flow" for conditional flows or ' - '"keboola.orchestrator" for legacy flows. This MUST match the existing flow type.' - ) - ), - ], - change_description: Annotated[str, Field(description='Description of changes made.')], - phases: Annotated[list[dict[str, Any]], Field(description='Updated list of phase definitions.')] = None, - tasks: Annotated[list[dict[str, Any]], Field(description='Updated list of task definitions.')] = None, - name: Annotated[str, Field(description='Updated flow name. Only updated if provided.')] = '', - description: Annotated[str, Field(description='Updated flow description. Only updated if provided.')] = '', - schedules: Annotated[ - Sequence[ScheduleRequest], - Field( - description=( - 'Optional sequence of schedule requests to add/update/remove schedules for this flow. ' - 'Each request must have "action": "add"|"update"|"remove". ' - 'For add: include "cron_tab", "state" ("enabled"|"disabled"), "timezone". ' - 'For update/remove: include "schedule_id". ' - 'Example: [{"action": "add", "cron_tab": "0 8 * * 1-5", "state": "enabled", "timezone": "UTC"}]' - ) - ), - ] = tuple(), - is_disabled: Annotated[ - bool | None, - Field( - description=( - "Enable or disable the flow. Set to True to disable execution (flow won't run), " - 'False to enable execution (flow will run). Only provide if changing the status, ' - 'leave as null to preserve current state.' - ), - ), - ] = None, - folder: Annotated[ - Optional[str], - Field(description=folder_field_description('flow', 'flows')), - ] = None, -) -> FlowToolOutput: - """ - Updates an existing flow configuration (either legacy `keboola.orchestrator` or conditional `keboola.flow`) or - manages schedules for this flow. - - PRE-REQUISITES: - - Always use `get_flow_schema` (and `get_flow_examples`) for that flow type you want to update to follow the - required structure and see the examples if unknown - - Only pass `phases`/`tasks` when you want to replace them; omit to keep the existing ones unchanged - - RULES (ALL FLOWS): - - `flow_type` must match the stored component id of the flow; do not switch flow types during update - - `phases` and `tasks` must follow the schema for the selected flow type; include at least `id` and `name` - - Tasks must reference existing component configurations; keep dependencies consistent - - Always provide a clear `change_description` and surface any links returned in the response to the user - - A flow can have multiple schedules for automation runs. Add/update/remove schedules only if requested. - - When updating a flow or a schedule, specify only the fields you want to update, others will be kept unchanged. - - CONDITIONAL FLOWS (`keboola.flow`): - - Maintain a single entry phase and ensure every phase is reachable; connect phases via `next` transitions - - No cycles or dangling phases; failed tasks already stop the flow, so only add retries/conditions if requested - - LEGACY FLOWS (`keboola.orchestrator`): - - Phases run sequentially; tasks inside a phase run in parallel; `dependsOn` references other phase ids - - Use `continueOnFailure` or best-effort patterns only when the user explicitly asks for them - - WHEN TO USE: - - Renaming a flow, updating descriptions, adding/removing phases or tasks, updating schedules, - adjusting dependencies, or enabling/disabling flow execution - """ - - project_info = await get_project_info(ctx) - if flow_type == CONDITIONAL_FLOW_COMPONENT_ID and not project_info.conditional_flows: - raise ValueError( - f'Conditional flows are not supported in this project. ' - f'Project "{project_info.project_name}" has conditional_flows=false. ' - f'If you want to use conditional flows, please enable them in your project settings. ' - f'Otherwise, use flow_type="{ORCHESTRATOR_COMPONENT_ID}" for legacy flows instead.' - ) - - client = KeboolaClient.from_state(ctx.session.state) - - response_message = None - has_config_changes = ( - bool(name) or bool(description) or phases is not None or tasks is not None or is_disabled is not None - ) - - if has_config_changes: - LOG.info(f'Updating flow configuration: {configuration_id} (type: {flow_type})') - _, flow_configuration, *_ = await update_flow_internal( - client=client, - configuration_id=configuration_id, - flow_type=flow_type, - change_description=change_description, - phases=phases, - tasks=tasks, - name=name, - description=description, - schedules=None, - is_disabled=is_disabled, - ) - updated_raw_configuration = await client.storage_client.configuration_update( - component_id=flow_type, - configuration_id=configuration_id, - configuration=flow_configuration, - change_description=change_description, - updated_name=name, - updated_description=description, - is_disabled=is_disabled, - ) - api_config = CreateConfigurationAPIResponse.model_validate(updated_raw_configuration) - await set_cfg_update_metadata( - client, - component_id=flow_type, - configuration_id=api_config.id, - configuration_version=api_config.version, - ) - else: - current_config = await client.storage_client.configuration_detail( - component_id=flow_type, - configuration_id=configuration_id, - ) - api_config = CreateConfigurationAPIResponse.model_validate(current_config) - - folder_hint = None - if folder is None: - try: - total, existing_folders, lower_bound = await get_config_folders(client, flow_type) - config_label = 'legacy flows' if flow_type == ORCHESTRATOR_COMPONENT_ID else 'conditional flows' - folder_hint = build_folder_hint( - total, existing_folders, config_label, 'modify_flow', lower_bound=lower_bound - ) - except Exception: - LOG.warning( - 'Unable to fetch flow folders for component "%s" when updating flow "%s".', - flow_type, - configuration_id, - ) - else: - folder_stripped = folder.strip() - if folder_stripped: - await set_configuration_folder_metadata(client, flow_type, configuration_id, folder_stripped) - else: - await clear_configuration_folder_metadata(client, flow_type, configuration_id) - - links_manager = await ProjectLinksManager.from_client(client) - flow_links = links_manager.get_flow_links(flow_id=api_config.id, flow_name=api_config.name, flow_type=flow_type) - # Process schedule requests if provided - if schedules is not None and len(schedules) > 0: - responses = await process_schedule_request( - client=client, - target_component_id=flow_type, - target_configuration_id=configuration_id, - requests=schedules, - ) - response_message = 'Schedules request processed successfully: \n' + '\n'.join(responses) - LOG.info(f'Successfully processed {len(schedules)} schedule request(s) for flow {configuration_id}') - flow_links.append(links_manager.get_scheduler_detail_link(configuration_id, flow_type)) - - tool_response = FlowToolOutput( - configuration_id=api_config.id, - component_id=flow_type, - description=api_config.description or '', - version=api_config.version, - timestamp=datetime.now(timezone.utc), - response=response_message, - change_summary=folder_hint, - success=True, - links=flow_links, - ) - LOG.info(f'Updated flow configuration: {api_config.id}') - return tool_response - - -async def update_flow_internal( - *, - client: KeboolaClient, - configuration_id: str, - flow_type: FlowType, - change_description: str, - phases: list[dict[str, Any]] | None = None, - tasks: list[dict[str, Any]] | None = None, - name: str = '', - description: str = '', - schedules: Sequence[ScheduleRequest] | None = tuple(), - is_disabled: bool | None = None, - folder: Optional[str] = None, -) -> tuple[JsonDict, JsonDict, dict[str, Any] | None]: - current_config = await client.storage_client.configuration_detail( - component_id=flow_type, configuration_id=configuration_id - ) - flow_configuration = cast(JsonDict, current_config.get('configuration', {})) - flow_configuration = copy.deepcopy(flow_configuration) - - updated_configuration = get_flow_configuration(phases=phases, tasks=tasks, flow_type=flow_type) - if updated_configuration.get('phases'): - flow_configuration['phases'] = updated_configuration['phases'] - if updated_configuration.get('tasks'): - flow_configuration['tasks'] = updated_configuration['tasks'] - - # Validate flow structure to catch semantic errors in the structure - validate_flow_structure(flow_configuration=flow_configuration, flow_type=flow_type) - # Validate flow configuration against schema to catch syntax errors in the configuration - schema = await resolve_flow_schema(client, flow_type) - validate_flow_configuration_against_schema(cast(JsonDict, flow_configuration), flow_type=flow_type, schema=schema) - - mutator_preview: dict[str, Any] | None = None - if schedules is not None and len(schedules) > 0: - mutator_preview = await compute_schedulers_preview( - client=client, - configuration_id=configuration_id, - flow_type=flow_type, - schedules=schedules, - ) - - folder_preview: dict[str, Any] | None = None - if folder is not None: - normalized_folder = folder.strip() - try: - current_metadata = await client.storage_client.configuration_metadata_get( - component_id=flow_type, configuration_id=configuration_id - ) - current_folder = next( - ( - m.get('value', '') - for m in current_metadata - if m.get('key') == MetadataField.CONFIGURATION_FOLDER_NAME - ), - '', - ) - if normalized_folder != current_folder: - folder_preview = {'original_folder': current_folder, 'updated_folder': normalized_folder} - except Exception as e: - LOG.warning( - 'Failed to fetch configuration metadata for folder preview ' - '(component_id=%s, configuration_id=%s): %s. Proceeding without folder preview.', - flow_type, - configuration_id, - e, - ) - - combined_preview: dict[str, Any] | None = {**(mutator_preview or {}), **(folder_preview or {})} or None - return current_config, flow_configuration, combined_preview - - -@tool_errors() -async def get_flows( - ctx: Context, - flow_ids: Annotated[ - Sequence[str], - Field( - description=( - 'IDs of flows to retrieve full details for. ' - 'When provided (non-empty), returns full flow configurations including phases and tasks. ' - 'When empty [], lists all flows in the project as summaries.' - ) - ), - ] = tuple(), -) -> GetFlowsOutput: - """ - Lists flows or retrieves full details for specific flows. - - WHEN NOT TO USE: - - Do NOT call with `flow_ids=[]` just to find a flow by name. Use `search` with - item_types=["flow"] instead. - - Only use `flow_ids=[]` when you need a complete list of all flows in the project. - - OPTIONS: - - `flow_ids=[]` → summaries of all flows in the project - - `flow_ids=["id1", ...]` → full details (including phases/tasks) for those flows - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - # Case 1: flow_ids provided - return full details for those flows - if flow_ids: - - async def fetch_flow_detail(flow_id: str) -> Flow: - api_flow, found_type = await resolve_flow_by_id(client, flow_id) - LOG.info(f'Found flow {flow_id} under flow type {found_type}.') - links = links_manager.get_flow_links( - api_flow.configuration_id, flow_name=api_flow.name, flow_type=found_type - ) - return Flow.from_api_response(api_config=api_flow, flow_component_id=found_type, links=links) - - results = await process_concurrently(flow_ids, fetch_flow_detail) - flows = unwrap_results(results, 'Failed to fetch one or more flows') - - LOG.info(f'Retrieved full details for {len(flows)} flows.') - flows = await fetch_schedules_for_flows(client=client, links_manager=links_manager, list_of_flows=flows) - return GetFlowsDetailOutput(flows=flows) - - # Case 2: no flow_ids - list all flows as summaries - flows = await get_all_flows(client) - LOG.info(f'Retrieved {len(flows)} flows.') - links = [ - links_manager.get_flows_dashboard_link(ORCHESTRATOR_COMPONENT_ID), - links_manager.get_flows_dashboard_link(CONDITIONAL_FLOW_COMPONENT_ID), - ] - return GetFlowsListOutput(flows=flows, links=links) - - -@tool_errors() -async def get_flow_examples( - ctx: Context, - flow_type: Annotated[FlowType, Field(description='The type of the flow to retrieve examples for.')], -) -> Annotated[str, Field(description='Examples of the flow configurations.')]: - """ - Retrieves examples of valid flow configurations. - - PRE-REQUISITES: - - Unknown examples for the target flow type: `keboola.flow` (conditional) or `keboola.orchestrator` (legacy) to help - build the specific flow configuration by mirroring the structure/fields. - - RULES: - - Conditional-flow examples require conditional flows to be enabled; otherwise use legacy orchestrator examples - - Present the examples or cite unavailability to the user - """ - project_info = await get_project_info(ctx) - if flow_type == CONDITIONAL_FLOW_COMPONENT_ID and not project_info.conditional_flows: - raise ValueError( - f'Conditional flows are not supported in this project. ' - f'Project "{project_info.project_name}" has conditional_flows=false. ' - f'If you want to use conditional flows, please enable them in your project settings. ' - f'Otherwise, use flow_type="{ORCHESTRATOR_COMPONENT_ID}" for legacy flow examples instead.' - ) - - filename = ( - 'conditional_flow_examples.jsonl' - if flow_type == CONDITIONAL_FLOW_COMPONENT_ID - else 'legacy_flow_examples.jsonl' - ) - file_path = pkg_resources.files(resources) / 'flow_examples' / filename - - markdown = f'# Flow Configuration Examples for `{flow_type}`\n\n' - - with file_path.open('r', encoding='utf-8') as f: - for i, line in enumerate(f, 1): - data = json.loads(line) - markdown += f'{i}. Flow Configuration:\n```json\n{json.dumps(data, indent=2)}\n```\n\n' - - return markdown diff --git a/src/keboola_mcp_server/tools/flow/utils.py b/src/keboola_mcp_server/tools/flow/utils.py deleted file mode 100644 index 0475bc810..000000000 --- a/src/keboola_mcp_server/tools/flow/utils.py +++ /dev/null @@ -1,472 +0,0 @@ -"""Utility functions for flow management.""" - -import json -import logging -from collections import Counter, defaultdict -from importlib import resources -from typing import Any, Mapping, Sequence - -from keboola_mcp_server.clients.client import ( - CONDITIONAL_FLOW_COMPONENT_ID, - FLOW_TYPES, - ORCHESTRATOR_COMPONENT_ID, - FlowType, - KeboolaClient, -) -from keboola_mcp_server.clients.storage import APIFlowResponse, JsonDict -from keboola_mcp_server.mcp import process_concurrently -from keboola_mcp_server.tools.components.utils import fetch_component -from keboola_mcp_server.tools.flow.model import ( - ConditionalFlowPhase, - ConditionalFlowTask, - FlowPhase, - FlowSummary, - FlowTask, -) -from keboola_mcp_server.tools.flow.scheduler import list_schedules_for_config - -LOG = logging.getLogger(__name__) - -RESOURCES = 'keboola_mcp_server.resources' -FLOW_SCHEMAS: Mapping[FlowType, str] = { - ORCHESTRATOR_COMPONENT_ID: 'flow-schema.json', -} - - -def _load_schema(flow_type: FlowType) -> JsonDict: - """Load a schema from the resources folder.""" - with resources.open_text(RESOURCES, FLOW_SCHEMAS[flow_type], encoding='utf-8') as f: - return json.load(f) - - -async def resolve_flow_schema(client: KeboolaClient, flow_type: FlowType) -> JsonDict: - """ - Resolve the JSON schema for a flow type. - - Conditional flows (``keboola.flow``) are sourced live from the Developer Portal via - ``fetch_component`` and cached per session. Legacy orchestrator flows stay bundled. - - :param client: Authenticated Keboola client instance. - :param flow_type: The flow type / component id to resolve the schema for. - :return: The configuration schema as a JSON dict. - :raises ValueError: If the live conditional schema cannot be retrieved or is empty. - """ - if flow_type != CONDITIONAL_FLOW_COMPONENT_ID: - return _load_schema(flow_type) # legacy orchestrator stays bundled - - cached = client.get_cached_flow_schema(flow_type) - if cached is not None: - return cached - - failure_message = ( - 'Could not retrieve the conditional flow (keboola.flow) configuration schema from the ' - 'Developer Portal. The schema is required to create or validate conditional flows. ' - 'Please retry; if this persists the keboola.flow component schema may be unavailable on ' - 'this stack.' - ) - try: - component = await fetch_component(client, CONDITIONAL_FLOW_COMPONENT_ID) - except Exception as e: - # Any failure to fetch the live schema must surface as the recoverable message: non-404 - # HTTPStatusError (re-raised by fetch_component), transport/network errors, and unexpected - # payloads (e.g. a pydantic ValidationError when the AI Service returns a malformed response) - # all map to the same hard-fail so the agent gets actionable guidance, never a raw traceback. - raise ValueError(failure_message) from e - - schema = component.configuration_schema - if not schema: - raise ValueError(failure_message) - - client.cache_flow_schema(flow_type, schema) - return schema - - -async def get_schema_as_markdown(client: KeboolaClient, flow_type: FlowType) -> str: - """Return the flow schema as a markdown formatted string.""" - schema = await resolve_flow_schema(client, flow_type) - return f'```json\n{json.dumps(schema, indent=2)}\n```' - - -def get_flow_configuration( - phases: list[dict[str, Any]] | None, tasks: list[dict[str, Any]] | None, flow_type: FlowType -) -> JsonDict: - """Get the flow configuration from tasks and phases. For legacy flows, apply necessary sanitization. - - :param phases: The list of phases. - :param tasks: The list of tasks. - :param flow_type: The type of flow to convert. - :return: The dictionary containing the flow configuration (phases and tasks) serialized to JSON. - """ - if flow_type == ORCHESTRATOR_COMPONENT_ID: - processed_phases = ensure_legacy_phase_ids(phases or []) - processed_tasks = ensure_legacy_task_ids(tasks or []) - return { - 'phases': [phase.model_dump(by_alias=True) for phase in processed_phases], - 'tasks': [task.model_dump(by_alias=True) for task in processed_tasks], - } - else: - processed_phases = [ConditionalFlowPhase.model_validate(phase) for phase in phases or []] - processed_tasks = [ConditionalFlowTask.model_validate(task) for task in tasks or []] - return { - 'phases': [phase.model_dump(exclude_unset=True, by_alias=True) for phase in processed_phases], - 'tasks': [task.model_dump(exclude_unset=True, by_alias=True) for task in processed_tasks], - } - - -def validate_flow_structure( - flow_configuration: JsonDict, - flow_type: FlowType, -) -> None: - """ - Validate that the flow structure is valid by checking logical and structural constraints. - - :param flow_configuration: The flow configuration to validate. - :param flow_type: The type of flow to validate. - :raises ValueError: If the flow structure is invalid. - """ - if flow_type == ORCHESTRATOR_COMPONENT_ID: - _validate_legacy_flow_structure( - phases=[FlowPhase.model_validate(phase) for phase in flow_configuration.get('phases', [])], - tasks=[FlowTask.model_validate(task) for task in flow_configuration.get('tasks', [])], - ) - else: - _validate_conditional_flow_structure( - phases=[ConditionalFlowPhase.model_validate(phase) for phase in flow_configuration.get('phases', [])], - tasks=[ConditionalFlowTask.model_validate(task) for task in flow_configuration.get('tasks', [])], - ) - - -def ensure_legacy_phase_ids(phases: list[dict[str, Any]]) -> list[FlowPhase]: - """Ensure all phases have unique IDs and proper structure for legacy flows""" - processed_phases = [] - used_ids = set() - - for i, phase in enumerate(phases): - phase_data = phase.copy() - - if 'id' not in phase_data or not phase_data['id']: - phase_id = i + 1 - while phase_id in used_ids: - phase_id += 1 - phase_data['id'] = phase_id - - if 'name' not in phase_data: - phase_data['name'] = f"Phase {phase_data['id']}" - - try: - validated_phase = FlowPhase.model_validate(phase_data) - used_ids.add(validated_phase.id) - processed_phases.append(validated_phase) - except Exception as e: - raise ValueError(f'Invalid phase configuration: {e}') - - return processed_phases - - -def ensure_legacy_task_ids(tasks: list[dict[str, Any]]) -> list[FlowTask]: - """Ensure all tasks have unique IDs and proper structure using Pydantic validation for legacy flows""" - processed_tasks = [] - used_ids = set() - - # Task ID pattern inspired by Kai-Bot implementation: - # https://github.com/keboola/kai-bot/blob/main/src/keboola/kaibot/backend/flow_backend.py - # - # ID allocation strategy: - # - Phase IDs: 1, 2, 3... (small sequential numbers) - # - Task IDs: 20001, 20002, 20003... (high sequential numbers) - # - # This namespace separation technique ensures phase and task IDs never collide - # while maintaining human-readable sequential numbering. - task_counter = 20001 - - for task in tasks: - task_data = task.copy() - - if 'id' not in task_data or not task_data['id']: - while task_counter in used_ids: - task_counter += 1 - task_data['id'] = task_counter - task_counter += 1 - - if 'name' not in task_data: - task_data['name'] = f"Task {task_data['id']}" - - if 'task' not in task_data: - raise ValueError(f"Task {task_data['id']} missing 'task' configuration") - - if 'componentId' not in task_data.get('task', {}): - raise ValueError(f"Task {task_data['id']} missing componentId in task configuration") - - task_obj = task_data.get('task', {}) - if 'mode' not in task_obj: - task_obj['mode'] = 'run' - task_data['task'] = task_obj - - try: - validated_task = FlowTask.model_validate(task_data) - used_ids.add(validated_task.id) - processed_tasks.append(validated_task) - except Exception as e: - raise ValueError(f'Invalid task configuration: {e}') - - return processed_tasks - - -async def resolve_flow_by_id(client: KeboolaClient, flow_id: str) -> tuple[APIFlowResponse, FlowType]: - """ - Resolve a flow by ID across all flow types. - - :param client: Keboola client instance. - :param flow_id: The flow configuration ID to resolve. - :return: Tuple of (APIFlowResponse, flow_type) if found. - :raises ValueError: If flow cannot be resolved in any flow type. - """ - for flow_type in FLOW_TYPES: - try: - raw_flow = await client.storage_client.configuration_detail( - component_id=flow_type, configuration_id=flow_id - ) - api_flow = APIFlowResponse.model_validate(raw_flow) - return api_flow, flow_type - except Exception: - continue - - raise ValueError(f'Flow configuration "{flow_id}" not found') - - -async def get_flows_by_ids(client: KeboolaClient, flow_ids: Sequence[str]) -> list[FlowSummary]: - flows: list[FlowSummary] = [] - - for flow_id in flow_ids: - try: - api_flow, flow_type = await resolve_flow_by_id(client, flow_id) - flow_summary = FlowSummary.from_api_response(api_config=api_flow, flow_component_id=flow_type) - flows.append(flow_summary) - except ValueError as e: - LOG.warning(f'Flow {flow_id} not found: {e}') - continue - - return flows - - -async def get_flows_by_type(client: KeboolaClient, flow_type: FlowType) -> list[FlowSummary]: - - async def _fetch_schedules_for_flow_summaries(flow_summary: FlowSummary) -> FlowSummary: - # Fetch schedule count - try: - schedules = await list_schedules_for_config( - client=client, component_id=flow_summary.component_id, configuration_id=flow_summary.configuration_id - ) - flow_summary.schedules_count = len(schedules) - except Exception as e: - LOG.warning(f'Failed to fetch schedules for flow {flow_summary.configuration_id}: {e}') - flow_summary.schedules_count = 0 - return flow_summary - - raw_flows = await client.storage_client.configuration_list(component_id=flow_type) - flows = [] - - for raw in raw_flows: - flow_summary = FlowSummary.from_api_response( - api_config=APIFlowResponse.model_validate(raw), flow_component_id=flow_type - ) - - flows.append(flow_summary) - - flows = await process_concurrently(flows, _fetch_schedules_for_flow_summaries) - return flows - - -async def get_all_flows(client: KeboolaClient) -> list[FlowSummary]: - all_flows = [] - for flow_type in FLOW_TYPES: - flows = await get_flows_by_type(client=client, flow_type=flow_type) - all_flows.extend(flows) - return all_flows - - -def _validate_legacy_flow_structure( - phases: list[FlowPhase], - tasks: list[FlowTask], -) -> None: - """Validate that the legacy flow structure is valid (phases exist and graph is not circular)""" - phase_ids = {phase.id for phase in phases} - - for phase in phases: - for dep_id in phase.depends_on: - if dep_id not in phase_ids: - raise ValueError(f'Phase {phase.id} depends on non-existent phase {dep_id}') - - for task in tasks: - if task.phase not in phase_ids: - raise ValueError(f'Task {task.id} references non-existent phase {task.phase}') - - # Check for circular dependencies - _check_legacy_circular_dependencies(phases) - - -def _check_legacy_circular_dependencies(phases: list[FlowPhase]) -> None: - """Check for circular dependencies in a legacy flow.""" - edges = {phase.id: phase.depends_on for phase in phases} - all_phase_ids = {phase.id for phase in phases} - _check_circular_dependencies(edges, all_phase_ids) - - -def _check_circular_dependencies(edges: dict[Any, list[Any]], all_node_ids: set[Any] | None = None) -> None: - """ - Generic circular dependency check that accepts edges in format {node_id: [target_node_id, ...]}. - - Optimized circular dependency check that: - 1. Uses O(n) dict lookup instead of O(n²) list search - 2. Returns detailed cycle path information for better debugging - - :param edges: Dictionary mapping node IDs to lists of target node IDs. - :param all_node_ids: Optional set of all node IDs in the graph. If provided, ensures all nodes are checked. - :raises ValueError: If a circular dependency is detected. - """ - - def _has_cycle(node_id: Any, _visited: set, rec_stack: set, path: list[Any]) -> list[Any] | None: - """ - Returns None if no cycle found, or List[node_ids] representing the cycle path. - """ - _visited.add(node_id) - rec_stack.add(node_id) - path.append(node_id) - - targets = edges.get(node_id, []) - - for target_id in targets: - if target_id not in _visited: - cycle = _has_cycle(target_id, _visited, rec_stack, path) - if cycle is not None: - return cycle - - elif target_id in rec_stack: - try: - cycle_start_index = path.index(target_id) - return path[cycle_start_index:] + [target_id] - except ValueError: - return [node_id, target_id] - - path.pop() - rec_stack.remove(node_id) - return None - - visited = set() - nodes_to_check = all_node_ids if all_node_ids is not None else set(edges.keys()) - - for node_id in nodes_to_check: - if node_id not in visited: - cycle_path = _has_cycle(node_id, visited, set(), []) - if cycle_path is not None: - cycle_str = ' -> '.join(str(pid) for pid in cycle_path) - raise ValueError(f'Circular dependency detected: {cycle_str}') - - -def _validate_conditional_flow_structure( - phases: list[ConditionalFlowPhase], - tasks: list[ConditionalFlowTask], -) -> None: - """ - Validate that the conditional flow structure is valid by checking reachability, existence of entry phase and ending - phase. - :param phases: List of conditional flow phases to validate. - :param tasks: List of conditional flow tasks to validate. - :raises ValueError: If the flow structure is invalid. - """ - - # Validate that there are no duplicate phase or task IDs - counter_phases = Counter([phase.id for phase in phases]) - phase_ids = set(counter_phases) - if counter_phases and counter_phases.most_common(1)[0][1] > 1: - duplicate_phase_ids = [pid for pid, count in counter_phases.most_common() if count > 1] - raise ValueError(f'Flow contains duplicate phase IDs: {duplicate_phase_ids}.') - counter_tasks = Counter([task.id for task in tasks]) - if counter_tasks and counter_tasks.most_common(1)[0][1] > 1: - duplicate_task_ids = [tid for tid, count in counter_tasks.most_common() if count > 1] - raise ValueError(f'Flow contains duplicate task IDs: {duplicate_task_ids}.') - - # Validate that all tasks reference existing phases - for task in tasks: - if task.phase not in phase_ids: - raise ValueError(f'Task {task.id} references non-existent phase {task.phase}') - - # Build graph of transitions: phase_id -> set of target phase IDs - # Also track which phases have incoming transitions - succ_phases = defaultdict[str, set[str]](set) - pred_phases = defaultdict[str, set[str]](set) - ending_phases = set[str]() - - for phase in phases: - if not phase.next: - ending_phases.add(phase.id) - else: - for transition in phase.next: - if transition.goto is None: - ending_phases.add(phase.id) - else: - if transition.goto not in phase_ids: - raise ValueError( - f'Phase {phase.id} has a transition that references non-existent phase {transition.goto}' - ) - succ_phases[phase.id].add(transition.goto) - pred_phases[transition.goto].add(phase.id) - - # Check that we have at least one ending phase - if not ending_phases: - raise ValueError( - 'Flow has no ending phases. Each conditional flow must have at least one ending phase. Any ending phase ' - 'has either no transitions at all or contains transition with goto: null referencing end of the flow.' - ) - - # Find entry phase (phase with no incoming transitions) - entry_phase = [pid for pid in phase_ids if not pred_phases[pid]] - - if not entry_phase: - raise ValueError( - 'Flow has no entry phase. Each conditional flow must have exactly one entry phase. An entry phase has no ' - 'incoming transitions; no transition from another phase leads to it.' - ) - if len(entry_phase) > 1: - raise ValueError( - f'Flow has multiple entry phases ({len(entry_phase)}): {entry_phase}. Each conditional flow must have ' - 'exactly one entry phase. Either merge the entry phases into one or redefine the transitions to form a ' - 'single entry phase.' - ) - - # All phases must be reachable from the entry point - _check_reachable_ids(entry_phase[0], succ_phases, phase_ids) - - # Check for circular dependencies - _check_circular_dependencies( - edges={phase_id: list(target_ids) for phase_id, target_ids in succ_phases.items()}, all_node_ids=phase_ids - ) - - -def _check_reachable_ids(start_id: str, edges: dict[str, set[str]], phase_ids: set[str]) -> None: - """ - Checks that all phases are reachable from a starting phase using DFS. - - :param start_id: The ID of the starting phase. - :param edges: Dictionary mapping phase IDs to sets of target phase IDs. - :param visited: Set of phase IDs that have been visited. - :param phase_ids: Set of all phase IDs in the flow. - :raises ValueError: If the flow has phases that are not reachable from the starting phase. - """ - - reachable_ids = _reachable_ids(start_id, edges, set[str]()) - if reachable_ids != phase_ids: - raise ValueError( - f'Flow has phases that are not reachable from the entry phase ({start_id}): ' - f'{phase_ids - reachable_ids}. All phases must be reachable from the entry phase by a valid path of ' - 'transitions.' - ) - - -def _reachable_ids(start_id: str, edges: dict[str, set[str]], visited: set[str]) -> set[str]: - """Find all phases reachable from a starting phase using DFS.""" - visited.add(start_id) - for target_id in edges.get(start_id, []): - if target_id not in visited: - visited.update(_reachable_ids(target_id, edges, visited)) - return visited diff --git a/src/keboola_mcp_server/tools/jobs.py b/src/keboola_mcp_server/tools/jobs.py deleted file mode 100644 index 08745b009..000000000 --- a/src/keboola_mcp_server/tools/jobs.py +++ /dev/null @@ -1,461 +0,0 @@ -import datetime -import logging -from typing import Annotated, Any, Literal, Optional, Sequence, Union - -from fastmcp import Context -from fastmcp.tools import FunctionTool -from mcp.types import ToolAnnotations -from pydantic import AliasChoices, BaseModel, Field, field_validator - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.links import Link, ProjectLinksManager -from keboola_mcp_server.mcp import KeboolaMcpServer, process_concurrently, toon_serializer_compact, unwrap_results - -LOG = logging.getLogger(__name__) - -JOB_TOOLS_TAG = 'jobs' - - -# Add jobs tools to MCP SERVER ################################## - - -def add_job_tools(mcp: KeboolaMcpServer) -> None: - """Add job tools to the MCP server.""" - mcp.add_tool( - FunctionTool.from_function( - get_jobs, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={JOB_TOOLS_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - run_job, - annotations=ToolAnnotations(destructiveHint=True), - tags={JOB_TOOLS_TAG}, - ) - ) - - LOG.info('Job tools added to the MCP server.') - - -# Job Base Models ######################################## - -JOB_STATUS = Literal[ - 'waiting', # job is waiting for other jobs to finish - 'processing', # job is being executed - 'success', # job finished successfully - 'error', # job finished with error - 'created', # job is created but not started executing - 'warning', # job finished but one of its child jobs failed - 'terminating', # user requested to abort the job - 'cancelled', # job was aborted before execution began - 'terminated', # job was aborted during execution -] - - -class JobListItem(BaseModel): - """Represents a summary of a job with minimal information, used in lists where detailed job data is not required.""" - - id: str = Field(description='The ID of the job.') - status: JOB_STATUS = Field(description='The status of the job.') - component_id: Optional[str] = Field( - description='The ID of the component that the job is running on.', - validation_alias=AliasChoices('componentId', 'component', 'component_id', 'component-id'), - serialization_alias='componentId', - default=None, - ) - config_id: Optional[str] = Field( - description='The ID of the component configuration that the job is running on.', - validation_alias=AliasChoices('configId', 'config', 'config_id', 'config-id'), - serialization_alias='configId', - default=None, - ) - is_finished: bool = Field( - description='Whether the job is finished.', - validation_alias=AliasChoices('isFinished', 'is_finished', 'is-finished'), - serialization_alias='isFinished', - default=False, - ) - created_time: Optional[datetime.datetime] = Field( - description='The creation time of the job.', - validation_alias=AliasChoices('createdTime', 'created_time', 'created-time'), - serialization_alias='createdTime', - default=None, - ) - start_time: Optional[datetime.datetime] = Field( - description='The start time of the job.', - validation_alias=AliasChoices('startTime', 'start_time', 'start-time'), - serialization_alias='startTime', - default=None, - ) - end_time: Optional[datetime.datetime] = Field( - description='The end time of the job.', - validation_alias=AliasChoices('endTime', 'end_time', 'end-time'), - serialization_alias='endTime', - default=None, - ) - duration_seconds: Optional[float] = Field( - description='The duration of the job in seconds.', - validation_alias=AliasChoices('durationSeconds', 'duration_seconds', 'duration-seconds'), - serialization_alias='durationSeconds', - default=None, - ) - - -class JobLogEvent(BaseModel): - """Represents a single log event from a job's execution.""" - - message: str = Field(description='The log message.') - type: str = Field(description='The event type: info, warn, error, or success.') - created: datetime.datetime = Field(description='When the event was created.') - - -class JobDetail(JobListItem): - """Represents a detailed job with all available information.""" - - url: str = Field(description='The URL of the job.') - - config_data: Optional[dict[str, Any]] = Field( - description='The data of the configuration.', - validation_alias=AliasChoices('configData', 'config_data', 'config-data'), - serialization_alias='configData', - default=None, - ) - config_row: Optional[str] = Field( - description='The configuration row ID.', - validation_alias=AliasChoices('configRow', 'config_row', 'config-row'), - serialization_alias='configRow', - default=None, - ) - run_id: Optional[str] = Field( - description='The ID of the run that the job is running on.', - validation_alias=AliasChoices('runId', 'run_id', 'run-id'), - serialization_alias='runId', - default=None, - ) - result: Optional[dict[str, Any]] = Field( - description='The results of the job.', - default=None, - ) - links: list[Link] = Field(..., description='The links relevant to the job.') - logs: Optional[list['JobLogEvent']] = Field( - description='Execution log events for the job, populated when include_logs=True.', - default=None, - ) - - @field_validator('result', 'config_data', mode='before') - @classmethod - def validate_dict_fields(cls, current_value: Union[list[Any], dict[str, Any], None]) -> dict[str, Any]: - # Ensures that if the result or config_data field is passed as an empty list [] or None, - # it gets converted to an empty dict {}.Why? Because the result is expected to be an Object, but create job - # endpoint sends [], perhaps it means "empty". This avoids type errors. - if not isinstance(current_value, dict): - if not current_value: - return dict() - if isinstance(current_value, list): - raise ValueError( - 'Field "result" or "config_data" cannot be a list, expecting dictionary, ' f'got: {current_value}.' - ) - return current_value - - -class GetJobsListOutput(BaseModel): - """Output of get_jobs tool when listing (no specific job_ids).""" - - jobs: list[JobListItem] = Field(..., description='List of jobs.') - links: list[Link] = Field(..., description='Links relevant to the jobs listing.') - - -class GetJobsDetailOutput(BaseModel): - """Output of get_jobs tool when retrieving specific job_ids.""" - - jobs: list[JobDetail] = Field(..., description='List of jobs with full details.') - - -GetJobsOutput = Union[GetJobsListOutput, GetJobsDetailOutput] - - -# End of Job Base Models ######################################## - -# MCP tools ######################################## - - -SORT_BY_VALUES = Literal['startTime', 'endTime', 'createdTime', 'durationSeconds', 'id'] -SORT_ORDER_VALUES = Literal['asc', 'desc'] - - -@tool_errors() -async def get_jobs( - ctx: Context, - job_ids: Annotated[ - Sequence[str], - Field( - description=( - 'IDs of jobs to retrieve full details for. ' - 'When provided (non-empty), returns full job details including status, parameters, ' - 'results, and metadata. ' - 'When empty [], lists jobs in the project as summaries with optional filtering.' - ) - ), - ] = tuple(), - status: Annotated[ - JOB_STATUS, - Field( - description=( - 'The optional status of the jobs to filter by when listing (ignored if job_ids is provided). ' - 'If None then all statuses are included.' - ), - ), - ] = None, - component_id: Annotated[ - str, - Field( - description=( - 'The optional ID of the component whose jobs you want to list ' - '(ignored if job_ids is provided). Default = None.' - ), - ), - ] = None, - config_id: Annotated[ - str, - Field( - description=( - 'The optional ID of the component configuration whose jobs you want to list ' - '(ignored if job_ids is provided). Default = None.' - ), - ), - ] = None, - limit: Annotated[ - int, - Field( - description=( - 'The number of jobs to list when listing (ignored if job_ids is provided), ' 'default = 100, max = 500.' - ), - ge=1, - le=500, - ), - ] = 100, - offset: Annotated[ - int, - Field( - description=('The offset of the jobs to list when listing (ignored if job_ids is provided), default = 0.'), - ge=0, - ), - ] = 0, - sort_by: Annotated[ - SORT_BY_VALUES, - Field( - description=( - 'The field to sort the jobs by when listing (ignored if job_ids is provided), ' 'default = "startTime".' - ), - ), - ] = 'startTime', - sort_order: Annotated[ - SORT_ORDER_VALUES, - Field( - description=( - 'The order to sort the jobs by when listing (ignored if job_ids is provided), default = "desc".' - ), - ), - ] = 'desc', - include_logs: Annotated[ - bool, - Field( - description=( - 'Whether to include execution logs for each job. Only used when job_ids is provided (MODE 1). ' - "Logs are fetched from the Storage API events endpoint using the job's runId. " - 'Default is False.' - ), - ), - ] = False, - log_tail_lines: Annotated[ - int, - Field( - description=( - 'Maximum number of log events to return per job (most recent first). ' - 'Only used when include_logs=True. Default = 50, max = 500.' - ), - ge=1, - le=500, - ), - ] = 50, - log_event_types: Annotated[ - Optional[Sequence[Literal['info', 'warn', 'error', 'success']]], - Field( - description=( - 'Filter log events by type. Only used when include_logs=True. ' - 'If None, all event types are included. ' - 'Example: ["error"] to only show errors, ["error", "warn"] for errors and warnings.' - ), - ), - ] = None, -) -> GetJobsOutput: - """ - Retrieves job execution information from the Keboola project. - - CONTEXT: - Jobs in Keboola are execution records of components (extractors, transformations, writers, flows). - Each job represents a single run with its status, timing, configuration, and results. - - TWO MODES OF OPERATION (controlled by job_ids parameter): - - MODE 1: GET DETAILS FOR SPECIFIC JOBS (job_ids is non-empty) - - Provide one or more job IDs: job_ids=["12345", "67890"] - - Returns: FULL details for each job including status, config_data, results, timing, and metadata - - Ignores: All filtering/sorting parameters (status, component_id, config_id, limit, offset, sort_by, sort_order) - - Use when: You know specific job IDs and need complete information about them - - MODE 2: LIST/SEARCH JOBS (job_ids is empty) - - Leave job_ids empty: job_ids=[] - - Returns: SUMMARY list of jobs (id, status, component_id, config_id, timing only - no config_data or results) - - Supports: Filtering by status/component_id/config_id, pagination with limit/offset, sorting - - Use when: You need to find jobs, see recent executions, or monitor job history - - DECISION GUIDE: - - Start with MODE 2 (list) to find jobs → then use MODE 1 (details) if you need full information - - If you already know job IDs → use MODE 1 directly - - For monitoring/browsing → use MODE 2 with filters - - NOTE: Jobs cannot be found by name using the `search` tool. However, always use the filtering - parameters (status, component_id, config_id) to narrow results rather than listing all jobs - with no filters. If you need to find jobs for a specific configuration but only know its name, - first use `search` to find the configuration ID, then filter jobs by that config_id. - - COMMON WORKFLOWS: - 1. Find failed jobs: job_ids=[], status="error" → identify problematic job IDs → get details with MODE 1 - 2. Check recent runs: job_ids=[], component_id="...", limit=10 → see latest executions - 3. Monitor specific job: job_ids=["123"] → poll for status and results - 4. Troubleshoot config: job_ids=[], component_id="...", config_id="...", status="error" → find which runs failed - - EXAMPLES: - - MODE 1 - Get full details: - - job_ids=["12345"] → detailed info for job 12345 - - job_ids=["12345", "67890"] → detailed info for multiple jobs - - MODE 2 - List/search jobs: - - job_ids=[] → list latest 100 jobs (default) - - job_ids=[], status="error" → list only failed jobs - - job_ids=[], status="processing" → list currently running jobs - - job_ids=[], component_id="keboola.ex-aws-s3" → list jobs for S3 extractor - - job_ids=[], component_id="keboola.ex-aws-s3", config_id="12345" → list jobs for specific configuration - - job_ids=[], limit=50, offset=100 → pagination (skip first 100, get next 50) - - job_ids=[], sort_by="endTime", sort_order="asc" → oldest completed first - - job_ids=[], sort_by="durationSeconds", sort_order="desc" → longest running first - - LOG RETRIEVAL (only in MODE 1): - - Set include_logs=True to fetch execution logs from the Storage API events - - Logs are fetched using the job's runId and returned in chronological order - - Use log_tail_lines to control how many recent log events to return (default 50, max 500) - - Use log_event_types to filter by event type: ["error"] for just errors, ["error", "warn"] for errors and warnings - - If a job has no runId (e.g., not yet started), logs will be None - - EXAMPLES WITH LOGS: - - job_ids=["12345"], include_logs=True → job details + last 50 log events - - job_ids=["12345"], include_logs=True, log_event_types=["error"] → job details + only error events - - job_ids=["12345"], include_logs=True, log_tail_lines=200 → job details + last 200 log events - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - # Case 1: job_ids provided - return full details for those jobs - if job_ids: - - async def fetch_job_detail(job_id: str) -> JobDetail: - raw_job = await client.jobs_queue_client.get_job_detail(job_id) - links = links_manager.get_job_links(job_id) - LOG.info(f'Found job details for {job_id}.' if raw_job else f'Job {job_id} not found.') - return JobDetail.model_validate(raw_job | {'links': links}) - - results = await process_concurrently(job_ids, fetch_job_detail) - jobs = unwrap_results(results, 'Failed to fetch one or more jobs') - - # Fetch logs if requested - if include_logs: - - async def fetch_logs_for_job(job: JobDetail) -> JobDetail: - if not job.id: - return job - raw_events = await client.storage_client.list_events( - job_id=job.id, - limit=log_tail_lines, - ) - # Filter by event type client-side if requested - if log_event_types: - type_set = set(log_event_types) - raw_events = [e for e in raw_events if e.get('type') in type_set] - # Events come newest-first from API; reverse to chronological order - raw_events.reverse() - job.logs = [JobLogEvent.model_validate(e) for e in raw_events] - return job - - log_results = await process_concurrently(jobs, fetch_logs_for_job) - jobs = unwrap_results(log_results, 'Failed to fetch logs for one or more jobs') - - LOG.info(f'Retrieved full details for {len(jobs)} jobs.') - return GetJobsDetailOutput(jobs=jobs) - - # Case 2: no job_ids - list jobs as summaries with optional filtering - _status = [status] if status else None - - raw_jobs = await client.jobs_queue_client.search_jobs_by( - component_id=component_id, - config_id=config_id, - limit=limit, - offset=offset, - status=_status, - sort_by=sort_by, - sort_order=sort_order, - ) - LOG.info(f'Found {len(raw_jobs)} jobs for limit {limit}, offset {offset}, status {status}.') - jobs = [JobListItem.model_validate(raw_job) for raw_job in raw_jobs] - links = [links_manager.get_jobs_dashboard_link()] - return GetJobsListOutput(jobs=jobs, links=links) - - -@tool_errors() -async def run_job( - ctx: Context, - component_id: Annotated[ - str, - Field(description='The ID of the component or transformation for which to start a job.'), - ], - configuration_id: Annotated[str, Field(description='The ID of the configuration for which to start a job.')], - configuration_row_ids: Annotated[ - list[str] | None, - Field( - default=None, - description='Optional list of configuration row IDs to run. If not provided, all rows are executed.', - ), - ] = None, -) -> JobDetail: - """ - Starts a new job for a given component or transformation. - """ - client = KeboolaClient.from_state(ctx.session.state) - - try: - raw_job = await client.jobs_queue_client.create_job( - component_id=component_id, - configuration_id=configuration_id, - configuration_row_ids=configuration_row_ids, - ) - links_manager = await ProjectLinksManager.from_client(client) - links = links_manager.get_job_links(str(raw_job['id'])) - job = JobDetail.model_validate(raw_job | {'links': links}) - LOG.info( - f'Started a new job with id: {job.id} for component {component_id} and configuration {configuration_id}.' - ) - return job - except Exception as exception: - LOG.exception( - f'Error when starting a new job for component {component_id} and configuration {configuration_id}: ' - f'{exception}' - ) - raise exception - - -# End of MCP tools ######################################## diff --git a/src/keboola_mcp_server/tools/oauth.py b/src/keboola_mcp_server/tools/oauth.py deleted file mode 100644 index 34bbdbebd..000000000 --- a/src/keboola_mcp_server/tools/oauth.py +++ /dev/null @@ -1,76 +0,0 @@ -"""OAuth URL generation tools for the MCP server.""" - -import logging -from typing import Annotated -from urllib.parse import urlencode, urlunsplit - -from fastmcp import Context -from fastmcp.tools import FunctionTool -from mcp.types import ToolAnnotations -from pydantic import Field - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.mcp import KeboolaMcpServer - -LOG = logging.getLogger(__name__) - -OAUTH_TOOLS_TAG = 'oauth' - - -def add_oauth_tools(mcp: KeboolaMcpServer) -> None: - """Adds OAuth tools to the MCP server.""" - mcp.add_tool( - FunctionTool.from_function( - create_oauth_url, - annotations=ToolAnnotations(destructiveHint=True), - tags={OAUTH_TOOLS_TAG}, - ) - ) - LOG.info('OAuth tools added to the MCP server.') - - -@tool_errors() -async def create_oauth_url( - component_id: Annotated[ - str, Field(description='The component ID to grant access to (e.g., "keboola.ex-google-analytics-v4").') - ], - config_id: Annotated[str, Field(description='The configuration ID for the component.')], - ctx: Context, -) -> Annotated[str, Field(description='The OAuth authorization URL.')]: - """ - Generates an OAuth authorization URL for a Keboola component configuration. - - When using this tool, be very concise in your response. Just guide the user to click the - authorization link. - - Note that this tool should be called specifically for the OAuth-requiring components after their - configuration is created e.g. keboola.ex-google-analytics-v4 and keboola.ex-gmail. - """ - client = KeboolaClient.from_state(ctx.session.state) - - # Create the token using the storage client - token_response = await client.storage_client.token_create( - description=f'Short-lived token for OAuth URL - {component_id}/{config_id}', - component_access=[component_id], - expires_in=3600, # 1 hour expiration - ) - - # Extract the token from response - sapi_token = token_response['token'] - - # Generate OAuth URL - query_params = urlencode({'token': sapi_token, 'sapiUrl': client.storage_api_url}) - fragment = f'/{component_id}/{config_id}' - - oauth_url = urlunsplit( - ( - 'https', # scheme - 'external.keboola.com', # netloc - '/oauth/index.html', # path - query_params, # query - fragment, # fragment - ) - ) - - return oauth_url diff --git a/src/keboola_mcp_server/tools/project.py b/src/keboola_mcp_server/tools/project.py deleted file mode 100644 index f9dc56a58..000000000 --- a/src/keboola_mcp_server/tools/project.py +++ /dev/null @@ -1,226 +0,0 @@ -import logging -from typing import Annotated, cast - -from fastmcp import Context, FastMCP -from fastmcp.tools import FunctionTool -from mcp.types import ToolAnnotations -from pydantic import BaseModel, Field - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.links import Link, ProjectLinksManager -from keboola_mcp_server.resources.prompts import get_project_system_prompt -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - -PROJECT_TOOLS_TAG = 'project' - - -def add_project_tools(mcp: FastMCP) -> None: - """Add project tools to the MCP server.""" - - LOG.info(f'Adding tool {get_project_info.__name__} to the MCP server.') - mcp.add_tool( - FunctionTool.from_function( - get_project_info, - annotations=ToolAnnotations(readOnlyHint=True), - tags={PROJECT_TOOLS_TAG}, - ) - ) - - LOG.info(f'Adding tool {update_project_description.__name__} to the MCP server.') - mcp.add_tool( - FunctionTool.from_function( - update_project_description, - annotations=ToolAnnotations(destructiveHint=True), - tags={PROJECT_TOOLS_TAG}, - ) - ) - - LOG.info('Project tools initialized.') - - -async def _resolve_branch_context(client: KeboolaClient) -> tuple[str | int, str, bool]: - """ - Resolves the current branch's id, name, and dev-branch flag from the storage API. - - `client.branch_id` is None on the default/production branch (normalized by - `KeboolaClient.with_branch_id`), so we look up the branches list and pick either - the entry matching the client's branch_id or the one with `isDefault=True`. - """ - target_branch_id = client.branch_id - branches = await client.storage_client.branches_list() - - selected: JsonDict | None = None - for branch in branches: - if target_branch_id is None: - if branch.get('isDefault') is True: - selected = branch - break - else: - if str(branch.get('id')) == str(target_branch_id): - selected = branch - break - - if selected is None: - # Should not happen in a healthy project, but stay defensive. - fallback_id: str | int = target_branch_id if target_branch_id is not None else 'default' - return fallback_id, 'unknown', target_branch_id is not None - - branch_id = cast( - str | int, - selected.get('id', target_branch_id if target_branch_id is not None else 'default'), - ) - branch_name = cast(str, selected.get('name', 'unknown')) - is_development_branch = selected.get('isDefault') is not True - return branch_id, branch_name, is_development_branch - - -def _get_toolset_restrictions(role: str) -> str | None: - """ - Returns a human-readable description of toolset restrictions for the given user role, - or None if no special restrictions apply. - """ - role = role.lower() - if role == 'readonly': - return ( - f'Your Keboola user role is "{role}". ' - 'Only read-only tools are available. ' - 'All write operations (creating, updating, or deleting resources) are disabled.' - ) - if not role or role == 'unknown': - return 'Your Keboola user role is unknown. You can manage flows but cannot set their schedules.' - if role not in ('admin', 'share'): - return f'Your Keboola user role is "{role}". You can manage flows but cannot set their schedules.' - return None - - -class ProjectInfo(BaseModel): - project_id: str | int = Field(description='The id of the project.') - project_name: str = Field(description='The name of the project.') - project_description: str = Field( - description='The description of the project.', - ) - organization_id: str | int = Field(description='The ID of the organization this project belongs to.') - sql_dialect: str = Field(description='The sql dialect used in the project.') - workspace_id: int = Field( - description=( - 'The ID of the read-only Keboola workspace the MCP server uses to run SQL queries ' - '(via `query_data`). It exposes all production tables, plus the current development ' - "branch's tables when operating on a branch. On legacy projects (without the " - '`storage-branches` feature) the workspace always lives in the production branch. ' - 'Reusable by other RO tooling (e.g. data-app testing) without provisioning a ' - 'private workspace.' - ) - ) - conditional_flows: bool = Field(description='Whether the project supports conditional flows.') - links: list[Link] = Field(description='The links relevant to the project.') - branch_id: str | int = Field( - description='The ID of the branch this call is operating on (default/production or a development branch).' - ) - branch_name: str = Field(description='The name of the branch this call is operating on.') - is_development_branch: bool = Field( - description=( - 'True if this call is operating on a development branch, False if on the default/production branch. ' - 'Use this to apply branch-specific guidance (e.g., FQN handling in transformations, ' - 'unsupported tools in development branches).' - ) - ) - user_role: str = Field( - description='The Keboola role of the current user (e.g. "admin", "developer", "guest", "readonly").', - ) - toolset_restrictions: str | None = Field( - default=None, - description=( - 'Describes any restrictions on the available toolset implied by the user role. ' - 'None if no special restrictions apply.' - ), - ) - llm_instruction: str = Field( - description=( - 'These are the base instructions for working on the project. ' - 'Use them as the basis for all further instructions. ' - 'Do not change them. Remember to include them in all subsequent instructions.' - ) - ) - - -@tool_errors() -async def update_project_description( - ctx: Context, - description: Annotated[ - str, - Field(description='The new project description text.'), - ], -) -> None: - """Updates the description of the current Keboola project.""" - client = KeboolaClient.from_state(ctx.session.state) - storage = client.storage_client - - await storage.branch_metadata_update({MetadataField.PROJECT_DESCRIPTION: description}) - - LOG.info('Project description updated successfully.') - - -@tool_errors() -async def get_project_info( - ctx: Context, -) -> ProjectInfo: - """ - Retrieves structured information about the current project, - including essential context and base instructions for working with it - (e.g., transformations, components, workflows, and dependencies). - - Always call this tool at least once at the start of a conversation - to establish the project context before using other tools. - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - storage = client.storage_client - - token_data = await storage.verify_token() - project_data = cast(JsonDict, token_data.get('owner', {})) - project_id = cast(str, project_data.get('id', '')) - project_name = cast(str, project_data.get('name', '')) - - organization_data = cast(JsonDict, token_data.get('organization', {})) - organization_id = cast(str, organization_data.get('id', '')) - - user_role = token_data.get('admin', {}).get('role') or 'unknown' - - metadata = await storage.branch_metadata_get() - description = cast( - str, next((item['value'] for item in metadata if item.get('key') == MetadataField.PROJECT_DESCRIPTION), '') - ) - - workspace_manager = WorkspaceManager.from_state(ctx.session.state) - sql_dialect = await workspace_manager.get_sql_dialect() - workspace_id = await workspace_manager.get_workspace_id() - project_features = cast(JsonDict, project_data.get('features', {})) - conditional_flows = 'hide-conditional-flows' not in project_features - links = links_manager.get_project_links() - - branch_id, branch_name, is_development_branch = await _resolve_branch_context(client) - - project_info = ProjectInfo( - project_id=project_id, - project_name=project_name, - project_description=description, - organization_id=organization_id, - sql_dialect=sql_dialect, - workspace_id=workspace_id, - conditional_flows=conditional_flows, - links=links, - branch_id=branch_id, - branch_name=branch_name, - is_development_branch=is_development_branch, - user_role=user_role, - toolset_restrictions=_get_toolset_restrictions(user_role), - llm_instruction=get_project_system_prompt(sql_dialect), - ) - - LOG.info('Returning unified project info.') - return project_info diff --git a/src/keboola_mcp_server/tools/search.py b/src/keboola_mcp_server/tools/search.py deleted file mode 100644 index 6fd81cec5..000000000 --- a/src/keboola_mcp_server/tools/search.py +++ /dev/null @@ -1,631 +0,0 @@ -import asyncio -import logging -from collections import defaultdict -from typing import Annotated, Any, AsyncGenerator, Sequence - -from fastmcp import Context, FastMCP -from fastmcp.exceptions import ToolError -from fastmcp.tools import FunctionTool -from mcp.types import ToolAnnotations -from pydantic import BaseModel, Field - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import ( - CONDITIONAL_FLOW_COMPONENT_ID, - DATA_APP_COMPONENT_ID, - ORCHESTRATOR_COMPONENT_ID, - KeboolaClient, - get_metadata_property, -) -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.links import Link, ProjectLinksManager -from keboola_mcp_server.mcp import toon_serializer_compact -from keboola_mcp_server.tools.components.utils import get_nested -from keboola_mcp_server.tools.search_global import _global_textual_search -from keboola_mcp_server.tools.search_models import ( - DEFAULT_GLOBAL_SEARCH_LIMIT, - GLOBAL_SEARCH_FEATURE, - MAX_GLOBAL_SEARCH_LIMIT, - WORKSPACE_COMPONENT_ID, - PatternMatch, - SearchComponentItemType, - SearchHit, - SearchItemType, - SearchOutput, - SearchPatternMode, - SearchSpec, - SearchType, -) -from keboola_mcp_server.tools.storage_helpers import merged_bucket_list, merged_bucket_table_list - -LOG = logging.getLogger(__name__) - -# Re-exported for backwards compatibility — models/aliases moved to search_models, the global-search -# path to search_global. Importers (server.py, generate_tool_docs, tools/storage/usage.py, tests) keep -# importing these names from `keboola_mcp_server.tools.search`. -__all__ = [ - 'SEARCH_TOOL_NAME', - 'SEARCH_TOOLS_TAG', - 'PatternMatch', - 'SearchComponentItemType', - 'SearchHit', - 'SearchItemType', - 'SearchOutput', - 'SearchSpec', - 'SuggestedComponentOutput', - 'add_search_tools', - 'fetch_configurations', - 'find_component_id', - 'search', -] - -SEARCH_TOOL_NAME = 'search' -SEARCH_TOOLS_TAG = 'search' - - -def add_search_tools(mcp: FastMCP) -> None: - """Add tools to the MCP server.""" - LOG.info(f'Adding tool {find_component_id.__name__} to the MCP server.') - mcp.add_tool( - FunctionTool.from_function( - find_component_id, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={SEARCH_TOOLS_TAG}, - ) - ) - - LOG.info(f'Adding tool {search.__name__} to the MCP server.') - mcp.add_tool( - FunctionTool.from_function( - search, - name=SEARCH_TOOL_NAME, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={SEARCH_TOOLS_TAG}, - ) - ) - - LOG.info('Search tools initialized.') - - -def _get_field_value(item: JsonDict, fields: Sequence[str]) -> Any | None: - for field in fields: - if value := get_nested(item, field): - return value - return None - - -def _check_column_match(table: JsonDict, cfg: SearchSpec) -> list[PatternMatch]: - """Check if any column name or description matches the patterns.""" - # Check column names (list of strings) - if col_names := table.get('columns', []): - if matched := cfg.match_texts(col_names): - return matched - - if col_metadata := table.get('columnMetadata', {}): - col_descs = (get_metadata_property(col_meta, MetadataField.DESCRIPTION) for col_meta in col_metadata.values()) - if matched := cfg.match_texts(filter(None, col_descs)): - return matched - return [] - - -async def _fetch_buckets(client: KeboolaClient, spec: SearchSpec) -> list[SearchHit]: - """Fetches and filters buckets.""" - hits = [] - for bucket in await merged_bucket_list(client): - if not (bucket_id := bucket.get('id')): - continue - - bucket_name = bucket.get('name') - bucket_display_name = bucket.get('displayName') - bucket_description = get_metadata_property(bucket.get('metadata', []), MetadataField.DESCRIPTION) - - if matches := spec.match_texts([bucket_id, bucket_name, bucket_display_name, bucket_description]): - hits.append( - SearchHit( - bucket_id=bucket_id, - item_type='bucket', - updated=_get_field_value(bucket, ['lastChangeDate', 'updated', 'created']) or '', - name=bucket_name, - display_name=bucket_display_name, - description=bucket_description, - ).set_matches(matches) - ) - return hits - - -async def _fetch_tables(client: KeboolaClient, spec: SearchSpec) -> list[SearchHit]: - """Fetches and filters tables from all buckets.""" - hits = [] - for bucket in await merged_bucket_list(client): - if not (bucket_id := bucket.get('id')): - continue - - tables = await merged_bucket_table_list(client, bucket_id, include=['columns', 'columnMetadata']) - for table in tables: - if not (table_id := table.get('id')): - continue - - table_name = table.get('name') - table_display_name = table.get('displayName') - table_description = get_metadata_property(table.get('metadata', []), MetadataField.DESCRIPTION) - - matches = spec.match_texts([table_id, table_name, table_display_name, table_description]) - matches.extend(_check_column_match(table, spec)) - if matches: - hits.append( - SearchHit( - table_id=table_id, - item_type='table', - updated=_get_field_value(table, ['lastChangeDate', 'created']) or '', - name=table_name, - display_name=table_display_name, - description=table_description, - ).set_matches(matches) - ) - return hits - - -async def fetch_configurations(client: KeboolaClient, spec: SearchSpec) -> list[SearchHit]: - """Fetches and filters configurations and configuration rows from all component types.""" - hits = [] - - if spec._component_types: - for component_type in spec._component_types: - async for hit in _fetch_configs(client, spec, component_type=component_type): - hits.append(hit) - - else: - async for hit in _fetch_configs(client, spec, component_type=None): - hits.append(hit) - - return hits - - -async def _fetch_configs( - client: KeboolaClient, spec: SearchSpec, component_type: str | None = None -) -> AsyncGenerator[SearchHit, None]: - components = await client.storage_client.component_list(component_type, include=['configuration', 'rows']) - - allowed_transformations = 'transformation' in spec.item_types or component_type is None - allowed_components = ( - 'configuration' in spec.item_types or 'configuration-row' in spec.item_types or component_type is None - ) - allowed_flows = 'flow' in spec.item_types or component_type is None - allowed_workspaces = 'workspace' in spec.item_types or component_type is None - allowed_data_apps = 'data-app' in spec.item_types or component_type is None - - for component in components: - if not (component_id := component.get('id')): - continue - - current_component_type = component.get('type') - if component_id in [ORCHESTRATOR_COMPONENT_ID, CONDITIONAL_FLOW_COMPONENT_ID]: - item_type: SearchItemType = 'flow' - if not allowed_flows: - continue - elif current_component_type == 'transformation': - item_type: SearchItemType = 'transformation' - if not allowed_transformations: - continue - elif component_id == WORKSPACE_COMPONENT_ID: - item_type: SearchItemType = 'workspace' - if not allowed_workspaces: - continue - elif component_id == DATA_APP_COMPONENT_ID: - item_type: SearchItemType = 'data-app' - if not allowed_data_apps: - continue - elif current_component_type in ['extractor', 'writer', 'application']: - item_type: SearchItemType = 'configuration' - if not allowed_components: - continue - else: - item_type: SearchItemType = 'configuration' - - for config in component.get('configurations', []): - if not (config_id := config.get('id')): - continue - - config_name = config.get('name') - config_description = config.get('description') - config_updated = _get_field_value(config, ['currentVersion.created', 'created']) or '' - - if spec.search_type == 'textual': - if matches := spec.match_texts([config_id, config_name, config_description]): - yield SearchHit( - component_id=component_id, - configuration_id=config_id, - item_type=item_type, - updated=config_updated, - name=config_name, - description=config_description, - ).set_matches(matches) - elif spec.search_type == 'config-based': - if matches := spec.match_configuration_scopes(config.get('configuration')): - yield SearchHit( - component_id=component_id, - configuration_id=config_id, - item_type=item_type, - updated=config_updated, - name=config_name, - description=config_description, - ).set_matches(matches) - - for row in config.get('rows', []): - if not (row_id := row.get('id')): - continue - - row_name = row.get('name') - row_description = row.get('description') - - if spec.search_type == 'textual': - if matches := spec.match_texts([row_id, row_name, row_description]): - yield SearchHit( - component_id=component_id, - configuration_id=config_id, - configuration_row_id=row_id, - item_type='configuration-row', - updated=config_updated or _get_field_value(row, ['created']), - name=row_name, - description=row_description, - ).set_matches(matches) - - elif spec.search_type == 'config-based': - if matches := spec.match_configuration_scopes(row.get('configuration')): - yield SearchHit( - component_id=component_id, - configuration_id=config_id, - configuration_row_id=row_id, - item_type='configuration-row', - updated=config_updated or _get_field_value(row, ['created']), - name=row_name, - description=row_description, - ).set_matches(matches) - - -@tool_errors() -async def search( - ctx: Context, - patterns: Annotated[ - list[str], - Field( - description='One or more search patterns. For textual search they match item names (server-side, ' - 'tokenized full-text); for config-based search they match the configuration JSON content. ' - 'Case-insensitive by default. Examples: ["customer"], ["sales", "revenue"], ["my_bucket"]. ' - 'Do not use empty strings or empty lists.' - ), - ], - item_types: Annotated[ - Sequence[SearchItemType], - Field( - description='Filter for specific Keboola item types. ' - 'Common values: "table" (data tables), "bucket" (table containers), "transformation" ' - '(SQL/Python transformations), "component" (extractor/writer/application components), ' - '"data-app" (data apps), "flow" (orchestration flows). ' - "Use when you know what type of item you're looking for or leave empty to search all types." - ), - ] = tuple(), - search_type: Annotated[ - SearchType, - Field( - description='Search mode: "textual" (name/id/description) or "config-based" (stringified configuration ' - 'payloads). (default: "textual")' - ), - ] = 'textual', - scopes: Annotated[ - Sequence[str], - Field( - description='JSONPath expressions to narrow config-based search to specific parts of the configuration. ' - 'Simple dot-notation (e.g. "parameters", "storage.input") and full JSONPath (e.g. "$.tasks[*]") are both ' - 'supported (e.g. "parameters.host", "storage.input[0].source"). ' - 'Leave empty to search the whole configuration.' - ), - ] = tuple(), - mode: Annotated[ - SearchPatternMode, - Field( - description='How to interpret patterns. Applies to config-based search only: "regex" for regular ' - 'expressions or "literal" for exact text (default: "literal"). Ignored by textual search, which is ' - 'always a tokenized full-text name query (not typo-corrected) and rejects "regex".' - ), - ] = 'literal', - limit: Annotated[ - int, - Field( - description=f'Maximum number of items to return (default: {DEFAULT_GLOBAL_SEARCH_LIMIT}, max: ' - f'{MAX_GLOBAL_SEARCH_LIMIT}).' - ), - ] = DEFAULT_GLOBAL_SEARCH_LIMIT, - offset: Annotated[int, Field(description='Number of matching items to skip for pagination (default: 0).')] = 0, -) -> SearchOutput: - """ - Searches for Keboola items (tables, buckets, components, configurations, transformations, flows, data-apps, etc.) - in the current project and returns matching ID + metadata. - - This tool supports two complementary search types: - - 1) textual - - Searches items by name, server-side (fast, independent of project size). - - Tokenized full-text name matching, case- and diacritics-insensitive. Pass the plain name; do NOT build - regex (rejected). It is NOT typo-corrected — misspellings may not match. - - Prefers the current branch context; when nothing is found there, automatically widens the search to all - branches of the project — such hits carry `branch_id`/`branch_name` so you can tell where they live. - - 2) config-based - - Searches item configurations (JSON objects) by matching patterns against the configuration values ​​converted - to a string, optionally narrowed by JSON path `scopes`. - - Returns also `match_scopes` with JSON paths and matched patterns per scope. - - THIS IS THE PRIMARY DISCOVERY TOOL. Always use it BEFORE any get_* tool when you need to find items - by name or specific configuration content. Do NOT enumerate items with get_buckets, get_tables, get_configs, - get_flows, or get_data_apps just to locate a specific item — use this tool instead. - - WHEN TO USE: - - User asks to "find", "locate", or "search for" something by name, keyword, text pattern, configuration content or - value - - User mentions a partial name and you need to find the full item (e.g., "find the customer table") - - User asks "what tables/configs/flows do I have with X in the name?" - - You need to discover items before performing operations on them - - User asks to "list all items with [name] or [configuration value/part] in it" - - User asks where a value, table, component, specific configuration ID, or specific settings is used in components, - data-apps, flows, or transformations - - You need to trace lineage by searching for IDs referenced in configurations, or to find flows using a - specific component, or find usage of a bucket/table in transformations or components, or to find items with - specific parameters. - - User asks to "what is the genesis of this item?" or "explain me business logic of this item?" - - HOW IT WORKS: - - Supports two types: - - search_type="textual": tokenized full-text name search, server-side. Names only — descriptions, column - names, IDs and configuration contents are NOT searched (use config-based search for configuration contents, - or get_tables for columns). Matching is case- and diacritics-insensitive but NOT typo-corrected. - - search_type="config-based": matches inside configuration JSON objects, optionally narrowed by JSON path `scopes` - - case-insensitive search - - mode for pattern search: applies to config-based only — `literal` (default) or `regex`. Textual search ignores - `mode` (always full-text) and rejects `regex`. - - Multiple patterns work as OR condition - matches items containing ANY of the patterns - - Each result includes the item's ID, name, creation date, and relevant metadata; the response also carries - `total` and `by_type` counts and the `branch_scope` the hits come from - - textual search prefers the current branch; on zero hits it automatically retries across all branches of the - project and marks the response with branch_scope="all-branches" - - scopes (config-based) narrow matching to specific JSONPath areas within configurations; matching is performed - against the stringified JSON node content in those areas. - - config-based always returns all matched paths per item in `match_scopes` (including matched patterns) - - IMPORTANT: - - Always use this tool when the user mentions a name but you don't have the exact ID - - The search returns IDs that you can use with other tools (e.g., get_tables, get_configs, get_flows) - - Results are ordered by the `updated` field, most recent first. `updated` is the item's last update time - when available, or its creation time otherwise (textual/global-search hits expose only the creation time). - - Textual search matches names only, with tokenized full-text matching (case/diacritics-insensitive; not - typo-corrected; no regex). It may not return every item the legacy enumeration did. To find items by - description or by table column, use get_tables; to find items by configuration content, use config-based search. - - For exact ID lookups, use specific tools like get_tables, get_configs, get_flows instead - - Use specific `scopes` only when you know the config structure (schema or real example); otherwise run config-based - search without scopes. - - Use find_component_id and get_configs tools to find configurations related to a specific component - - If results are too numerous or empty, ask the user to refine their query rather than enumerating all items. - - USAGE EXAMPLES: - 1) textual search examples: - - user_input: "Find all tables with 'customer' in the name" - → patterns=["customer"], item_types=["table"] - → Returns all tables whose name matches "customer" - - - user_input: "Search for the sales transformation" - → patterns=["sales"], item_types=["transformation"] - → Returns transformations with "sales" in the name - - - user_input: "Find items named 'daily report' or 'weekly summary'" - → patterns=["daily report", "weekly summary"], item_types=[] - → Returns all items matching any of these patterns - - - user_input: "Show me all configurations related to Google Analytics" - → patterns=["google analytics"], item_types=["configuration"] - → Returns configurations with matching names - - 2) config-based search examples: - - user_input: "Find transformations/configs/components referencing table in.c-prod.customers" - -> patterns=["in.c-prod.customers"], item_types=["transformation", "configuration"], - search_type="config-based" - -> No scopes = search whole stringified config; result includes `match_scopes` with exact paths + patterns - - - user_input: "Find configurations/transformations (etc.) using specific setting / id anywhere" - -> patterns=["setting", "id"], item_types=["configuration", "transformations"], search_type="config-based", - - - user_input: "Find configurations/transformations (etc.) using specific setting / id in parameters" - -> patterns=["setting", "id"], item_types=["configuration", "transformations"], search_type="config-based", - scopes=["parameters"] - - - user_input: "Find configurations/transformations (etc.) using specific setting / id in storage" - -> patterns=["setting", "id"], item_types=["configuration", "transformations"], search_type="config-based", - scopes=["storage"] - - - user_input: "Find configurations/transformations (etc.) using specific setting / id in authorization" - -> patterns=["setting", "id"], item_types=["configuration", "transformations"], search_type="config-based", - scopes=["parameters.authorization", "authorization"] - - - user_input: "Find components/transformations using my_bucket in input or output mappings" - -> patterns=["my_bucket"], item_types=["configuration", "transformation"], search_type="config-based", - scopes=["storage.input", "storage.output"] - -> Returns matches with paths like `storage.input.tables[0].source`, `storage.input.files[0].source`, - or `storage.output.tables[0].destination` - - - user_input: "Find flows using configuration ID 01k9cz233cvd1rga3zzx40g8qj" - -> patterns=["01k9cz233cvd1rga3zzx40g8qj"], item_types=["flow"], search_type="config-based", - scopes=["tasks", "phases"] - - - user_input: "Find transformations using this table / column / specific code in its script" - -> patterns=["element"], item_types=["transformation"], search_type="config-based", - scopes=["parameters", "storage"] - - - user_input: "Find data apps using something in its config / python code / setting" - -> patterns=["something"], item_types=["data-app"], search_type="config-based" - -> Returns data apps where script/config sections contain the keyword and includes `match_scopes` - """ - - spec = SearchSpec( - patterns=patterns, - item_types=item_types, - pattern_mode=mode, - search_type=search_type, - search_scopes=scopes, - return_all_matched_patterns=(search_type == 'config-based'), - ) - - offset = max(0, offset) - if not 0 < limit <= MAX_GLOBAL_SEARCH_LIMIT: - LOG.warning( - f'The "limit" parameter is out of range (0, {MAX_GLOBAL_SEARCH_LIMIT}], setting to default value ' - f'{DEFAULT_GLOBAL_SEARCH_LIMIT}.' - ) - limit = DEFAULT_GLOBAL_SEARCH_LIMIT - - client = KeboolaClient.from_state(ctx.session.state) - - if search_type == 'textual' and await client.storage_client.is_enabled(GLOBAL_SEARCH_FEATURE): - if mode == 'regex': - raise ToolError( - 'Regex patterns are not supported for textual search — it is a tokenized full-text name search. ' - 'Pass the plain name as the pattern, or use search_type="config-based" for regex matching inside ' - 'configurations.' - ) - # The global-search feature flag does not guarantee the project's index is populated (the bulk - # backfill is asynchronous) and the endpoint can fail transiently, so global search is a fast path - # with a safety net: fall back to client-side enumeration on any error, or when it finds nothing. - try: - output = await _global_textual_search(client, spec, limit=limit, offset=offset) - except Exception: - LOG.warning('Global search failed; falling back to client-side enumeration.', exc_info=True) - output = await _enumeration_search(client, spec, limit=limit, offset=offset) - else: - if not output.hits and offset == 0: - LOG.info('Global search returned no hits; falling back to client-side enumeration.') - output = await _enumeration_search(client, spec, limit=limit, offset=offset) - else: - # Projects without the global-search feature use the legacy client-side enumeration; - # config-based search has no server-side equivalent and always runs client-side. - output = await _enumeration_search(client, spec, limit=limit, offset=offset) - - # Get links for the hits - links_manager = await ProjectLinksManager.from_client(client) - for hit in output.hits: - hit.links.extend( - links_manager.get_links( - bucket_id=hit.bucket_id, - table_id=hit.table_id, - component_id=hit.component_id, - configuration_id=hit.configuration_id, - name=hit.name, - ) - ) - - return output - - -async def _enumeration_search(client: KeboolaClient, spec: SearchSpec, limit: int, offset: int) -> SearchOutput: - """ - Searches by enumerating the project's items client-side. Used for config-based search (which has no - server-side equivalent) and as the legacy fallback for textual search in projects without the - global-search feature. - """ - # Determine which types to fetch - types_to_fetch = set(spec.item_types) if spec.item_types else set() - - # Fetch items concurrently based on requested types - tasks = [] - all_hits: list[SearchHit] = [] - - if not types_to_fetch or 'bucket' in types_to_fetch: - tasks.append(_fetch_buckets(client, spec)) - - if not types_to_fetch or 'table' in types_to_fetch: - tasks.append(_fetch_tables(client, spec)) - - if not types_to_fetch: - tasks.append(fetch_configurations(client, spec)) - elif types_to_fetch & { - 'configuration', - 'transformation', - 'flow', - 'configuration-row', - 'workspace', - 'data-app', - }: - tasks.append(fetch_configurations(client, spec)) - - # Gather all results - results = await asyncio.gather(*tasks, return_exceptions=True) - - # Process results - for result in results: - if isinstance(result, Exception): - # TODO: report this somehow to the AI assistant - LOG.warning(f'Error fetching items: {result}') - continue - else: - all_hits.extend(result) - - # The configuration endpoint returns every config type at once, so narrow to the requested types to match - # the global-search path (e.g. item_types=['configuration-row'] must not leak 'configuration' hits). - if types_to_fetch: - all_hits = [hit for hit in all_hits if hit.item_type in types_to_fetch] - - # TODO: Should we sort by the item type too? - all_hits.sort( - key=lambda x: ( - x.updated, - x.bucket_id or x.table_id or x.component_id or x.configuration_id or x.configuration_row_id, - ), - reverse=True, - ) - - by_type: dict[str, int] = defaultdict(int) - for hit in all_hits: - by_type[hit.item_type] += 1 - - return SearchOutput( - hits=all_hits[offset : offset + limit], - total=len(all_hits), - by_type=dict(by_type), - branch_scope='current-branch', - ) - - -class SuggestedComponentOutput(BaseModel): - """Output of find_component_id tool.""" - - component_id: str = Field(description='The component ID.') - score: float = Field(description='Score of the component suggestion.') - links: list[Link] = Field(description='Links to the component.', default_factory=list) - - -@tool_errors() -async def find_component_id( - ctx: Context, - query: Annotated[str, Field(description='Natural language query to find the requested component.')], -) -> list[SuggestedComponentOutput]: - """ - Returns list of component IDs that match the given query. - - WHEN TO USE: - - Use when you want to find the component for a specific purpose. - - USAGE EXAMPLES: - - user_input: "I am looking for a salesforce extractor component" - → Returns a list of component IDs that match the query, ordered by relevance/best match. - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - suggestion_response = await client.ai_service_client.suggest_component(query) - - components = [] - for component in suggestion_response.components: - links = [links_manager.get_config_dashboard_link(component_id=component.component_id, component_name=None)] - components.append( - SuggestedComponentOutput(component_id=component.component_id, score=component.score, links=links) - ) - return components diff --git a/src/keboola_mcp_server/tools/search_global.py b/src/keboola_mcp_server/tools/search_global.py deleted file mode 100644 index 3ab7d61e4..000000000 --- a/src/keboola_mcp_server/tools/search_global.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Server-side global-search backed textual search for the `search` tool. - -Extracted from `search.py` to keep that module focused on the tool entry point and the -legacy client-side enumeration path. Depends only on `search_models` (shared models and -constants), so there is no import cycle with `search.py`. -""" - -import asyncio -import logging -from collections import defaultdict -from typing import Any, Literal, Sequence, cast - -from keboola_mcp_server.clients.client import ( - CONDITIONAL_FLOW_COMPONENT_ID, - DATA_APP_COMPONENT_ID, - ORCHESTRATOR_COMPONENT_ID, - KeboolaClient, -) -from keboola_mcp_server.clients.storage import GlobalSearchResponse -from keboola_mcp_server.clients.storage import ItemType as ApiItemType -from keboola_mcp_server.tools.search_models import ( - MAX_GLOBAL_SEARCH_LIMIT, - SEARCH_ITEM_TYPE_TO_API_TYPES, - WORKSPACE_COMPONENT_ID, - SearchHit, - SearchItemType, - SearchOutput, - SearchSpec, -) - -LOG = logging.getLogger(__name__) - - -def _api_types_for(item_types: Sequence[SearchItemType]) -> list[ApiItemType]: - """Maps the tool's item types to a deduplicated list of API types for the global-search endpoint.""" - api_types: list[ApiItemType] = [] - for item_type in item_types: - for api_type in SEARCH_ITEM_TYPE_TO_API_TYPES.get(item_type, ()): - if api_type not in api_types: - api_types.append(api_type) - return api_types - - -def _retype_configuration(component_id: str | None) -> SearchItemType: - """Maps a 'configuration' global-search item to the tool's more specific item type by its component.""" - if component_id in (ORCHESTRATOR_COMPONENT_ID, CONDITIONAL_FLOW_COMPONENT_ID): - return 'flow' - if component_id == DATA_APP_COMPONENT_ID: - return 'data-app' - if component_id == WORKSPACE_COMPONENT_ID: - return 'workspace' - return 'configuration' - - -def _global_search_hit(item: GlobalSearchResponse.Item) -> SearchHit | None: - """Maps a global-search item to a SearchHit; returns None for items that cannot be mapped.""" - common: dict[str, Any] = { - 'updated': item.created.isoformat(), - 'name': item.name, - 'branch_id': item.branch_id, - 'branch_name': item.branch_name, - } - - if item.type == 'bucket': - return SearchHit(bucket_id=item.id, item_type='bucket', **common) - - if item.type == 'table': - bucket = item.full_path.get('bucket') - bucket_id = str(bucket['id']) if isinstance(bucket, dict) and bucket.get('id') else None - return SearchHit(table_id=item.id, bucket_id=bucket_id, item_type='table', **common) - - if item.type in ('configuration-row', 'rows'): - configuration = item.full_path.get('configuration') - configuration_id = ( - str(configuration['id']) if isinstance(configuration, dict) and configuration.get('id') else None - ) - if not (item.component_id and configuration_id): - LOG.warning(f'Skipping global-search row hit with no parent configuration in fullPath: {item.id}') - return None - return SearchHit( - component_id=item.component_id, - configuration_id=configuration_id, - configuration_row_id=item.id, - item_type='configuration-row', - **common, - ) - - # The remaining types (configuration, transformation, flow, workspace, shared-code, state) are all - # configuration-like items whose id is the configuration ID. - component_id = item.component_id or (WORKSPACE_COMPONENT_ID if item.type == 'workspace' else None) - if not component_id: - LOG.warning(f'Skipping global-search hit with no component id: {item.type} {item.id}') - return None - item_type = _retype_configuration(component_id) if item.type == 'configuration' else cast(SearchItemType, item.type) - return SearchHit(component_id=component_id, configuration_id=item.id, item_type=item_type, **common) - - -async def _global_textual_search( - client: KeboolaClient, - spec: SearchSpec, - limit: int, - offset: int, -) -> SearchOutput: - """ - Searches item names server-side via the SAPI global-search endpoint, scoped to the current project. - - Runs one request per pattern (patterns are OR-ed, mirroring the legacy behavior) against the current - branch context first; when nothing is found, widens the search to the whole project (all branches). - """ - api_types = _api_types_for(spec.item_types) - # 'rows' hits are reported as 'configuration-row' and 'component' expands to configuration (rows); - # normalize the requested types accordingly for the client-side narrowing. - requested_types = {'configuration-row' if t == 'rows' else t for t in spec.item_types if t != 'component'} - - # The 'configuration' API type is lossy: a single server-side 'configuration' item may re-type to - # configuration/data-app/flow/workspace, so when the caller filters by type the server can fill a page - # with items that are dropped during client-side narrowing, under-filling the page. Over-fetch up to the - # server max in that case so the narrowed page is more likely to reach `limit`. Deep pagination (large - # offset) remains approximate because the server paginates in un-narrowed space. - needs_overfetch = bool(requested_types) and 'configuration' in api_types - fetch_limit = MAX_GLOBAL_SEARCH_LIMIT if needs_overfetch else limit - - async def query(branch_scope: Literal['current', 'all']) -> list[GlobalSearchResponse]: - return list( - await asyncio.gather( - *( - client.storage_client.global_search( - query=pattern, types=api_types, limit=fetch_limit, offset=offset, branch_scope=branch_scope - ) - for pattern in spec.patterns - ) - ) - ) - - def collect(responses: list[GlobalSearchResponse]) -> list[SearchHit]: - hits_by_key: dict[tuple[str, str], SearchHit] = {} - for response in responses: - for item in response.items: - if (hit := _global_search_hit(item)) is None: - continue - if requested_types and hit.item_type not in requested_types: - continue - hits_by_key.setdefault((item.type, item.id), hit) - return list(hits_by_key.values()) - - branch_scope: Literal['current', 'all'] = 'current' - responses = await query(branch_scope) - hits = collect(responses) - if not hits and offset == 0: - # Nothing in the current branch context — widen to the whole project so that items living - # in other branches can be discovered. Hits carry branch_id/branch_name for attribution. - branch_scope = 'all' - responses = await query(branch_scope) - hits = collect(responses) - - hits.sort( - key=lambda x: ( - x.updated, - x.bucket_id or x.table_id or x.component_id or x.configuration_id or x.configuration_row_id, - ), - reverse=True, - ) - - by_type: dict[str, int] = defaultdict(int) - for response in responses: - for type_name, count in response.by_type.items(): - by_type[type_name] += count - - return SearchOutput( - hits=hits[:limit], - total=sum(response.all for response in responses), - by_type=dict(by_type), - branch_scope='current-branch' if branch_scope == 'current' else 'all-branches', - ) diff --git a/src/keboola_mcp_server/tools/search_models.py b/src/keboola_mcp_server/tools/search_models.py deleted file mode 100644 index f27911cf3..000000000 --- a/src/keboola_mcp_server/tools/search_models.py +++ /dev/null @@ -1,372 +0,0 @@ -"""Shared models, type aliases and constants for the `search` tool. - -Extracted from `search.py` so the textual (global-search) and the legacy enumeration paths -can both depend on these without importing each other (avoids a circular import between -`search.py` and `search_global.py`). -""" - -import json -import logging -import re -from collections import defaultdict -from typing import Any, Iterable, Literal, Mapping, Sequence - -import jsonpath_ng -from jsonpath_ng.jsonpath import JSONPath -from pydantic import BaseModel, Field, PrivateAttr, model_validator - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.storage import ItemType as ApiItemType -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.components.utils import _normalize_jsonpath - -LOG = logging.getLogger(__name__) - -MAX_GLOBAL_SEARCH_LIMIT = 100 -DEFAULT_GLOBAL_SEARCH_LIMIT = 50 - -SearchItemType = Literal[ - 'bucket', - 'table', - 'data-app', - 'flow', - 'transformation', - 'component', - 'configuration', - 'configuration-row', - 'workspace', - 'shared-code', - 'rows', - 'state', -] - - -SearchComponentItemType = Literal[ - 'flow', - 'transformation', - 'component', - 'configuration', - 'configuration-row', - 'workspace', -] - - -SEARCH_ITEM_TYPE_TO_COMPONENT_TYPES: Mapping[SearchItemType, Sequence[str]] = { - 'data-app': ['other'], - 'flow': ['other'], - 'transformation': ['transformation'], - 'configuration': ['extractor', 'writer', 'application'], - 'configuration-row': ['extractor', 'writer', 'application'], - 'component': ['extractor', 'writer', 'application'], - 'workspace': ['other'], -} - -GLOBAL_SEARCH_FEATURE = 'global-search' -WORKSPACE_COMPONENT_ID = 'keboola.sandboxes' - -# Maps the tool's item types to the API types requested from the global-search endpoint. Some tool -# types (data-app, flow, workspace) exist server-side as 'configuration' items distinguished only -# by their component ID, so 'configuration' is over-fetched and narrowed client-side after re-typing. -SEARCH_ITEM_TYPE_TO_API_TYPES: Mapping[SearchItemType, Sequence[ApiItemType]] = { - 'bucket': ('bucket',), - 'table': ('table',), - 'transformation': ('transformation',), - 'configuration': ('configuration',), - 'configuration-row': ('configuration-row',), - 'component': ('configuration', 'configuration-row'), - 'flow': ('flow', 'configuration'), - 'data-app': ('configuration',), - 'workspace': ('workspace', 'configuration'), - 'shared-code': ('shared-code',), - 'rows': ('rows',), - 'state': ('state',), -} - -SearchType = Literal['textual', 'config-based'] -SearchPatternMode = Literal['regex', 'literal'] -SearchBranchScope = Literal['current-branch', 'all-branches'] - - -class PatternMatch(BaseModel): - scope: str | None - patterns: list[str] - - -class SearchHit(BaseModel): - bucket_id: str | None = Field(default=None, description='The ID of the bucket.') - table_id: str | None = Field(default=None, description='The ID of the table.') - component_id: str | None = Field(default=None, description='The ID of the component.') - configuration_id: str | None = Field(default=None, description='The ID of the configuration.') - configuration_row_id: str | None = Field(default=None, description='The ID of the configuration row.') - - item_type: SearchItemType = Field(description='The type of the item (e.g. table, bucket, configuration, etc.).') - updated: str = Field( - description='The date and time the item was last updated (or created, when the update time is not ' - 'available) in ISO 8601 format.' - ) - - name: str | None = Field(default=None, description='Name of the item.') - display_name: str | None = Field(default=None, description='Display name of the item.') - description: str | None = Field(default=None, description='Description of the item.') - branch_id: str | None = Field( - default=None, description='ID of the branch the item belongs to, when reported by the search backend.' - ) - branch_name: str | None = Field( - default=None, description='Name of the branch the item belongs to, when reported by the search backend.' - ) - matches: list[PatternMatch] = Field( - default_factory=list, - description='Most specific JSONPath scopes with grouped matched patterns (config-based search only).', - ) - links: list[Link] = Field(default_factory=list, description='Links to the item.') - - def __eq__(self, other: object) -> bool: - if isinstance(other, SearchHit): - return self.model_dump() == other.model_dump() - return False - - @model_validator(mode='after') - def check_id_fields(self) -> 'SearchHit': - id_fields = [ - self.bucket_id, - self.table_id, - self.component_id, - self.configuration_id, - self.configuration_row_id, - ] - - if not any(field for field in id_fields if field): - raise ValueError('At least one ID field must be filled.') - - if self.configuration_row_id and not all([self.component_id, self.configuration_id]): - raise ValueError( - 'If configuration_row_id is filled, ' 'both component_id and configuration_id must be filled.' - ) - - if self.configuration_id and not self.component_id: - raise ValueError('If configuration_id is filled, component_id must be filled.') - - return self - - def set_matches(self, matches: list['PatternMatch']) -> 'SearchHit': - """Assign pattern matches to this search hit and return self for chaining.""" - patterns_by_scope: dict[str, set[str]] = defaultdict(set) - for match in matches: - if not match.scope: - continue - patterns_by_scope[match.scope].update(match.patterns) - - unique_scopes = list(patterns_by_scope) - most_specific_scopes = [ - scope - for scope in unique_scopes - if not any( - other.startswith(scope) and len(other) > len(scope) and other[len(scope)] in ('.', '[') - for other in unique_scopes - ) - ] - self.matches = [ - PatternMatch(scope=scope, patterns=sorted(patterns_by_scope[scope])) for scope in most_specific_scopes - ] - return self - - -class SearchOutput(BaseModel): - """Paginated search results with total counts.""" - - hits: list[SearchHit] = Field(description='The matching items (paginated).') - total: int = Field( - description='Approximate total number of matching items before pagination; treat it as an upper bound on ' - 'the items reachable via pagination. With multiple patterns, an item matching more than one pattern is ' - 'counted once per pattern; for textual search the count may also include items later removed by client-side ' - 'type narrowing (e.g. configurations re-typed to data-apps/flows/workspaces).' - ) - by_type: dict[str, int] = Field( - default_factory=dict, - description='Number of matching items per item type (before pagination and client-side narrowing).', - ) - branch_scope: SearchBranchScope = Field( - default='current-branch', - description="Branch scope the hits come from. 'all-branches' means nothing was found in the current " - "branch context and the search was widened to the whole project; check each hit's branch_id/branch_name " - 'to see where it lives.', - ) - - -class SearchSpec(BaseModel): - patterns: Sequence[str] - item_types: Sequence[SearchItemType] - pattern_mode: SearchPatternMode = 'regex' - case_sensitive: bool = False - search_scopes: Sequence[str] = tuple() - search_type: SearchType = 'textual' - return_all_matched_patterns: bool = False - - _component_types: Sequence[str] = PrivateAttr(default_factory=tuple) - _compiled_patterns: list[re.Pattern] = PrivateAttr(default_factory=list) - _clean_patterns: list[str] = PrivateAttr(default_factory=list) - _all_nodes_expr: JSONPath | None = PrivateAttr(default=None) - # Tuple fields: (original_scope, parsed_scope_expr, parsed_descendants_expr) - _scope_exprs: list[tuple[str, JSONPath, JSONPath]] = PrivateAttr(default_factory=list) - - @model_validator(mode='after') - def _compile_patterns(self) -> 'SearchSpec': - cleaned_patterns = [str(item).strip() for item in self.patterns if item is not None and str(item).strip()] - if not cleaned_patterns: - raise ValueError('At least one search pattern must be provided.') - - self.patterns = cleaned_patterns - flags = 0 if self.case_sensitive else re.IGNORECASE - if self.pattern_mode == 'literal': - self._compiled_patterns = [re.compile(re.escape(pattern), flags) for pattern in cleaned_patterns] - else: - self._compiled_patterns = [re.compile(pattern, flags) for pattern in cleaned_patterns] - - self._clean_patterns = cleaned_patterns - return self - - @model_validator(mode='after') - def _validate_component_args(self) -> 'SearchSpec': - if not self._component_types: - self._component_types = list( - set( - component_type - for item in self.item_types - for component_type in SEARCH_ITEM_TYPE_TO_COMPONENT_TYPES.get(item, []) - ) - ) - return self - - @model_validator(mode='after') - def _validate_item_types(self) -> 'SearchSpec': - if 'component' in self.item_types: - self.item_types = list({*self.item_types, 'configuration', 'configuration-row'}) - return self - - @model_validator(mode='after') - def _compile_jsonpath_exprs(self) -> 'SearchSpec': - # Compile commonly used expressions once per SearchSpec instance. - self._all_nodes_expr = jsonpath_ng.parse('$..*') - self._scope_exprs = [] - for scope in self.search_scopes: - normalized = _normalize_jsonpath(scope if scope.startswith('$') else f'$.{scope}') - try: - self._scope_exprs.append((scope, jsonpath_ng.parse(normalized), jsonpath_ng.parse(f'{normalized}..*'))) - except Exception as e: - LOG.warning(f'Invalid JSONPath scope "{scope}": {e}') - return self - - @staticmethod - def _stringify(value: Any) -> str: - try: - return json.dumps(value, sort_keys=True, default=str, ensure_ascii=False) - except (TypeError, ValueError): - return str(value) - - def match_patterns(self, value: str | JsonDict | None) -> list[str]: - """ - Matches a string or dictionary value against the patterns. - - :param value: The value to match against the patterns. - :return: A list of patterns that matched the value; empty list if no matches. - """ - if value is None: - return [] - haystack = value if isinstance(value, str) else self._stringify(value) - if not haystack: - return [] - - matches: list[str] = [] - for pattern, compiled in zip(self._clean_patterns, self._compiled_patterns): - if compiled.search(haystack): - matches.append(pattern) - if not self.return_all_matched_patterns: - break - - return matches - - def _find_matches_for_expr( - self, configuration: JsonDict, parsed_expr: JSONPath, scalar_only: bool = False - ) -> list[PatternMatch]: - """Find pattern matches on JSON nodes matched by a JSONPath expression. If scalar_only is True, only scalar - nodes are matched.""" - matches: list[PatternMatch] = [] - for jpath_match in parsed_expr.find(configuration): - value = jpath_match.value - if scalar_only and isinstance(value, (dict, list)): - continue - if matched := self.match_patterns(value): - matches.append( - PatternMatch( - scope=_clean_jsonpath_path_str(str(jpath_match.full_path)), - patterns=matched, - ) - ) - if not self.return_all_matched_patterns: - return matches - return matches - - def match_configuration_scopes(self, configuration: JsonDict | None) -> list[PatternMatch]: - """ - Checks configuration fields within specified JSONPath scopes for pattern matches. - Walks matching nodes within each scope and returns the exact path where the match - was found. When no scopes are specified, walks the entire configuration. - - :param configuration: The configuration to match against the patterns. - :return: List of PatternMatch with matching JSONPath scopes; empty list if no matches. - """ - if configuration is None: - return [] - - if self.search_scopes: - all_matches: list[PatternMatch] = [] - # Deduplicate hits when scopes overlap (e.g. "parameters" + "parameters.query") - # or the same logical scope is provided multiple times. - seen: set[str | None] = set() - for _scope, self_expr, desc_expr in self._scope_exprs: - # Search in self expression node for scalar matches first - self_matches = self._find_matches_for_expr(configuration, self_expr, scalar_only=True) - # If no scalar matches, search in descendants nodes - desc_matches: list[PatternMatch] = [] - if not self_matches: - desc_matches = self._find_matches_for_expr(configuration, desc_expr) - for match in self_matches or desc_matches: - if match.scope in seen: - continue - seen.add(match.scope) - all_matches.append(match) - if not self.return_all_matched_patterns: - return all_matches - return all_matches - else: - # No scope provided – search all descendants and return exact match paths. - return self._find_matches_for_expr(configuration, self._all_nodes_expr) - - def match_texts(self, texts: Iterable[str]) -> list[PatternMatch]: - """ - Matches a sequence of strings against the patterns. - - :param texts: The sequence of strings to match against the patterns. - :return: A list of PatternMatch objects. - """ - matches: list[PatternMatch] = [] - for text in texts: - if matched := self.match_patterns(text): - matches.append(PatternMatch(scope=None, patterns=matched)) - if not self.return_all_matched_patterns: - break - return matches - - -def _clean_jsonpath_path_str(path_str: str) -> str: - """Normalize a jsonpath_ng full_path string across library versions. - - jsonpath_ng >= 1.8.0 wraps Child nodes in parentheses and single-quotes field names - with special characters, e.g. "(authorization.'#apiKey')" instead of "authorization.#apiKey". - """ - # Strip parentheses added by jsonpath_ng >= 1.8.0 - result = path_str.replace('(', '').replace(')', '') - # Remove surrounding quotes from field name segments, e.g. "'#apiKey'" -> "#apiKey" - result = re.sub(r"['\"]([^'\"]+)['\"]", r'\1', result) - # Normalize .[N] -> [N] - return re.sub(r'\.\[', '[', result) diff --git a/src/keboola_mcp_server/tools/semantic/__init__.py b/src/keboola_mcp_server/tools/semantic/__init__.py deleted file mode 100644 index 2eb7f78fb..000000000 --- a/src/keboola_mcp_server/tools/semantic/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from keboola_mcp_server.tools.constants import SEMANTIC_TOOLS_TAG -from keboola_mcp_server.tools.semantic.tools import add_semantic_tools - -__all__ = ['add_semantic_tools', 'SEMANTIC_TOOLS_TAG'] diff --git a/src/keboola_mcp_server/tools/semantic/model.py b/src/keboola_mcp_server/tools/semantic/model.py deleted file mode 100644 index bd556a9f4..000000000 --- a/src/keboola_mcp_server/tools/semantic/model.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Shared semantic tool models.""" - -from __future__ import annotations - -from enum import Enum -from typing import Any - -from pydantic import BaseModel, ConfigDict, Field - -# Shared input models - - -class SemanticObjectType(str, Enum): - SEMANTIC_MODEL = 'semantic-model' - SEMANTIC_DATASET = 'semantic-dataset' - SEMANTIC_METRIC = 'semantic-metric' - SEMANTIC_RELATIONSHIP = 'semantic-relationship' - SEMANTIC_GLOSSARY = 'semantic-glossary' - SEMANTIC_CONSTRAINT = 'semantic-constraint' - - def __str__(self) -> str: - return self.value - - -class SemanticObjectTypeSelection(BaseModel): - """Semantic object type selection used by semantic tools.""" - - object_type: SemanticObjectType = Field(description='Semantic object type to load.') - ids: tuple[str, ...] = Field( - default=tuple(), - description='Specific object UUIDs to include. Empty list [] means include all objects of this type.', - ) - - -class SemanticObjectRef(BaseModel): - """Typed semantic object reference.""" - - object_type: SemanticObjectType = Field(description='Semantic object type.') - id: str = Field(description='Semantic object UUID.') - - -class SemanticSchemaDefinition(BaseModel): - """Semantic schema definition returned by the semantic schema tool.""" - - model_config = ConfigDict(populate_by_name=True) - - semantic_type: SemanticObjectType = Field(description='Semantic object type.') - schema_definition: dict[str, Any] = Field( - validation_alias='schema', - serialization_alias='schema', - description='JSON schema for the semantic object type.', - ) diff --git a/src/keboola_mcp_server/tools/semantic/service.py b/src/keboola_mcp_server/tools/semantic/service.py deleted file mode 100644 index 47e306170..000000000 --- a/src/keboola_mcp_server/tools/semantic/service.py +++ /dev/null @@ -1,1059 +0,0 @@ -"""Semantic service layer shared by semantic read tools.""" - -from __future__ import annotations - -import json -import re -from collections.abc import Sequence - -import jsonpath_ng -from pydantic import BaseModel, Field - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.metastore import MetastoreObject -from keboola_mcp_server.mcp import process_concurrently, unwrap_results -from keboola_mcp_server.tools.semantic.model import SemanticObjectType - -SEMANTIC_OBJECT_TYPES: tuple[SemanticObjectType, ...] = ( - SemanticObjectType.SEMANTIC_MODEL, - SemanticObjectType.SEMANTIC_DATASET, - SemanticObjectType.SEMANTIC_METRIC, - SemanticObjectType.SEMANTIC_RELATIONSHIP, - SemanticObjectType.SEMANTIC_GLOSSARY, - SemanticObjectType.SEMANTIC_CONSTRAINT, -) -VALIDATION_OBJECT_TYPES: tuple[SemanticObjectType, ...] = ( - SemanticObjectType.SEMANTIC_MODEL, - SemanticObjectType.SEMANTIC_DATASET, - SemanticObjectType.SEMANTIC_METRIC, - SemanticObjectType.SEMANTIC_RELATIONSHIP, - SemanticObjectType.SEMANTIC_CONSTRAINT, -) - -# Some metastore endpoints return 500 for large responses unless paged aggressively. -DEFAULT_PAGE_LIMIT = 20 -DEFAULT_PAGE_LIMITS: dict[SemanticObjectType, int] = { - SemanticObjectType.SEMANTIC_DATASET: 1, - SemanticObjectType.SEMANTIC_METRIC: 5, -} - -ALL_ATTRIBUTE_NODES_EXPR = jsonpath_ng.parse('$..*') -POST_QUERY_CONSTRAINT_TYPES = {'inequality', 'equality', 'range', 'temporal', 'conditional'} - -# Regex that captures the single column name from a simple aggregate metric SQL expression, -# e.g. SUM("REVENUE_YTD") → "REVENUE_YTD", AVG(margin_pct) → "margin_pct". -# Complex expressions (CASE, arithmetic, multi-arg) do not match and return None. -_AGGREGATE_COLUMN_RE = re.compile(r'^\s*\w+\s*\(\s*"?([A-Za-z_][A-Za-z0-9_]*)\"?\s*\)\s*$') - -# SQL function names and keywords that should never be treated as column identifiers when -# parsing relationship ON clauses. Upper-case only because _extract_join_columns only -# looks at uppercase tokens. -_SQL_KEYWORDS_UPPER = frozenset( - { - 'AND', - 'OR', - 'NOT', - 'IN', - 'IS', - 'NULL', - 'TRUE', - 'FALSE', - 'LEFT', - 'RIGHT', - 'INNER', - 'OUTER', - 'FULL', - 'CROSS', - 'JOIN', - 'ON', - 'WHERE', - 'SELECT', - 'FROM', - 'AS', - 'BY', - 'GROUP', - 'AVG', - 'SUM', - 'COUNT', - 'MIN', - 'MAX', - 'COALESCE', - 'NULLIF', - 'CAST', - 'CONCAT', - 'TRIM', - 'LENGTH', - 'UPPER', - 'LOWER', - 'IFF', - 'CASE', - 'WHEN', - 'THEN', - 'ELSE', - 'END', - } -) - - -class SemanticTypeData(BaseModel): - """Minimal typed semantic object used by the service layer.""" - - semantic_type: SemanticObjectType = Field(description='Semantic object type.') - id: str = Field(description='Semantic object UUID.') - data: MetastoreObject = Field(description='Raw metastore object backing this typed service model.') - - @property - def display_name(self) -> str | None: - name = getattr(self, 'name', None) - if isinstance(name, str) and name: - return name - return getattr(self.data.meta, 'name', None) or None - - -class SemanticModelData(SemanticTypeData): - name: str | None = None - description: str | None = None - sql_dialect: str | None = None - - @classmethod - def from_metastore(cls, obj: MetastoreObject) -> 'SemanticModelData': - attributes = obj.attributes or {} - return cls( - semantic_type=SemanticObjectType.SEMANTIC_MODEL, - id=obj.id, - data=obj, - name=attributes.get('name') or getattr(obj.meta, 'name', None), - description=attributes.get('description'), - sql_dialect=attributes.get('sql_dialect'), - ) - - -class SemanticDatasetData(SemanticTypeData): - name: str | None = None - table_id: str | None = None - fqn: str | None = None - description: str | None = None - model_uuid: str | None = None - - @classmethod - def from_metastore(cls, obj: MetastoreObject) -> 'SemanticDatasetData': - attributes = obj.attributes or {} - return cls( - semantic_type=SemanticObjectType.SEMANTIC_DATASET, - id=obj.id, - data=obj, - name=attributes.get('name') or getattr(obj.meta, 'name', None), - table_id=attributes.get('tableId'), - fqn=attributes.get('fqn'), - description=attributes.get('description'), - model_uuid=attributes.get('modelUUID'), - ) - - -class SemanticMetricData(SemanticTypeData): - name: str | None = None - sql: str | None = None - dataset: str | None = None - description: str | None = None - model_uuid: str | None = None - - @classmethod - def from_metastore(cls, obj: MetastoreObject) -> 'SemanticMetricData': - attributes = obj.attributes or {} - return cls( - semantic_type=SemanticObjectType.SEMANTIC_METRIC, - id=obj.id, - data=obj, - name=attributes.get('name') or getattr(obj.meta, 'name', None), - sql=attributes.get('sql'), - dataset=attributes.get('dataset'), - description=attributes.get('description'), - model_uuid=attributes.get('modelUUID'), - ) - - -class SemanticRelationshipData(SemanticTypeData): - name: str | None = None - from_dataset: str | None = None - to_dataset: str | None = None - relationship_type: str | None = None - on: str | None = None - model_uuid: str | None = None - - @classmethod - def from_metastore(cls, obj: MetastoreObject) -> 'SemanticRelationshipData': - attributes = obj.attributes or {} - return cls( - semantic_type=SemanticObjectType.SEMANTIC_RELATIONSHIP, - id=obj.id, - data=obj, - name=attributes.get('name') or getattr(obj.meta, 'name', None), - from_dataset=attributes.get('from'), - to_dataset=attributes.get('to'), - relationship_type=attributes.get('type'), - on=attributes.get('on'), - model_uuid=attributes.get('modelUUID'), - ) - - -class SemanticGlossaryData(SemanticTypeData): - term: str | None = None - definition: str | None = None - model_uuid: str | None = None - - @classmethod - def from_metastore(cls, obj: MetastoreObject) -> 'SemanticGlossaryData': - attributes = obj.attributes or {} - return cls( - semantic_type=SemanticObjectType.SEMANTIC_GLOSSARY, - id=obj.id, - data=obj, - term=attributes.get('term'), - definition=attributes.get('definition'), - model_uuid=attributes.get('modelUUID'), - ) - - @property - def display_name(self) -> str | None: - return self.term or super().display_name - - -class SemanticConstraintData(SemanticTypeData): - name: str | None = None - description: str | None = None - constraint_type: str | None = None - severity: str | None = None - rule: str | None = None - metrics: tuple[str, ...] = () - datasets: tuple[str, ...] = () - model_uuid: str | None = None - error_message: str | None = None - remediation: str | None = None - pre_query_check: bool = False - validation_query: dict[str, str] | None = None - - @classmethod - def from_metastore(cls, obj: MetastoreObject) -> 'SemanticConstraintData': - attributes = obj.attributes or {} - ai = attributes.get('ai') - validation_query = attributes.get('validationQuery') - return cls( - semantic_type=SemanticObjectType.SEMANTIC_CONSTRAINT, - id=obj.id, - data=obj, - name=attributes.get('name') or getattr(obj.meta, 'name', None), - description=attributes.get('description'), - constraint_type=attributes.get('constraintType'), - severity=attributes.get('severity'), - rule=attributes.get('rule'), - metrics=tuple(metric for metric in attributes.get('metrics', []) if isinstance(metric, str) and metric), - datasets=tuple( - dataset for dataset in attributes.get('datasets', []) if isinstance(dataset, str) and dataset - ), - model_uuid=attributes.get('modelUUID'), - error_message=attributes.get('errorMessage'), - remediation=attributes.get('remediation'), - pre_query_check=isinstance(ai, dict) and ai.get('preQueryCheck') is True, - validation_query=validation_query if isinstance(validation_query, dict) else None, - ) - - -SemanticServiceData = ( - SemanticModelData - | SemanticDatasetData - | SemanticMetricData - | SemanticRelationshipData - | SemanticGlossaryData - | SemanticConstraintData -) - - -class SemanticServiceDataTypeGroup(BaseModel): - """Semantic service objects grouped by semantic object type.""" - - object_type: SemanticObjectType = Field(description='Semantic object type.') - objects: list[SemanticServiceData] = Field( - default_factory=list, - description='Typed semantic objects of the requested type.', - ) - - -class SemanticSearchHit(BaseModel): - """Raw semantic search hit returned by the service layer.""" - - object_type: SemanticObjectType = Field(description='Matched semantic object type.') - object: SemanticServiceData = Field(description='Matched semantic object.') - semantic_model_id: str = Field(description='Parent semantic model UUID.') - matched_patterns: list[str] = Field(default_factory=list, description='Regex patterns that matched.') - matched_paths: list[str] = Field(default_factory=list, description='Search sources where the match happened.') - - -class ConstraintValidationFinding(BaseModel): - """Tool-facing semantic constraint finding.""" - - constraint_id: str = Field(description='Constraint UUID.') - constraint_name: str = Field(description='Constraint name.') - severity: str = Field(description='Constraint severity.') - status: str = Field(description='Validation status.') - message: str = Field(description='Human-readable validation finding.') - validation_query: str | None = Field( - default=None, - description='Optional SQL validation query suggested by the semantic constraint.', - ) - - -class SemanticValidationServiceOutput(BaseModel): - """Output for semantic SQL validation.""" - - valid: bool = Field(description='False when an error-severity pre-execution finding was detected.') - used_object_groups: list['SemanticServiceDataTypeGroup'] = Field( - default_factory=list, - description='Used semantic objects grouped by semantic object type.', - ) - matched_relationships: list[str] = Field( - default_factory=list, - description='Relationship names heuristically detected in the SQL.', - ) - violations: list[ConstraintValidationFinding] = Field( - default_factory=list, - description='Pre-execution semantic violations.', - ) - post_execution_checks: list[ConstraintValidationFinding] = Field( - default_factory=list, - description='Checks that should be verified against query results.', - ) - - -def _to_semantic_service_data(object_type: SemanticObjectType, obj: MetastoreObject) -> SemanticServiceData: - if object_type == SemanticObjectType.SEMANTIC_MODEL: - return SemanticModelData.from_metastore(obj) - if object_type == SemanticObjectType.SEMANTIC_DATASET: - return SemanticDatasetData.from_metastore(obj) - if object_type == SemanticObjectType.SEMANTIC_METRIC: - return SemanticMetricData.from_metastore(obj) - if object_type == SemanticObjectType.SEMANTIC_RELATIONSHIP: - return SemanticRelationshipData.from_metastore(obj) - if object_type == SemanticObjectType.SEMANTIC_GLOSSARY: - return SemanticGlossaryData.from_metastore(obj) - if object_type == SemanticObjectType.SEMANTIC_CONSTRAINT: - return SemanticConstraintData.from_metastore(obj) - raise ValueError(f'Unsupported semantic object type "{object_type.value}".') - - -def _get_semantic_model_id(obj: SemanticTypeData | MetastoreObject) -> str: - if isinstance(obj, SemanticTypeData): - if isinstance(obj, SemanticModelData): - return obj.id - else: - return obj.model_uuid or '' - elif isinstance(obj, MetastoreObject): - if obj.type == SemanticObjectType.SEMANTIC_MODEL.value: - return obj.id - else: - model_id = (obj.attributes or {}).get('modelUUID') - return str(model_id) if model_id else '' - raise ValueError(f'Unsupported object type "{type(obj)}".') - - -def _clean_jsonpath_path_str(path_str: str) -> str: - """Normalize a jsonpath_ng full_path string across library versions.""" - result = path_str.replace('(', '').replace(')', '') - result = re.sub(r"['\"]([^'\"]+)['\"]", r'\1', result) - return re.sub(r'\.\[', '[', result) - - -def _stringify_value(value: object) -> str: - if isinstance(value, str): - return value - try: - return json.dumps(value, sort_keys=True, default=str, ensure_ascii=False) - except (TypeError, ValueError): - return str(value) - - -def _find_matches( - semantic_object: SemanticServiceData, - compiled_patterns: Sequence[re.Pattern[str]], - cleaned_patterns: Sequence[str], -) -> tuple[list[str], list[str]]: - matched_paths: set[str] = set() - matched_patterns: set[str] = set() - - if semantic_object.display_name: - for pattern, compiled in zip(cleaned_patterns, compiled_patterns, strict=False): - if compiled.search(semantic_object.display_name): - matched_paths.add('meta.name') - matched_patterns.add(pattern) - - attrs = semantic_object.data.attributes or {} - if any(compiled.search(_stringify_value(attrs)) for compiled in compiled_patterns): - for jpath_match in ALL_ATTRIBUTE_NODES_EXPR.find(attrs): - value = jpath_match.value - if isinstance(value, (dict, list)): - continue - - haystack = _stringify_value(value) - if not haystack: - continue - - path = _clean_jsonpath_path_str(str(jpath_match.full_path)) - for pattern, compiled in zip(cleaned_patterns, compiled_patterns, strict=False): - if compiled.search(haystack): - matched_paths.add(path) - matched_patterns.add(pattern) - - return sorted(matched_paths), sorted(matched_patterns) - - -async def _list_semantic_type_objects( - client: KeboolaClient, - object_type: SemanticObjectType, - semantic_model_ids: Sequence[str] | None = None, -) -> list[SemanticServiceData]: - """List all semantic objects of a given type, optionally filtered by a set of semantic model IDs.""" - metastore = client.metastore_client - limit = DEFAULT_PAGE_LIMITS.get(object_type, DEFAULT_PAGE_LIMIT) - offset = 0 - data: list[SemanticServiceData] = [] - model_id_set = set(semantic_model_ids) if semantic_model_ids else None - - while True: - page = await metastore.list_objects(object_type, limit=limit, offset=offset) - data.extend( - _to_semantic_service_data(object_type, obj) - for obj in page - if model_id_set is None or _get_semantic_model_id(obj) in model_id_set - ) - if len(page) < limit: - return data - offset += limit - - -def _matches_sql(sql_query: str, candidate: str) -> bool: - if not candidate: - return False - - candidate_lower = candidate.lower() - sql_lower = sql_query.lower() - if re.fullmatch(r'[a-zA-Z_][a-zA-Z0-9_]*', candidate): - pattern = rf'(? str | None: - validation_query = constraint.validation_query - if validation_query is None: - return None - - dialect_key = (sql_dialect or '').strip().lower() - if dialect_key == 'snowflake' and isinstance(validation_query.get('snowflake'), str): - return validation_query['snowflake'] - if dialect_key == 'bigquery' and isinstance(validation_query.get('bigquery'), str): - return validation_query['bigquery'] - - default_query = validation_query.get('default') - return default_query if isinstance(default_query, str) else None - - -def _constraint_message(constraint: SemanticConstraintData, default_message: str) -> str: - if constraint.error_message and constraint.error_message.strip(): - if constraint.remediation and constraint.remediation.strip(): - return f'{constraint.error_message.strip()} Remediation: {constraint.remediation.strip()}' - return constraint.error_message.strip() - if constraint.remediation and constraint.remediation.strip(): - return f'{default_message} Remediation: {constraint.remediation.strip()}' - return default_message - - -def _dataset_identifiers(dataset: SemanticDatasetData) -> list[str]: - candidates = [dataset.fqn] - return [str(candidate).strip() for candidate in candidates if isinstance(candidate, str) and candidate.strip()] - - -def _extract_metric_column(sql: str) -> str | None: - """Extract the bare column name from a simple aggregate metric SQL expression. - - Handles forms like ``SUM("REVENUE_YTD")``, ``AVG(margin_pct)``, ``SUM(AMOUNT)``. - Returns *None* for complex expressions (CASE, arithmetic, multi-argument, ``COUNT(*)``). - - The extracted column is added as a secondary match candidate so that the metric is - detected even when the column appears with a table-alias prefix in the SQL query - (e.g. ``ep."REVENUE_YTD"``), which would otherwise defeat full-string matching. - """ - m = _AGGREGATE_COLUMN_RE.match(sql) - return m.group(1) if m else None - - -def _metric_identifiers(metric: SemanticMetricData) -> list[str]: - candidates: list[str] = [] - if metric.sql: - candidates.append(metric.sql) - # Also add the bare column name so that `SUM("REVENUE_YTD")` matches even when - # the SQL writes `SUM(ep."REVENUE_YTD")` — the alias prefix breaks substring - # matching but word-boundary matching on the column name still works. - col = _extract_metric_column(metric.sql) - if col: - candidates.append(col) - return [c.strip() for c in candidates if c.strip()] - - -def _detect_used_datasets(sql_query: str, datasets: Sequence[SemanticDatasetData]) -> list[SemanticDatasetData]: - return [ - dataset - for dataset in datasets - if any(_matches_sql(sql_query, candidate) for candidate in _dataset_identifiers(dataset)) - ] - - -def _detect_used_metrics_for_datasets( - sql_query: str, - metrics: Sequence[SemanticMetricData], - used_dataset_ids: set[str], -) -> list[SemanticMetricData]: - matches: list[SemanticMetricData] = [] - for metric in metrics: - # Metric SQL snippets such as SUM("AMOUNT") are often reused across different datasets. - # We therefore only accept a metric match when its source dataset was already detected - # in the query; otherwise the metric match would be too noisy. - if metric.dataset is None or metric.dataset not in used_dataset_ids: - continue - if any(_matches_sql(sql_query, candidate) for candidate in _metric_identifiers(metric)): - matches.append(metric) - return matches - - -def _extract_join_columns(on_clause: str) -> list[str]: - """Extract bare column identifiers from a relationship ON clause. - - Strategy: - 1. Strip single-quoted string literals so constant values like ``'AVG'`` or ``'USD'`` - are not mistaken for column names. - 2. Find all uppercase identifiers of three or more characters (the convention used in - Snowflake/BigQuery schemas for column names), de-duplicated and in order of appearance. - 3. Drop known SQL function names and keywords from ``_SQL_KEYWORDS_UPPER``. - - Returns an empty list when no uppercase identifiers are found — this happens for - all-lowercase on-clauses (test fixtures, BigQuery style), which signals the caller to - fall back to the original full-string match. - """ - cleaned = re.sub(r"'[^']*'", '', on_clause) - tokens = re.findall(r'\b([A-Z][A-Z0-9_]{2,})\b', cleaned) - seen: set[str] = set() - result: list[str] = [] - for token in tokens: - if token not in _SQL_KEYWORDS_UPPER and token not in seen: - seen.add(token) - result.append(token) - return result - - -def _detect_used_relationships( - sql_query: str, - relationships: Sequence[SemanticRelationshipData], - used_dataset_ids: set[str], -) -> list[SemanticRelationshipData]: - matches: list[SemanticRelationshipData] = [] - for relationship in relationships: - if relationship.from_dataset is None or relationship.to_dataset is None: - continue - # Relationships are only considered when both datasets were already detected. - # This keeps relationship matching conservative and avoids claiming a join path just - # because an "on" fragment happens to appear in an unrelated query. - if relationship.from_dataset not in used_dataset_ids or relationship.to_dataset not in used_dataset_ids: - continue - if relationship.on and relationship.on.strip(): - col_names = _extract_join_columns(relationship.on) - if col_names: - # Column-based matching: require ALL column names from the ON clause to - # appear in the SQL. Word-boundary matching handles quoted identifiers - # (e.g. "FK_COL") and tolerates different table-alias conventions - # (fact./dim. in the definition vs. bsu./coa. in the actual query). - if not all(_matches_sql(sql_query, col) for col in col_names): - continue - else: - # No uppercase columns found (all-lowercase on-clause): fall back to - # the original full-string match to preserve existing behaviour. - if not _matches_sql(sql_query, relationship.on): - continue - matches.append(relationship) - return matches - - -def _constraint_is_relevant( - constraint: SemanticConstraintData, - used_metric_names: set[str], - used_dataset_ids: set[str], -) -> bool: - constraint_metrics = {metric.strip() for metric in constraint.metrics if metric.strip()} - constraint_datasets = {dataset.strip() for dataset in constraint.datasets if dataset.strip()} - # Check if the constraint references any used metrics or datasets. - # If it does, the constraint is relevant. - if constraint_metrics and used_metric_names.intersection(constraint_metrics): - return True - if constraint_datasets and used_dataset_ids.intersection(constraint_datasets): - return True - # Scope-less constraints are currently treated as model-global constraints. This is a - # pragmatic default so such constraints are not silently ignored, but it may over-match - # if the semantic model contains broad or underspecified rules. - return not constraint_metrics and not constraint_datasets - - -async def search_semantic_context( - client: KeboolaClient, - patterns: Sequence[str], - *, - semantic_types: Sequence[SemanticObjectType] = tuple(), - semantic_model_ids: Sequence[str] | None = None, - case_sensitive: bool = False, - max_results: int = 50, -) -> list[SemanticSearchHit]: - """Search semantic objects by regex patterns for selected semantic object types.""" - cleaned_patterns = [pattern.strip() for pattern in patterns if pattern and pattern.strip()] - if not cleaned_patterns: - raise ValueError('At least one regex pattern must be provided.') - if max_results <= 0: - raise ValueError('max_results must be a positive integer.') - - target_types = tuple(semantic_types) if semantic_types else SEMANTIC_OBJECT_TYPES - flags = 0 if case_sensitive else re.IGNORECASE - compiled_patterns: list[re.Pattern[str]] = [] - for pattern in cleaned_patterns: - try: - compiled_patterns.append(re.compile(pattern, flags)) - except re.error as e: - raise ValueError(f'Invalid regex pattern "{pattern}": {e}') from e - - matches: list[SemanticSearchHit] = [] - for object_type in target_types: - if len(matches) >= max_results: - break - - objects = await _list_semantic_type_objects(client, object_type, semantic_model_ids) - for semantic_object in objects: - if len(matches) >= max_results: - break - - field_hits, pattern_hits = _find_matches(semantic_object, compiled_patterns, cleaned_patterns) - if not pattern_hits: - continue - - matches.append( - SemanticSearchHit( - object_type=object_type, - semantic_model_id=_get_semantic_model_id(semantic_object), - object=semantic_object, - matched_patterns=sorted(pattern_hits), - matched_paths=sorted(field_hits), - ) - ) - return matches[:max_results] - - -async def load_semantic_context_for_semantic_type( - client: KeboolaClient, - object_type: SemanticObjectType, - *, - ids: Sequence[str] = tuple(), - semantic_model_ids: Sequence[str] | None = None, -) -> SemanticServiceDataTypeGroup: - """Get semantic context for a semantic object type, optionally filtered by semantic model IDs or object IDs.""" - if ids: - results = await process_concurrently( - ids, - lambda object_id: client.metastore_client.get_object(object_type.value, object_id), - max_concurrency=min(len(ids), 10), - ) - raw_objects = unwrap_results( - results, - f'Failed to fetch semantic objects for type "{object_type.value}".', - ) - objects = [_to_semantic_service_data(object_type, obj) for obj in raw_objects] - else: - objects = await _list_semantic_type_objects(client, object_type, semantic_model_ids) - - return SemanticServiceDataTypeGroup(object_type=object_type, objects=objects) - - -async def load_semantic_context_for_semantic_model( - client: KeboolaClient, - semantic_model_id: str, - *, - required_types: Sequence[SemanticObjectType] | None = None, -) -> dict[SemanticObjectType, SemanticServiceDataTypeGroup]: - """Load semantic context grouped by type for the given semantic model.""" - required_types = required_types or VALIDATION_OBJECT_TYPES - - results = await process_concurrently( - required_types, - lambda object_type: load_semantic_context_for_semantic_type( - client, - object_type, - semantic_model_ids=[semantic_model_id], - ), - max_concurrency=min(len(required_types), 10), - ) - groups = unwrap_results(results, 'Failed to fetch semantic context.') - return {group.object_type: group for group in groups} - - -def detect_used_objects_from_context( - sql_query: str, - context_by_type: dict[SemanticObjectType, SemanticServiceDataTypeGroup], - *, - used_objects_by_type: dict[SemanticObjectType, SemanticServiceDataTypeGroup] | None = None, -) -> dict[SemanticObjectType, SemanticServiceDataTypeGroup]: - """Detect semantic objects used by the SQL query from raw semantic context. If used objects are provided, - they will be combined with the detected objects if not detected by the heuristics""" - used_objects_by_type = used_objects_by_type or {} - datasets = context_by_type.get( - SemanticObjectType.SEMANTIC_DATASET, - SemanticServiceDataTypeGroup(object_type=SemanticObjectType.SEMANTIC_DATASET), - ) - metrics = context_by_type.get( - SemanticObjectType.SEMANTIC_METRIC, - SemanticServiceDataTypeGroup(object_type=SemanticObjectType.SEMANTIC_METRIC), - ) - relationships = context_by_type.get( - SemanticObjectType.SEMANTIC_RELATIONSHIP, - SemanticServiceDataTypeGroup(object_type=SemanticObjectType.SEMANTIC_RELATIONSHIP), - ) - # Detection is intentionally layered: - # 1. detect datasets first - # 2. detect metrics only within those datasets - # 3. detect relationships only between those detected datasets - # This keeps later detections narrower and reduces false positives. - - used_dataset_objects = _detect_used_datasets(sql_query, datasets.objects) - if expected := used_objects_by_type.get(SemanticObjectType.SEMANTIC_DATASET): - expected_objects = expected.objects - ids = {obj.id for obj in used_dataset_objects} - used_dataset_objects = used_dataset_objects + [obj for obj in expected_objects if obj.id not in ids] - used_dataset_ids = { - item.table_id.strip() for item in used_dataset_objects if item.table_id and item.table_id.strip() - } - - used_metric_objects = _detect_used_metrics_for_datasets(sql_query, metrics.objects, used_dataset_ids) - if expected := used_objects_by_type.get(SemanticObjectType.SEMANTIC_METRIC): - expected_objects = expected.objects - ids = {obj.id for obj in used_metric_objects} - used_metric_objects = used_metric_objects + [obj for obj in expected_objects if obj.id not in ids] - - used_relationship_objects = _detect_used_relationships(sql_query, relationships.objects, used_dataset_ids) - if expected := used_objects_by_type.get(SemanticObjectType.SEMANTIC_RELATIONSHIP): - expected_objects = expected.objects - ids = {obj.id for obj in used_relationship_objects} - used_relationship_objects = used_relationship_objects + [obj for obj in expected_objects if obj.id not in ids] - - used_groups: dict[SemanticObjectType, SemanticServiceDataTypeGroup] = {} - if used_dataset_objects: - used_groups[SemanticObjectType.SEMANTIC_DATASET] = SemanticServiceDataTypeGroup( - object_type=SemanticObjectType.SEMANTIC_DATASET, - objects=used_dataset_objects, - ) - if used_metric_objects: - used_groups[SemanticObjectType.SEMANTIC_METRIC] = SemanticServiceDataTypeGroup( - object_type=SemanticObjectType.SEMANTIC_METRIC, - objects=used_metric_objects, - ) - if used_relationship_objects: - used_groups[SemanticObjectType.SEMANTIC_RELATIONSHIP] = SemanticServiceDataTypeGroup( - object_type=SemanticObjectType.SEMANTIC_RELATIONSHIP, - objects=used_relationship_objects, - ) - return used_groups - - -def evaluate_constraints_from_context( - context_by_type: dict[SemanticObjectType, SemanticServiceDataTypeGroup], - used_object_groups_by_type: dict[SemanticObjectType, SemanticServiceDataTypeGroup], -) -> SemanticValidationServiceOutput: - """Evaluate relevant semantic constraints for the used semantic objects.""" - model_group = context_by_type.get( - SemanticObjectType.SEMANTIC_MODEL, - SemanticServiceDataTypeGroup(object_type=SemanticObjectType.SEMANTIC_MODEL), - ) - model = next(iter(model_group.objects), None) - constraint_group = context_by_type.get( - SemanticObjectType.SEMANTIC_CONSTRAINT, - SemanticServiceDataTypeGroup(object_type=SemanticObjectType.SEMANTIC_CONSTRAINT), - ) - constraints = constraint_group.objects - used_dataset_objects = ( - used_object_groups_by_type[SemanticObjectType.SEMANTIC_DATASET].objects - if SemanticObjectType.SEMANTIC_DATASET in used_object_groups_by_type - else [] - ) - used_metric_objects = ( - used_object_groups_by_type[SemanticObjectType.SEMANTIC_METRIC].objects - if SemanticObjectType.SEMANTIC_METRIC in used_object_groups_by_type - else [] - ) - used_relationship_objects = ( - used_object_groups_by_type[SemanticObjectType.SEMANTIC_RELATIONSHIP].objects - if SemanticObjectType.SEMANTIC_RELATIONSHIP in used_object_groups_by_type - else [] - ) - - used_dataset_ids = { - item.table_id.strip() for item in used_dataset_objects if item.table_id and item.table_id.strip() - } - used_metric_names = {item.name.strip() for item in used_metric_objects if item.name and item.name.strip()} - matched_relationships = sorted( - item.name or getattr(item.data.meta, 'name', None) or item.id for item in used_relationship_objects - ) - - sql_dialect_str = model.sql_dialect if model is not None else None - violations: list[ConstraintValidationFinding] = [] - post_execution_checks: list[ConstraintValidationFinding] = [] - has_error = False - - for constraint in constraints: - assert isinstance(constraint, SemanticConstraintData) - if not _constraint_is_relevant(constraint, used_metric_names, used_dataset_ids): - continue - - constraint_name = constraint.name or getattr(constraint.data.meta, 'name', None) or constraint.id - severity = constraint.severity or 'error' - constraint_type = constraint.constraint_type or 'unknown' - validation_query = _pick_validation_query(constraint, sql_dialect_str) - constraint_metrics = [metric.strip() for metric in constraint.metrics if metric.strip()] - constraint_datasets = [dataset.strip() for dataset in constraint.datasets if dataset.strip()] - pre_query_check = constraint.pre_query_check - - if constraint_type == 'composition': - # Composition constraints are the one class we can reliably check before execution: - # they usually declare that if one semantic metric family is used, other metrics - # must also be present in the same SQL. - missing_metrics = [metric for metric in constraint_metrics if metric not in used_metric_names] - if missing_metrics: - if severity == 'error': - has_error = True - violations.append( - ConstraintValidationFinding( - constraint_id=constraint.id, - constraint_name=constraint_name, - severity=severity, - status='missing_metrics', - message=_constraint_message( - constraint, - ( - f'Constraint "{constraint_name}" expects metrics present in the SQL: ' - f'{", ".join(missing_metrics)}.' - ), - ), - validation_query=validation_query, - ) - ) - continue - - if constraint_type == 'exclusion': - # Exclusion constraints model forbidden combinations. We only flag them when the - # query appears to use more than one excluded item from the same constraint scope. - used_excluded_metrics = [metric for metric in constraint_metrics if metric in used_metric_names] - used_excluded_datasets = [dataset for dataset in constraint_datasets if dataset in used_dataset_ids] - if len(used_excluded_metrics) > 1 or len(used_excluded_datasets) > 1: - if severity == 'error': - has_error = True - violations.append( - ConstraintValidationFinding( - constraint_id=constraint.id, - constraint_name=constraint_name, - severity=severity, - status='excluded_combination', - message=_constraint_message( - constraint, - f'Constraint "{constraint_name}" forbids this combination of semantic objects.', - ), - validation_query=validation_query, - ) - ) - continue - - if pre_query_check: - # Some constraints explicitly ask for manual pre-query review even when we cannot - # mechanically prove a violation from SQL text alone. - if severity == 'error': - has_error = True - violations.append( - ConstraintValidationFinding( - constraint_id=constraint.id, - constraint_name=constraint_name, - severity=severity, - status='pre_query_check', - message=_constraint_message( - constraint, - ( - f'Constraint "{constraint_name}" should be explicitly checked before trusting ' - f'the query result.' - ), - ), - validation_query=validation_query, - ) - ) - continue - - # Everything else is treated as a post-query concern unless the constraint has no - # recognized post-query semantics and does not provide its own validation SQL. - if constraint_type not in POST_QUERY_CONSTRAINT_TYPES and validation_query is None: - continue - - post_execution_checks.append( - ConstraintValidationFinding( - constraint_id=constraint.id, - constraint_name=constraint_name, - severity=severity, - status='post_query_check', - message=_constraint_message( - constraint, - ( - f'Constraint "{constraint_name}" is relevant for this SQL and should be verified ' - f'against the result.' - ), - ), - validation_query=validation_query, - ) - ) - - return SemanticValidationServiceOutput( - valid=not has_error, - used_object_groups=list(used_object_groups_by_type.values()), - matched_relationships=matched_relationships, - violations=violations, - post_execution_checks=post_execution_checks, - ) - - -def _merge_contexts( - contexts: list[dict[SemanticObjectType, SemanticServiceDataTypeGroup]], -) -> dict[SemanticObjectType, SemanticServiceDataTypeGroup]: - """Merge semantic contexts from multiple models into a single combined context.""" - merged: dict[SemanticObjectType, list[SemanticServiceData]] = {} - for context in contexts: - for object_type, group in context.items(): - merged.setdefault(object_type, []).extend(group.objects) - return { - object_type: SemanticServiceDataTypeGroup(object_type=object_type, objects=objects) - for object_type, objects in merged.items() - } - - -def _filter_used_objects_by_model( - used_object_groups_by_type: dict[SemanticObjectType, SemanticServiceDataTypeGroup], - model_id: str, -) -> dict[SemanticObjectType, SemanticServiceDataTypeGroup]: - """Filter used object groups to only objects belonging to the given model.""" - filtered: dict[SemanticObjectType, SemanticServiceDataTypeGroup] = {} - for object_type, group in used_object_groups_by_type.items(): - model_objects = [obj for obj in group.objects if _get_semantic_model_id(obj) == model_id] - if model_objects: - filtered[object_type] = SemanticServiceDataTypeGroup(object_type=object_type, objects=model_objects) - return filtered - - -async def _load_validation_contexts( - client: KeboolaClient, - semantic_model_ids: Sequence[str], -) -> list[dict[SemanticObjectType, SemanticServiceDataTypeGroup]]: - if not semantic_model_ids: - raise ValueError('At least one semantic_model_id must be provided.') - - results = await process_concurrently( - semantic_model_ids, - lambda model_id: load_semantic_context_for_semantic_model(client, model_id), - max_concurrency=min(len(semantic_model_ids), 10), - ) - return unwrap_results(results, 'Failed to fetch semantic context.') - - -def _merge_used_object_groups( - used_object_groups: Sequence[SemanticServiceDataTypeGroup], -) -> dict[SemanticObjectType, SemanticServiceDataTypeGroup]: - merged: dict[SemanticObjectType, list[SemanticServiceData]] = {} - for group in used_object_groups: - merged.setdefault(group.object_type, []).extend(group.objects) - - return { - object_type: SemanticServiceDataTypeGroup(object_type=object_type, objects=objects) - for object_type, objects in merged.items() - } - - -def _evaluate_used_objects_for_contexts( - semantic_model_ids: Sequence[str], - contexts_per_model: Sequence[dict[SemanticObjectType, SemanticServiceDataTypeGroup]], - used_object_groups_by_type: dict[SemanticObjectType, SemanticServiceDataTypeGroup], -) -> SemanticValidationServiceOutput: - all_violations: list[ConstraintValidationFinding] = [] - all_post_checks: list[ConstraintValidationFinding] = [] - has_error = False - for model_id, context_by_type in zip(semantic_model_ids, contexts_per_model, strict=True): - model_used_objects = _filter_used_objects_by_model(used_object_groups_by_type, model_id) - per_model_result = evaluate_constraints_from_context(context_by_type, model_used_objects) - all_violations.extend(per_model_result.violations) - all_post_checks.extend(per_model_result.post_execution_checks) - if not per_model_result.valid: - has_error = True - - used_relationships = used_object_groups_by_type.get( - SemanticObjectType.SEMANTIC_RELATIONSHIP, - SemanticServiceDataTypeGroup(object_type=SemanticObjectType.SEMANTIC_RELATIONSHIP), - ) - matched_relationships = sorted( - item.name or getattr(item.data.meta, 'name', None) or item.id for item in used_relationships.objects - ) - - return SemanticValidationServiceOutput( - valid=not has_error, - used_object_groups=list(used_object_groups_by_type.values()), - matched_relationships=matched_relationships, - violations=all_violations, - post_execution_checks=all_post_checks, - ) - - -async def validate_semantic_query_with_used_objects( - client: KeboolaClient, - sql_query: str, - semantic_model_ids: Sequence[str], - *, - used_object_groups: Sequence[SemanticServiceDataTypeGroup] | None = None, - contexts_per_model: list[dict[SemanticObjectType, SemanticServiceDataTypeGroup]] | None = None, -) -> SemanticValidationServiceOutput: - """Validate SQL against one or more semantic models without executing it. - If used objects are provided, they will be combined with the detected objects if not detected from sql query - by the heuristics. - Contexts from all requested models are merged into a single universe for object detection. - Constraint evaluation is performed per model to avoid cross-model rule contamination. - """ - if not sql_query.strip(): - raise ValueError('sql_query must not be empty.') - - cleaned_model_ids = list(dict.fromkeys(mid.strip() for mid in semantic_model_ids if mid and mid.strip())) - if not cleaned_model_ids: - raise ValueError('At least one semantic_model_id must be provided.') - - if contexts_per_model is None: - contexts_per_model = await _load_validation_contexts(client, cleaned_model_ids) - used_object_groups = used_object_groups or [] - - merged_context = _merge_contexts(contexts_per_model) - used_object_groups_by_type = _merge_used_object_groups(used_object_groups) - used_object_groups_by_type = detect_used_objects_from_context( - sql_query, merged_context, used_objects_by_type=used_object_groups_by_type - ) - return _evaluate_used_objects_for_contexts(cleaned_model_ids, contexts_per_model, used_object_groups_by_type) - - -async def get_object_by_id( - client: KeboolaClient, - object_type: SemanticObjectType, - object_id: str, -) -> SemanticServiceData: - raw_obj = await client.metastore_client.get_object(object_type.value, object_id) - if raw_obj.type != object_type.value: - raise ValueError( - f'Expected object "{object_id}" to be of type "{object_type.value}", ' - f'got "{raw_obj.type}" from the Metastore API.' - ) - return _to_semantic_service_data(object_type, raw_obj) diff --git a/src/keboola_mcp_server/tools/semantic/tools.py b/src/keboola_mcp_server/tools/semantic/tools.py deleted file mode 100644 index 5cbca3d3e..000000000 --- a/src/keboola_mcp_server/tools/semantic/tools.py +++ /dev/null @@ -1,879 +0,0 @@ -"""Semantic read tools backed by the semantic service layer.""" - -from collections.abc import Sequence -from typing import Annotated, Any - -from fastmcp import Context, FastMCP -from fastmcp.tools import FunctionTool -from mcp.types import ToolAnnotations -from pydantic import BaseModel, ConfigDict, Field - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.mcp import process_concurrently, toon_serializer_compact, unwrap_results -from keboola_mcp_server.tools.constants import SEMANTIC_TOOLS_TAG -from keboola_mcp_server.tools.semantic import service as semantic_service -from keboola_mcp_server.tools.semantic.model import ( - SemanticObjectRef, - SemanticObjectType, - SemanticObjectTypeSelection, - SemanticSchemaDefinition, -) - - -class ConstraintValidationFinding(BaseModel): - """Tool-facing semantic constraint finding.""" - - constraint_id: str = Field(description='Constraint UUID.') - constraint_name: str = Field(description='Constraint name.') - severity: str = Field(description='Constraint severity.') - status: str = Field(description='Validation status.') - message: str = Field(description='Human-readable validation finding.') - validation_query: str | None = Field( - default=None, - description='Optional SQL validation query suggested by the semantic constraint.', - ) - - -class CompactSemanticObject(BaseModel): - id: str - name: str | None = None - - -class SemanticModelCompact(CompactSemanticObject): - description: str | None = None - sql_dialect: str | None = None - - @classmethod - def from_semantic_service_data(cls, obj: semantic_service.SemanticServiceData) -> 'SemanticModelCompact': - attributes = obj.data.attributes or {} - return cls( - id=obj.id, - name=obj.display_name, - description=attributes.get('description'), - sql_dialect=attributes.get('sql_dialect'), - ) - - -class SemanticDatasetCompact(CompactSemanticObject): - model_config = ConfigDict(populate_by_name=True) - - table_id: str | None = Field(default=None, validation_alias='tableId', serialization_alias='tableId') - description: str | None = None - model_uuid: str | None = None - fqn: str | None = None - - @classmethod - def from_semantic_service_data(cls, obj: semantic_service.SemanticServiceData) -> 'SemanticDatasetCompact': - attributes = obj.data.attributes or {} - return cls( - id=obj.id, - name=obj.display_name, - table_id=attributes.get('tableId'), - description=attributes.get('description'), - model_uuid=attributes.get('modelUUID'), - fqn=attributes.get('fqn'), - ) - - -class SemanticMetricCompact(CompactSemanticObject): - description: str | None = None - dataset: str | None = None - model_uuid: str | None = None - - @classmethod - def from_semantic_service_data(cls, obj: semantic_service.SemanticServiceData) -> 'SemanticMetricCompact': - attributes = obj.data.attributes or {} - return cls( - id=obj.id, - name=obj.display_name, - description=attributes.get('description'), - dataset=attributes.get('dataset'), - model_uuid=attributes.get('modelUUID'), - ) - - -class SemanticRelationshipCompact(CompactSemanticObject): - from_dataset: str | None = None - to_dataset: str | None = None - type: str | None = None - on: str | None = None - model_uuid: str | None = None - - @classmethod - def from_semantic_service_data(cls, obj: semantic_service.SemanticServiceData) -> 'SemanticRelationshipCompact': - attributes = obj.data.attributes or {} - return cls( - id=obj.id, - name=obj.display_name, - from_dataset=attributes.get('from'), - to_dataset=attributes.get('to'), - type=attributes.get('type'), - on=attributes.get('on'), - model_uuid=attributes.get('modelUUID'), - ) - - -class SemanticGlossaryCompact(CompactSemanticObject): - term: str | None = None - definition: str | None = None - model_uuid: str | None = None - - @classmethod - def from_semantic_service_data(cls, obj: semantic_service.SemanticServiceData) -> 'SemanticGlossaryCompact': - attributes = obj.data.attributes or {} - return cls( - id=obj.id, - name=obj.display_name, - term=attributes.get('term'), - definition=attributes.get('definition'), - model_uuid=attributes.get('modelUUID'), - ) - - -class SemanticConstraintCompact(CompactSemanticObject): - description: str | None = None - type: str | None = None - rule: str | None = None - severity: str | None = None - model_uuid: str | None = None - - @classmethod - def from_semantic_service_data(cls, obj: semantic_service.SemanticServiceData) -> 'SemanticConstraintCompact': - attributes = obj.data.attributes or {} - return cls( - id=obj.id, - name=obj.display_name, - description=attributes.get('description'), - type=attributes.get('constraintType'), - rule=attributes.get('rule'), - severity=attributes.get('severity'), - model_uuid=attributes.get('modelUUID'), - ) - - -class SemanticObject(CompactSemanticObject): - attributes: dict[str, Any] = Field(default_factory=dict) - - @classmethod - def from_semantic_service_data(cls, obj: semantic_service.SemanticServiceData) -> 'SemanticObject': - return cls( - id=obj.id, - name=obj.display_name, - attributes=obj.data.attributes or {}, - ) - - -SemanticCompactObject = ( - SemanticModelCompact - | SemanticDatasetCompact - | SemanticMetricCompact - | SemanticRelationshipCompact - | SemanticGlossaryCompact - | SemanticConstraintCompact -) - - -SemanticContextObject = SemanticCompactObject | SemanticObject - - -class SemanticObjectMatchOutput(BaseModel): - """Matched semantic object returned by semantic search.""" - - object_type: SemanticObjectType = Field(description='Matched semantic object type.') - matched_paths: list[str] = Field(default_factory=list, description='Matched paths inside the semantic object.') - data: SemanticCompactObject = Field(description='Compact matched semantic object detail.') - - -class SemanticSearchModelGroup(BaseModel): - """Search matches grouped by semantic model.""" - - semantic_model_id: str = Field(description='Semantic model UUID.') - matches: list[SemanticObjectMatchOutput] = Field( - default_factory=list, - description='Matched objects for this model.', - ) - - -class SemanticObjectTypeContext(BaseModel): - """Tool output context for a single semantic object type.""" - - object_type: SemanticObjectType = Field(description='Semantic object type.') - objects: list[SemanticContextObject] = Field( - default_factory=list, - description='Semantic objects of the requested type.', - ) - - -class SemanticUsedDataset(BaseModel): - """Dataset referenced by the validated SQL.""" - - model_config = ConfigDict(populate_by_name=True) - - id: str = Field(description='Dataset UUID.') - name: str = Field(description='Dataset name.') - table_id: str = Field(description='Keboola table ID.', validation_alias='tableId', serialization_alias='tableId') - description: str = Field(description='Dataset description.') - fqn: str = Field(description='Dataset fully qualified SQL name.') - - @classmethod - def from_semantic_service_data(cls, obj: semantic_service.SemanticDatasetData) -> 'SemanticUsedDataset': - return cls( - id=obj.id, - name=obj.name or '', - table_id=obj.table_id or '', - description=obj.description or '', - fqn=obj.fqn or '', - ) - - -class SemanticUsedMetric(BaseModel): - """Metric referenced by the validated SQL.""" - - id: str = Field(description='Metric UUID.') - name: str = Field(description='Metric name.') - description: str = Field(description='Metric description.') - sql: str = Field(description='Metric SQL expression.') - dataset: str = Field(description='Source dataset table ID.') - - @classmethod - def from_semantic_service_data(cls, obj: semantic_service.SemanticMetricData) -> 'SemanticUsedMetric': - return cls( - id=obj.id, - name=obj.name or '', - description=obj.description or '', - sql=obj.sql or '', - dataset=obj.dataset or '', - ) - - -class SemanticQueryValidationResult(BaseModel): - """One semantic validation result view.""" - - valid: bool = Field(description='False when an error-severity pre-execution finding was detected.') - semantic_models: list[SemanticModelCompact] = Field( - default_factory=list, - description='Semantic models against which the SQL was validated.', - ) - sql_dialects: list[str] = Field( - default_factory=list, - description=( - 'SQL dialects of the semantic models. ' - 'Contains more than one entry when models use different dialects, which is a sign of incompatibility.' - ), - ) - used_datasets: list[SemanticUsedDataset] = Field( - default_factory=list, - description='Semantic datasets referenced by the SQL.', - ) - used_metrics: list[SemanticUsedMetric] = Field( - default_factory=list, - description='Semantic metrics referenced by the SQL.', - ) - matched_relationships: list[str] = Field( - default_factory=list, - description='Relationship names detected in SQL.', - ) - violations: list[ConstraintValidationFinding] = Field( - default_factory=list, - description='Pre-execution semantic violations.', - ) - post_execution_checks: list[ConstraintValidationFinding] = Field( - default_factory=list, - description='Checks that should be verified against query results.', - ) - summary: str = Field(description='Short validation summary.') - - -class ValidateSemanticQueryOutput(BaseModel): - """Output for semantic SQL validation.""" - - validation_auto_detected: SemanticQueryValidationResult = Field( - description='Validation result built from semantic objects auto-detected from the SQL query.', - ) - validation_detected_from_expected: SemanticQueryValidationResult | None = Field( - default=None, - description='Validation result built only from explicitly provided expected semantic object IDs.', - ) - matched_expected_objects: list[SemanticObjectRef] = Field( - default_factory=list, - description='Expected semantic objects that were also detected in the SQL.', - ) - missing_expected_objects: list[SemanticObjectRef] = Field( - default_factory=list, - description='Expected semantic objects that were not detected in the SQL.', - ) - unexpected_detected_objects: list[SemanticObjectTypeContext] = Field( - default_factory=list, - description='Detected semantic objects that fall outside the expected semantic scope.', - ) - - -def _compact_semantic_object(obj: semantic_service.SemanticServiceData) -> CompactSemanticObject: - if obj.semantic_type == SemanticObjectType.SEMANTIC_MODEL: - return SemanticModelCompact.from_semantic_service_data(obj) - elif obj.semantic_type == SemanticObjectType.SEMANTIC_DATASET: - return SemanticDatasetCompact.from_semantic_service_data(obj) - elif obj.semantic_type == SemanticObjectType.SEMANTIC_METRIC: - return SemanticMetricCompact.from_semantic_service_data(obj) - elif obj.semantic_type == SemanticObjectType.SEMANTIC_RELATIONSHIP: - return SemanticRelationshipCompact.from_semantic_service_data(obj) - elif obj.semantic_type == SemanticObjectType.SEMANTIC_GLOSSARY: - return SemanticGlossaryCompact.from_semantic_service_data(obj) - elif obj.semantic_type == SemanticObjectType.SEMANTIC_CONSTRAINT: - return SemanticConstraintCompact.from_semantic_service_data(obj) - raise ValueError(f'Unsupported semantic object type "{obj.semantic_type.value}"') - - -def _compare_expected_and_detected_objects( - expected_semantic_objects: Sequence[SemanticObjectTypeSelection], - used_object_groups: Sequence[semantic_service.SemanticServiceDataTypeGroup], -) -> tuple[list[SemanticObjectRef], list[SemanticObjectRef], list[SemanticObjectTypeContext]]: - if not expected_semantic_objects: - return [], [], [] - expected_ids_by_type: dict[SemanticObjectType, set[str]] = {} - for selection in expected_semantic_objects: - if selection.ids: - expected_ids_by_type.setdefault(selection.object_type, set()).update(selection.ids) - expected_types = {selection.object_type for selection in expected_semantic_objects} - - matched_expected_objects: list[SemanticObjectRef] = [] - missing_expected_objects: list[SemanticObjectRef] = [] - unexpected_detected_objects: list[SemanticObjectTypeContext] = [] - - for object_type, expected_ids in expected_ids_by_type.items(): - detected_ids = { - obj.id for group in used_object_groups if group.object_type == object_type for obj in group.objects - } - matched_expected_objects.extend( - SemanticObjectRef(object_type=object_type, id=object_id) - for object_id in sorted(expected_ids & detected_ids) - ) - missing_expected_objects.extend( - SemanticObjectRef(object_type=object_type, id=object_id) - for object_id in sorted(expected_ids - detected_ids) - ) - - for group in used_object_groups: - selection_ids = expected_ids_by_type.get(group.object_type) - if group.object_type not in expected_types: - unexpected_objects = [_compact_semantic_object(obj) for obj in group.objects] - elif selection_ids: - unexpected_objects = [_compact_semantic_object(obj) for obj in group.objects if obj.id not in selection_ids] - else: - unexpected_objects = [] - - if unexpected_objects: - unexpected_detected_objects.append( - SemanticObjectTypeContext(object_type=group.object_type, objects=unexpected_objects) - ) - - return ( - matched_expected_objects, - missing_expected_objects, - unexpected_detected_objects, - ) - - -def _to_tool_finding(finding: semantic_service.ConstraintValidationFinding) -> ConstraintValidationFinding: - return ConstraintValidationFinding( - constraint_id=finding.constraint_id, - constraint_name=finding.constraint_name, - severity=finding.severity, - status=finding.status, - message=finding.message, - validation_query=finding.validation_query, - ) - - -def _format_validation_result( - raw_result: semantic_service.SemanticValidationServiceOutput, - *, - models: Sequence[semantic_service.SemanticModelData] = tuple(), - summary_notes: Sequence[str] = tuple(), -) -> SemanticQueryValidationResult: - used_dataset_objects = [] - used_metric_objects = [] - for group in raw_result.used_object_groups: - if group.object_type == SemanticObjectType.SEMANTIC_DATASET: - used_dataset_objects = [item for item in group.objects] - elif group.object_type == SemanticObjectType.SEMANTIC_METRIC: - used_metric_objects = [item for item in group.objects] - - used_datasets = [SemanticUsedDataset.from_semantic_service_data(item) for item in used_dataset_objects] - used_metrics = [SemanticUsedMetric.from_semantic_service_data(item) for item in used_metric_objects] - - semantic_model_outputs = [SemanticModelCompact.from_semantic_service_data(m) for m in models] - sql_dialects = sorted({m.sql_dialect for m in models if m.sql_dialect}) - - summary_parts: list[str] = [] - if len(sql_dialects) > 1: - summary_parts.append( - f'Warning: semantic models use different SQL dialects ({", ".join(sql_dialects)}). ' - 'The query may not be portable across all models.' - ) - if raw_result.violations: - summary_parts.append('Semantic validation found pre-execution issues that should be fixed before running.') - if raw_result.post_execution_checks: - summary_parts.append('Some checks should be verified after execution.') - summary_parts.extend(summary_notes) - - if summary_parts: - summary = '\n'.join(summary_parts) - else: - summary = 'Semantic validation finished without relevant findings.' - - return SemanticQueryValidationResult( - valid=raw_result.valid, - semantic_models=semantic_model_outputs, - sql_dialects=sql_dialects, - used_datasets=used_datasets, - used_metrics=used_metrics, - matched_relationships=raw_result.matched_relationships, - violations=[_to_tool_finding(finding) for finding in raw_result.violations], - post_execution_checks=[_to_tool_finding(finding) for finding in raw_result.post_execution_checks], - summary=summary, - ) - - -def add_semantic_tools(mcp: FastMCP) -> None: - """Register semantic read tools.""" - mcp.add_tool( - FunctionTool.from_function( - search_semantic_context, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={SEMANTIC_TOOLS_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - get_semantic_context, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={SEMANTIC_TOOLS_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - get_semantic_schema, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={SEMANTIC_TOOLS_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - validate_semantic_query, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={SEMANTIC_TOOLS_TAG}, - ) - ) - - -@tool_errors() -async def search_semantic_context( - ctx: Context, - patterns: Annotated[ - Sequence[str], - Field( - description=( - 'One or more regex patterns used to search semantic metadata. ' - 'The search checks semantic model names plus semantic object names and nested attribute values. ' - 'Use multiple patterns when you need to find objects related to several business terms at once.' - ) - ), - ], - semantic_types: Annotated[ - Sequence[SemanticObjectType], - Field( - description=( - 'Optional semantic object types to search. ' - 'Empty list [] means ALL semantic object types are searched. ' - 'Use this to narrow the search when you already know whether you want datasets, metrics, ' - 'relationships, glossary terms, constraints, or models.' - ) - ), - ] = tuple(), - semantic_model_ids: Annotated[ - Sequence[str], - Field( - description=( - 'Optional list of semantic model IDs to restrict the search to specific models. ' - 'Empty list [] means search across all semantic models.' - ) - ), - ] = tuple(), - case_sensitive: Annotated[ - bool, - Field( - description=( - 'Whether regex matching should be case-sensitive. ' - 'Leave false for normal discovery; set true only when exact casing matters.' - ) - ), - ] = False, - max_results: Annotated[ - int, - Field( - description=( - 'Maximum number of matched semantic objects to return. ' - 'Use a smaller value for quick discovery and a larger value only when you need a broader result set.' - ) - ), - ] = 100, -) -> list[SemanticSearchModelGroup]: - """ - Searches semantic models and semantic objects using regex patterns matched against their names, descriptions and - stringified JSON attributes. - - Returns compact matches grouped by semantic model. Each match includes the semantic object type, - the paths where the patterns matched, and compact object view. - - CONSIDERATIONS: - - The search is case-insensitive by default. Use `case_sensitive=True` when exact casing matters. - - The search is performed against semantic object names and data attributes which are stringified JSON objects - following their corresponding JSON schema. - - The search can be scoped to specific semantic models or semantic object types but prefer broader search without - scoping unless required by the context. - - WHEN TO USE: - - When you need to discover which semantic objects are relevant to a user request. - - When you know business terms, column names, metric fragments, or rule names, but not exact object UUIDs. - - When you need to find semantic objects by keyword or values used in their attributes. - - WHEN NOT TO USE: - - When you know the exact IDs. - - EXAMPLES: - - Find semantic objects by business concepts for revenue or sales: - `patterns=["revenue", "sales"]` - - Find semantic objects using a Keboola table ID: - `patterns=["out.c-sales-main.fact_orders"]` - - Find semantic dataset for a certain table: - `patterns=["in.c-sales-main.fact_orders"], semantic_types=["semantic-dataset"]` - - Find semantic datasets that mention a column name: - `patterns=["column_name"], semantic_types=["semantic-dataset"]` - - Search semantic objects e.g. semantic metrics, relationships, and constraints using a certain semantic dataset: - `patterns=["table-id-of-the-dataset"], semantic_types=["semantic-metric",` - `"semantic-relationship", "semantic-constraint"]` - - Search semantic constraints using e.g. certain semantic metrics and certain semantic datasets: - `patterns=["metric-name-1", "metric-name-2", "table-id-from-the-dataset"],` - `semantic_types=["semantic-metric", "semantic-relationship"]` - - Search something within specific semantic models only: - `patterns=["something"], semantic_model_ids=["", ""]` - """ - cleaned_patterns = [pattern.strip() for pattern in patterns if pattern and pattern.strip()] - if not cleaned_patterns: - raise ValueError('At least one regex pattern must be provided.') - if max_results <= 0: - raise ValueError('max_results must be a positive integer.') - - client = KeboolaClient.from_state(ctx.session.state) - hits = await semantic_service.search_semantic_context( - client, - cleaned_patterns, - semantic_types=semantic_types, - semantic_model_ids=semantic_model_ids or None, - case_sensitive=case_sensitive, - max_results=max_results, - ) - - grouped_matches: dict[str, list[SemanticObjectMatchOutput]] = {} - - for hit in hits: - grouped_matches.setdefault(hit.semantic_model_id, []).append( - SemanticObjectMatchOutput( - object_type=hit.object_type, - matched_paths=list(hit.matched_paths), - data=_compact_semantic_object(hit.object), - ) - ) - - model_results = [ - SemanticSearchModelGroup( - semantic_model_id=model_id, - matches=sorted(grouped, key=lambda item: item.data.name or item.data.id), - ) - for model_id, grouped in grouped_matches.items() - ] - model_results.sort(key=lambda item: item.semantic_model_id) - - return model_results - - -@tool_errors() -async def get_semantic_context( - ctx: Context, - semantic_objects: Annotated[ - Sequence[SemanticObjectTypeSelection], - Field( - description=( - 'List of semantic object selections to load. ' - 'Each item contains "object_type" and optional "ids". ' - 'If "ids" is empty, all objects of that type are returned in compact form. ' - 'If "ids" is non-empty, only those objects are returned with full attributes.' - ) - ), - ], - semantic_model_ids: Annotated[ - Sequence[str], - Field( - description=( - 'Optional list of semantic model IDs to restrict loading to specific models. ' - 'Empty list [] means load across all semantic models.' - ) - ), - ] = tuple(), -) -> list[SemanticObjectTypeContext]: - """ - Loads semantic objects grouped by semantic object type. - - CONSIDERATIONS: - - If a selection has empty `ids`, the tool returns all objects of that type in compact form. - - If a selection has non-empty `ids`, the tool returns only those specific objects with full attributes. - - `semantic_model_ids` optionally narrows the lookup to specific semantic models. - - WHEN TO USE: - - When you already know IDs of the semantic objects you want to load and want to inspect them in detail. - - When you want to list all semantic objects of certain types or specific semantic models. - - When you want to list semantic models. - - WHEN NOT TO USE: - - When you need to discover semantic objects. - - EXAMPLES: - - List all semantic models: - `semantic_objects=[{"object_type": "semantic-model"}]` - - List semantic datasets and metrics for specific semantic models: - `semantic_objects=[{"object_type": "semantic-dataset"}, {"object_type": "semantic-metric"}],` - `semantic_model_ids=["model-uuid-1", "model-uuid-2"]` - - Get detailed context for specific semantic objects by their id: - `semantic_objects=[{"object_type": "semantic-dataset", "ids": ["dataset-uuid-1"]},` - `{"object_type": "semantic-metric", "ids": ["metric-uuid-1", "metric-uuid-2"]}]` - - List all constraints for specific semantic models: - `semantic_objects=[{"object_type": "semantic-constraint"}], semantic_model_ids=["model-uuid-1"]` - """ - if not semantic_objects: - raise ValueError('At least one semantic object type must be provided.') - - client = KeboolaClient.from_state(ctx.session.state) - - results = await process_concurrently( - semantic_objects, - lambda selection: semantic_service.load_semantic_context_for_semantic_type( - client, selection.object_type, semantic_model_ids=semantic_model_ids or None, ids=selection.ids - ), - max_concurrency=min(len(semantic_objects), 10), - ) - groups = unwrap_results(results, 'Failed to fetch semantic context.') - - # Normalize the contexts to the SemanticObjectTypeContext format - normalized_contexts: list[SemanticObjectTypeContext] = [] - for selection, context in zip(semantic_objects, groups, strict=True): - assert isinstance( - context, semantic_service.SemanticServiceDataTypeGroup - ), f'Expected SemanticServiceDataTypeGroup, got {type(context)}' - assert ( - selection.object_type == context.object_type - ), f'Semantic object type mismatch: {selection.object_type} != {context.object_type}' - if selection.ids: - # Detail context with specific IDs - normalized_contexts.append( - SemanticObjectTypeContext( - object_type=context.object_type, - objects=[SemanticObject.from_semantic_service_data(obj) for obj in context.objects], - ) - ) - else: - normalized_contexts.append( - SemanticObjectTypeContext( - object_type=context.object_type, - objects=[_compact_semantic_object(obj) for obj in context.objects], - ) - ) - - return normalized_contexts - - -@tool_errors() -async def get_semantic_schema( - ctx: Context, - semantic_types: Annotated[ - Sequence[SemanticObjectType], - Field( - description=( - 'List of semantic object types for which JSON schemas should be returned. ' - 'Each returned item contains the requested semantic type and its metastore schema.' - ) - ), - ], -) -> list[SemanticSchemaDefinition]: - """ - Returns JSON schemas for the requested semantic object types. - - WHEN TO USE: - - When you want to know the JSON schema of a semantic object type, e.g. before searching something specific. - - """ - if not semantic_types: - raise ValueError('At least one semantic type must be provided.') - - client = KeboolaClient.from_state(ctx.session.state) - results = await process_concurrently( - semantic_types, - lambda semantic_type: client.metastore_client.get_schema(semantic_type.value), - max_concurrency=min(len(semantic_types), 10), - ) - schemas = unwrap_results(results, 'Failed to fetch one or more semantic schemas.') - - return [ - SemanticSchemaDefinition(semantic_type=semantic_type, schema_definition=schema) - for semantic_type, schema in zip(semantic_types, schemas, strict=True) - ] - - -@tool_errors() -async def validate_semantic_query( - ctx: Context, - sql_query: Annotated[ - str, - Field( - description=( - 'SQL query that should be checked against the semantic layer. ' - 'The query is not executed; the tool performs best-effort semantic detection and rule validation ' - 'using heuristic string matching, so the detected objects may be incomplete or imperfect.' - ) - ), - ], - semantic_model_ids: Annotated[ - Sequence[str], - Field( - description=( - 'One or more semantic model IDs against which the SQL should be validated. ' - 'Contexts from all models are merged into a single universe for object detection. ' - 'Constraint evaluation is performed per model to avoid cross-model rule contamination.' - ) - ), - ], - expected_semantic_objects: Annotated[ - Sequence[SemanticObjectTypeSelection], - Field( - description=( - 'Optional semantic object selections that define the expected semantic scope of the query. ' - 'These expectations are compared with the objects actually detected in the SQL. ' - 'Use `ids` when you want to assert that specific semantic objects should be present.' - ) - ), - ] = tuple(), -) -> ValidateSemanticQueryOutput: - """ - Performs best-effort semantic validation of an SQL query against one or more semantic models and compares it with - the expected semantic objects provided. - - RETURNS: - - `validation_auto_detected`: semantic validation built from objects heuristically detected in the SQL - - `validation_detected_from_expected`: semantic validation built only from explicitly provided expected object IDs - - expected semantic objects that were matched or missing in the auto-detected result - - unexpected auto-detected objects outside the expected semantic scope - - LIMITATIONS: - - Detection is heuristic and based on string matching over SQL and semantic metadata. - - The tool does not parse SQL semantically and does not execute the query. - - Auto-detected objects, missing objects, and relationship matches may therefore be imperfect. - - Use the result as a best-effort semantic check, not as a formal proof that the query is correct. - - CONSIDERATIONS: - - Prefer calling this tool before executing any SQL that touches semantic objects. - - This tool confirms the SQL dialect, surfaces semantic constraint violations, and provides post-execution checks. - - Only proceed to query_data once this tool returns valid=True and violations is empty. If violations are found, - fix the query first or consider the limitations of this tool. - - WHEN TO USE: - - Before generating or approving a query that should follow a semantic model. - - When you want to validate a SQL query against the semantic objects before executing it using "query_data" tool - or creating a new SQL transformation out of it, especially when investigating data quality issues. - - When you want to verify that a query uses the intended semantic objects. - - When you need to surface semantic business-rule violations or follow-up checks. - - EXAMPLES: - - Validate a SQL query against one semantic model: - `sql_query="SELECT SUM(\\"REVENUE\\") FROM ...", semantic_model_ids=["semantic-model-uuid"],` - `expected_semantic_objects=[{"object_type": "semantic-dataset"}]` - - Validate a cross-model query against two semantic models: - `sql_query="SELECT * FROM ...", semantic_model_ids=["model-uuid-1", "model-uuid-2"],` - `expected_semantic_objects=[{"object_type": "semantic-dataset", "ids": ["dataset-uuid-1"]}]` - - Validate a query and compare it against expected objects: - `sql_query="SELECT SUM(\\"REVENUE\\") FROM ...", semantic_model_ids=["semantic-model-uuid"],` - `expected_semantic_objects=[{"object_type": "semantic-metric", "ids": ["metric-uuid-1"]}]` - - """ - if not sql_query.strip(): - raise ValueError('sql_query must not be empty.') - cleaned_model_ids = list(dict.fromkeys(mid.strip() for mid in semantic_model_ids if mid and mid.strip())) - if not cleaned_model_ids: - raise ValueError('At least one semantic_model_id must be provided.') - - client = KeboolaClient.from_state(ctx.session.state) - - model_results = await process_concurrently( - cleaned_model_ids, - lambda model_id: semantic_service.get_object_by_id(client, SemanticObjectType.SEMANTIC_MODEL, model_id), - max_concurrency=min(len(cleaned_model_ids), 10), - ) - models = unwrap_results(model_results, 'Failed to fetch semantic models.') - assert all(isinstance(m, semantic_service.SemanticModelData) for m in models) - - # Pre-load contexts once when both validation paths will run, avoiding a double round-trip. - pre_loaded_contexts = None - if expected_semantic_objects: - pre_loaded_contexts = await semantic_service._load_validation_contexts(client, cleaned_model_ids) - - raw_auto_detected = await semantic_service.validate_semantic_query_with_used_objects( - client, sql_query, cleaned_model_ids, contexts_per_model=pre_loaded_contexts - ) - matched_expected_objects = [] - missing_expected_objects = [] - unexpected_detected_objects = [] - raw_from_expected = None - if expected_semantic_objects: - matched_expected_objects, missing_expected_objects, unexpected_detected_objects = ( - _compare_expected_and_detected_objects(expected_semantic_objects, raw_auto_detected.used_object_groups) - ) - results = await process_concurrently( - expected_semantic_objects, - lambda selection: semantic_service.load_semantic_context_for_semantic_type( - client, selection.object_type, semantic_model_ids=semantic_model_ids or None, ids=selection.ids - ), - max_concurrency=min(len(expected_semantic_objects), 10), - ) - expected_object_groups = unwrap_results(results, 'Failed to fetch semantic context.') - if expected_object_groups: - raw_from_expected = await semantic_service.validate_semantic_query_with_used_objects( - client, - sql_query, - cleaned_model_ids, - used_object_groups=expected_object_groups, - contexts_per_model=pre_loaded_contexts, - ) - - auto_detected_summary_notes: list[str] = [] - if missing_expected_objects: - auto_detected_summary_notes.append('Some expected semantic objects were not detected in the SQL query.') - if unexpected_detected_objects: - auto_detected_summary_notes.append('Some detected semantic objects fall outside the expected semantic scope.') - - return ValidateSemanticQueryOutput( - validation_auto_detected=_format_validation_result( - raw_auto_detected, - models=models, - summary_notes=auto_detected_summary_notes, - ), - validation_detected_from_expected=( - _format_validation_result(raw_from_expected, models=models) if raw_from_expected is not None else None - ), - matched_expected_objects=matched_expected_objects, - missing_expected_objects=missing_expected_objects, - unexpected_detected_objects=unexpected_detected_objects, - ) diff --git a/src/keboola_mcp_server/tools/sql.py b/src/keboola_mcp_server/tools/sql.py deleted file mode 100644 index 0b800e003..000000000 --- a/src/keboola_mcp_server/tools/sql.py +++ /dev/null @@ -1,336 +0,0 @@ -import asyncio -import contextlib -import csv -import logging -from io import StringIO -from typing import Annotated, Awaitable - -from fastmcp import Context, FastMCP -from fastmcp.tools import FunctionTool -from mcp.types import ( - ProgressNotification, - ProgressNotificationParams, - ProgressToken, - ServerNotification, - ToolAnnotations, -) -from pydantic import BaseModel, Field -from starlette.requests import Request - -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.mcp import get_http_request_or_none -from keboola_mcp_server.workspace import JobSubmittedInfo, QueryResult, SqlSelectData, WorkspaceManager - -LOG = logging.getLogger(__name__) - -SQL_TOOLS_TAG = 'sql' -MAX_ROWS = 1_000 -MAX_CHARS = 50_000 -# How often to check whether the HTTP client has disconnected during a long query. -# Mirrors the 1 s job-poll cadence in `_Workspace.execute_query`. -_DISCONNECT_POLL_INTERVAL = 1.0 - - -async def _watch_for_http_disconnect(request: Request, poll_interval: float | None = None) -> None: - """Return when the underlying HTTP `request` is torn down. - - In stateless streamable-HTTP mode (`stateless_http=True` in `cli.py`), the MCP - `notifications/cancelled` payload arrives on a fresh transport instance and cannot - reach the in-flight tool call's session — so `asyncio.CancelledError` is never - raised inside the running tool. Watching the underlying ASGI request for an - `http.disconnect` event lets us notice when the client gave up (closed the tab, - hit "stop" in Kai, lost network) and trigger the same cancellation path we - already have for SDK-driven cancels. - - Only started when there is an HTTP request bound (see `query_data`); on stdio / - background workers there is nothing to watch, so the caller skips the race entirely. - - Any error from `is_disconnected()` is treated as "still connected" so a transient - ASGI hiccup never cancels an otherwise-working query. - - `poll_interval` is resolved at call time (not bound as a default) so tests can - patch `_DISCONNECT_POLL_INTERVAL` and have it take effect here. - """ - interval = poll_interval if poll_interval is not None else _DISCONNECT_POLL_INTERVAL - while True: - try: - if await request.is_disconnected(): - return - except Exception: - LOG.debug('HTTP is_disconnected() check failed; treating as still-connected', exc_info=True) - await asyncio.sleep(interval) - - -async def _drain(*tasks: asyncio.Task) -> None: - """Cancel and drain `tasks` under a shield so their cancellation cleanup — notably the - backend `cancel_job` in `_Workspace.execute_query` — finishes even when `query_data` is - itself being torn down. - - The shield protects the in-flight cleanup, but the `await` here is deliberately NOT - protected: if `query_data` is cancelled at this boundary, the `CancelledError` propagates - out (the caller's cancellation must win — we must not go on to return a result or raise a - synthetic error as if nothing happened) while the drain runs to completion in the background. - """ - await asyncio.shield(asyncio.gather(*(_cancel_and_drain(t) for t in tasks))) - - -async def _execute_watching_disconnect( - query_coro: Awaitable[QueryResult], request: Request, query_name: str -) -> QueryResult: - """Run `query_coro`, cancelling it — and thus the backend job, via `execute_query`'s - `CancelledError` handler — if the HTTP client disconnects first. - - Returns the query result when the query wins the race. Raises `ValueError('Query was - cancelled')` if the client disconnected (or the disconnect watcher itself failed) before the - query finished: a plain `ValueError` keeps the cancelled call on the normal `@tool_errors` - path, so it is logged as an error (not a success) and the client still gets a response — a - raised `CancelledError` would be a `BaseException` that bypasses the decorator entirely. - """ - query_task = asyncio.create_task(query_coro) - disconnect_task = asyncio.create_task(_watch_for_http_disconnect(request)) - try: - done, _pending = await asyncio.wait([query_task, disconnect_task], return_when=asyncio.FIRST_COMPLETED) - except BaseException: - # `query_data` itself was cancelled (e.g. SDK-driven MCP cancellation on a non-stateless - # transport). Drain both tasks so `execute_query`'s CancelledError handler runs long enough - # to propagate `cancel_job` and neither task leaks as pending; suppress the drain's own - # CancelledError so the original exception propagates via `raise`. - with contextlib.suppress(asyncio.CancelledError): - await _drain(query_task, disconnect_task) - raise - - if query_task in done: - # The query won; the watcher is still pending, so drain it. - await _drain(disconnect_task) - return query_task.result() - - # The watcher won: either the client disconnected, or the watcher itself raised. Cancel the - # query either way, but keep the log honest so a watcher bug isn't disguised as a disconnect. - watcher_exc = disconnect_task.exception() - if watcher_exc is not None: - # Pass an explicit (type, exc, tb) tuple: stdlib logging treats a truthy non-tuple - # `exc_info` as True and falls back to sys.exc_info(), which is empty here, so the - # watcher's traceback would otherwise be lost. - LOG.warning( - f'HTTP disconnect watcher for query_data "{query_name}" failed; cancelling underlying query', - exc_info=(type(watcher_exc), watcher_exc, watcher_exc.__traceback__), - ) - else: - LOG.info(f'HTTP client disconnected during query_data "{query_name}"; cancelling underlying query') - await _drain(query_task) - raise ValueError('Query was cancelled') - - -async def _cancel_and_drain(task: asyncio.Task) -> None: - """Cancel a task and await its unwind, suppressing the resulting error. - - Draining (rather than fire-and-forget `task.cancel()`) lets any shielded - cleanup the task runs on cancellation — notably the backend `cancel_job` in - `_Workspace.execute_query` — finish before we move on. `KeyboardInterrupt` / - `SystemExit` are intentionally NOT suppressed. - - Only the expected `CancelledError` is swallowed. Any other exception raised - while the task unwinds is a real bug in the cancellation cleanup path (e.g. the - shielded backend cancel) — log it with a traceback rather than silently dropping - it, but still don't re-raise so draining stays best-effort. - """ - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - except Exception: - LOG.warning('Unexpected error while draining cancelled task', exc_info=True) - - -def _client_progress_token(ctx: Context) -> ProgressToken | None: - """Returns the progress token the client included in the original `tools/call`, or None. - - Per MCP spec, a server may only send `notifications/progress` for a request when the client - explicitly provided a `progressToken` in that request's `_meta`. Tools that fall through here - without a token must stay silent — emitting unsolicited progress can confuse strict clients. - """ - rc = ctx.request_context - if rc is None or rc.meta is None: - return None - # The MCP spec allows `_meta` to omit `progressToken`. The typed `RequestParams.Meta` - # always carries the attribute (default None), but transports that surface `_meta` as a - # plain mapping would not — so look it up defensively rather than assume the attribute. - return getattr(rc.meta, 'progressToken', None) - - -async def _emit_job_submitted_progress(ctx: Context, progress_token: ProgressToken, info: JobSubmittedInfo) -> None: - """Surfaces the backend job handle to the client so it can cancel out-of-band by POSTing to - `info.cancellation_url`. The structured data lives under `params._meta`; the human-readable - `message` is for clients that surface progress as text only and ignore `_meta`. - - We deliberately call the low-level `ctx.session.send_notification(...)` with - `related_request_id=ctx.request_id` instead of FastMCP's high-level `ctx.send_notification(...)`. - The MCP SDK's streamable_http message router (`mcp/server/streamable_http.py`, the - "Extract related_request_id from meta" branch) uses that field to pick which request's SSE - response stream receives the notification. Without it, notifications are addressed to - `GET_STREAM_KEY`, the standalone GET stream — which doesn't exist in `stateless_http=True` - mode (our deployment shape), so the notification is silently dropped and the client never - sees the job handle. `ctx.send_notification(...)` does NOT set this field, hence the bypass. - """ - params = ProgressNotificationParams.model_validate( - { - 'progressToken': progress_token, - 'progress': 0, - 'message': f'Submitted to {info.backend}', - '_meta': { - 'keboola.queryJobId': info.job_id, - 'keboola.backend': info.backend, - # `cancellation_url` may be None when a backend does not expose an out-of-band - # cancel endpoint; clients should treat the field as optional. - 'keboola.cancellationUrl': info.cancellation_url, - }, - } - ) - notification = ProgressNotification(method='notifications/progress', params=params) - # `ctx.request_id` is a property that RAISES RuntimeError when `request_context` is None — - # `getattr(ctx, 'request_id', None)` would NOT catch that (its default only suppresses - # AttributeError). Read the request id off `request_context` directly so a missing context - # yields None and we hit the graceful-skip branch below instead of raising. - rc = ctx.request_context - request_id = str(rc.request_id) if rc is not None and rc.request_id is not None else None - if request_id is None: - # Without a request id we cannot route the notification — see the docstring above for why. - # Sending with `related_request_id=None` reproduces the bug we built this fix to prevent - # (silent drop onto GET_STREAM_KEY in stateless mode). Skip the emit and warn instead so - # the failure mode is at least visible in the logs; the query itself continues normally. - LOG.warning( - f'Skipping notifications/progress for job_id={info.job_id}: request id is unavailable — ' - f'cannot route to originating SSE stream. Out-of-band cancellation will be unavailable.' - ) - return - await ctx.session.send_notification(ServerNotification(notification), related_request_id=request_id) - LOG.info( - f'Emitted notifications/progress for job_id={info.job_id} ' - f'related_request_id={request_id!r} backend={info.backend}' - ) - - -class QueryDataOutput(BaseModel): - """Output model for SQL query results.""" - - query_name: str = Field(description='The name of the executed query') - csv_data: str = Field(description='The retrieved data in CSV format') - message: str | None = Field(default=None, description='A message from the query execution') - - -def add_sql_tools(mcp: FastMCP) -> None: - """Add tools to the MCP server.""" - mcp.add_tool( - FunctionTool.from_function( - query_data, - annotations=ToolAnnotations(readOnlyHint=True), - tags={SQL_TOOLS_TAG}, - ) - ) - LOG.info('SQL tools added to the MCP server.') - - -@tool_errors() -async def query_data( - sql_query: Annotated[str, Field(description='SQL SELECT query to run.')], - query_name: Annotated[ - str, - Field( - description=( - 'A concise, human-readable name for this query based on its purpose and what data it retrieves. ' - 'Use normal words with spaces (e.g., "Customer Orders Last Month", "Top Selling Products", ' - '"User Activity Summary").' - ) - ), - ], - ctx: Context, -) -> QueryDataOutput: - """ - Executes an SQL SELECT query to get the data from the underlying database. - - BEFORE QUERYING: - * Always verify the table has a non-null fullyQualifiedName from get_tables tool. - If it does not, the table is not SQL-accessible from this workspace — do not attempt the query and inform user. - - CRITICAL SQL REQUIREMENTS: - - * ALWAYS check the SQL dialect before constructing queries. - * Do not include any comments in the SQL code - * Use delimited identifiers and FQN format for the current SQL dialect. - - TABLE AND COLUMN REFERENCES: - * Always use fully qualified table names in the exact FQN format provided by table information tools - * Follow the identifier structure exactly as shown by table info tools for the current SQL dialect - * Always use delimited identifiers when referring to table columns - - CTE (WITH CLAUSE) RULES: - * ALL column references in main query MUST match exact case used in the CTE - * If you alias a column in a CTE, reference it under the aliased name in the subsequent queries - * Define all column aliases explicitly in CTEs - * Use delimited identifiers in both CTE definition and references to preserve case - - FUNCTION COMPATIBILITY: - * Check data types before using date functions (DATE_TRUNC, EXTRACT require proper date/timestamp types) - * Cast VARCHAR columns to appropriate types before using in date/numeric functions - - ERROR PREVENTION: - * Never pass empty strings ('') where numeric or date values are expected - * Use NULLIF or CASE statements to handle empty values - * Always use TRY_CAST or similar safe casting functions when converting data types - * Check for division by zero using NULLIF(denominator, 0) - * Always use the LIMIT clause in your SELECT statements when fetching data. There are hard limits imposed - by this tool on the maximum number of rows that can be fetched and the maximum number of characters. - The tool will truncate the data if those limits are exceeded. - - DATA VALIDATION: - * When querying columns with categorical values, use query_data tool to inspect distinct values beforehand - * Ensure valid filtering by checking actual data values first - """ - workspace_manager = WorkspaceManager.from_state(ctx.session.state) - - progress_token = _client_progress_token(ctx) - - async def _on_job_submitted(info: JobSubmittedInfo) -> None: - await _emit_job_submitted_progress(ctx, progress_token, info) - - query_coro = workspace_manager.execute_query( - sql_query, - max_rows=MAX_ROWS, - max_chars=MAX_CHARS, - on_job_submitted=_on_job_submitted if progress_token is not None else None, - ) - # The disconnect race only buys us anything on the HTTP path, where the client can actually - # drop the socket (Kai kills the sandbox SDK process on STOP). With no HTTP request bound - # (stdio / background workers) nothing can disconnect, so run the query directly. - request = get_http_request_or_none() - if request is None: - result = await query_coro - else: - result = await _execute_watching_disconnect(query_coro, request, query_name) - if result.is_ok: - LOG.info(' '.join(filter(None, [f'Query "{query_name}" executed successfully.', result.message]))) - if result.data: - data = result.data - else: - # non-SELECT query, this should not really happen, because this tool is for running SELECT queries - data = SqlSelectData(columns=['message'], rows=[{'message': result.message}]) - - # Convert to CSV - output = StringIO() - writer = csv.DictWriter(output, fieldnames=data.columns) - writer.writeheader() - writer.writerows(data.rows) - - return QueryDataOutput(query_name=query_name, csv_data=output.getvalue(), message=result.message) - - else: - # Surface cancellation cleanly: the workspace already produced a precise message - # ("Query was cancelled") for the cancel-by-client case, so don't wrap it in a - # generic "Failed to run SQL query, error: ..." prefix that hides what happened. - # A client-initiated cancel is expected, so log it at INFO; genuine failures at WARNING. - if result.message == 'Query was cancelled': - LOG.info(f'Query "{query_name}" was cancelled.') - raise ValueError('Query was cancelled') - LOG.warning(' '.join(filter(None, [f'Query "{query_name}" failed.', result.message]))) - raise ValueError(f'Failed to run SQL query, error: {result.message}') diff --git a/src/keboola_mcp_server/tools/storage/__init__.py b/src/keboola_mcp_server/tools/storage/__init__.py deleted file mode 100644 index 5d9c898eb..000000000 --- a/src/keboola_mcp_server/tools/storage/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from keboola_mcp_server.tools.storage.tools import STORAGE_TOOLS_TAG, add_storage_tools - -__all__ = ['add_storage_tools', 'STORAGE_TOOLS_TAG'] diff --git a/src/keboola_mcp_server/tools/storage/tools.py b/src/keboola_mcp_server/tools/storage/tools.py deleted file mode 100644 index 12afce05c..000000000 --- a/src/keboola_mcp_server/tools/storage/tools.py +++ /dev/null @@ -1,1077 +0,0 @@ -"""Storage-related tools for the MCP server (buckets, tables, etc.).""" - -import logging -from collections import defaultdict -from datetime import datetime -from typing import Annotated, Any, Iterable, Literal, Sequence, cast - -from fastmcp import Context -from fastmcp.tools import FunctionTool -from mcp.types import ToolAnnotations -from pydantic import AliasChoices, BaseModel, Field, SerializeAsAny, field_serializer, model_validator - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import KeboolaClient, get_metadata_property -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.errors import tool_errors -from keboola_mcp_server.links import Link, ProjectLinksManager -from keboola_mcp_server.mcp import ( - KeboolaMcpServer, - process_concurrently, - toon_serializer, - toon_serializer_compact, - unwrap_results, -) -from keboola_mcp_server.tools.components.utils import get_nested -from keboola_mcp_server.tools.storage.usage import ( - ComponentUsageReference, - find_id_usage, - get_created_by, - get_last_updated_by, -) -from keboola_mcp_server.tools.storage_helpers import ( - has_storage_branches, - merged_bucket_detail, - merged_bucket_list, - merged_table_detail, -) -from keboola_mcp_server.utils import parse_iso_timestamp -from keboola_mcp_server.workspace import WorkspaceManager - -LOG = logging.getLogger(__name__) - -STORAGE_TOOLS_TAG = 'storage' - -BUCKET_ID_PARTS = 2 -TABLE_ID_PARTS = 3 -COLUMN_ID_PARTS = 4 - - -def add_storage_tools(mcp: KeboolaMcpServer) -> None: - """Adds tools to the MCP server.""" - mcp.add_tool( - FunctionTool.from_function( - get_buckets, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={STORAGE_TOOLS_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - get_tables, - annotations=ToolAnnotations(readOnlyHint=True), - serializer=toon_serializer_compact, - tags={STORAGE_TOOLS_TAG}, - ) - ) - mcp.add_tool( - FunctionTool.from_function( - update_descriptions, - annotations=ToolAnnotations(destructiveHint=True), - serializer=toon_serializer, - tags={STORAGE_TOOLS_TAG}, - ) - ) - - LOG.info('Storage tools added to the MCP server.') - - -def _sum(a: int | None, b: int | None) -> int | None: - if a is None and b is None: - return None - else: - return (a or 0) + (b or 0) - - -def _max_timestamp(*timestamps: str | None) -> str | None: - """Return the most recent timestamp from the given ISO 8601 strings, or None if all are None.""" - valid = [ts for ts in timestamps if ts] - if not valid: - return None - - def _parse(ts: str) -> tuple: - try: - return (1, parse_iso_timestamp(ts)) - except ValueError: - return (0, ts) - - return max(valid, key=_parse) - - -class BucketDetail(BaseModel): - id: str = Field(description='Unique identifier for the bucket.') - name: str = Field(description='Name of the bucket.') - display_name: str = Field( - description='The display name of the bucket.', - validation_alias=AliasChoices('displayName', 'display_name', 'display-name'), - serialization_alias='displayName', - ) - description: str | None = Field(None, description='Description of the bucket.') - stage: str = Field(description='Stage of the bucket (in for input stage, out for output stage).') - created: str = Field(description='Creation timestamp of the bucket.') - updated: str | None = Field(default=None, description='Timestamp of the most recent change to the bucket.') - data_size_bytes: int | None = Field( - None, - description='Total data size of the bucket in bytes.', - validation_alias=AliasChoices('dataSizeBytes', 'data_size_bytes', 'data-size-bytes'), - serialization_alias='dataSizeBytes', - ) - tables_count: int | None = Field( - default=None, - description='Number of tables in the bucket.', - validation_alias=AliasChoices('tablesCount', 'tables_count', 'tables-count'), - serialization_alias='tablesCount', - ) - links: list[Link] | None = Field(default=None, description='The links relevant to the bucket.') - source_project: str | None = Field( - default=None, description='The source Keboola project of the linked bucket, None otherwise.' - ) - created_by: ComponentUsageReference | None = Field( - default=None, description='Configuration that created the bucket (component/config ID and timestamp).' - ) - last_updated_by: ComponentUsageReference | None = Field( - default=None, description='Configuration that last updated the bucket (component/config ID and timestamp).' - ) - - # these are internal fields not meant to be exposed to LLMs - branch_id: str | None = Field(default=None, exclude=True, description='The ID of the branch the bucket belongs to.') - prod_id: str = Field(default='', exclude=True, description='The ID of the production branch bucket.') - # TODO: add prod_name too to strip the '{branch_id}-' prefix from the name' - backend_path: list[str] | None = Field(default=None, exclude=True) - - def shade_by( - self, - other: 'BucketDetail', - branch_id: str | None, - links: list[Link] | None = None, - storage_branches: bool = False, - ) -> 'BucketDetail': - if self.branch_id: - raise ValueError( - f'Dev branch buckets cannot be shaded: ' f'bucket.id={self.id}, bucket.branch_id={self.branch_id}' - ) - if not other.branch_id: - raise ValueError( - f'Prod branch buckets cannot shade others: ' f'bucket.id={other.id}, bucket.branch_id={other.branch_id}' - ) - if other.branch_id != branch_id: - raise ValueError( - f'Dev branch mismatch: ' - f'bucket.id={other.id}, bucket.branch_id={other.branch_id}, branch_id={branch_id}' - ) - if other.prod_id != self.id: - raise ValueError(f'Prod and dev buckets mismatch: prod_bucket.id={self.id}, dev_bucket.id={other.id}') - if storage_branches: - # With storage-branches the branched bucket is independent; use its values directly - changes: dict[str, int | None | list[Link] | str] = { - 'data_size_bytes': other.data_size_bytes, - 'tables_count': _sum(self.tables_count, other.tables_count), - } - else: - changes = { - # TODO: These bytes and counts are approximated by summing the values of the two buckets. - 'data_size_bytes': _sum(self.data_size_bytes, other.data_size_bytes), - 'tables_count': _sum(self.tables_count, other.tables_count), - } - if links is not None: - changes['links'] = links if links else None - return self.model_copy(update=changes) - - def with_lineage_metadata(self, values: dict[str, Any]) -> 'BucketDetail': - metadata = values.get('metadata', []) - if not metadata or not isinstance(metadata, list): - return self - last_updated_by = get_last_updated_by(metadata) - return self.model_copy( - update={ - 'created_by': get_created_by(metadata), - 'last_updated_by': last_updated_by, - 'updated': _max_timestamp( - self.updated, - last_updated_by.timestamp if last_updated_by else None, - ), - } - ) - - @model_validator(mode='before') - @classmethod - def set_updated(cls, values: dict[str, Any]) -> dict[str, Any]: - if not values.get('updated'): - values['updated'] = _max_timestamp(values.get('lastChangeDate')) - return values - - @model_validator(mode='before') - @classmethod - def set_table_count(cls, values: dict[str, Any]) -> dict[str, Any]: - if isinstance(values.get('tables'), list): - values['tables_count'] = len(values['tables']) - else: - values['tables_count'] = None - return values - - @model_validator(mode='before') - @classmethod - def set_description(cls, values: dict[str, Any]) -> dict[str, Any]: - # KBC metadata holds the curated, user-editable description (update_descriptions writes here). - # The legacy top-level `description` field is auto-generated (e.g. "Bucket created by - # Transformation API") and stale for linked/shared buckets, so metadata must take precedence. - metadata = values.get('metadata', []) - description = ( - get_metadata_property(metadata, MetadataField.SHARED_DESCRIPTION) - or get_metadata_property(metadata, MetadataField.DESCRIPTION) - or values.get('description') - ) - values['description'] = description or None - return values - - @model_validator(mode='before') - @classmethod - def set_branch_id(cls, values: dict[str, Any]) -> dict[str, Any]: - branch_id = get_metadata_property(values.get('metadata', []), MetadataField.FAKE_DEVELOPMENT_BRANCH) - if branch_id: - values['branch_id'] = branch_id - values['prod_id'] = values['id'].replace(f'c-{branch_id}-', 'c-') - else: - values['branch_id'] = None - values['prod_id'] = values['id'] - return values - - @model_validator(mode='before') - @classmethod - def set_source_project(cls, values: dict[str, Any]) -> dict[str, Any]: - if source_project_raw := cast(dict[str, Any], get_nested(values, 'sourceBucket.project')): - values['source_project'] = f'{source_project_raw["name"]} (ID: {source_project_raw["id"]})' - return values - - @model_validator(mode='before') - @classmethod - def set_backend_path(cls, values: dict[str, Any]) -> dict[str, Any]: - raw = values.get('backendPath') - values['backend_path'] = raw if isinstance(raw, list) else None - return values - - -class BucketCounts(BaseModel): - total_buckets: int = Field(description='Total number of buckets.') - input_buckets: int = Field(description='Number of input stage buckets.') - output_buckets: int = Field(description='Number of output stage buckets.') - - -class GetBucketsOutput(BaseModel): - buckets: list[BucketDetail] = Field(description='List of buckets.') - links: list[Link] = Field(description='Links relevant to the bucket listing.') - buckets_not_found: list[str] | None = Field(default=None, description='List of bucket IDs that were not found.') - bucket_counts: BucketCounts | None = Field(default=None, description='Bucket counts by stage.') - - def pack_links(self) -> 'GetBucketsOutput': - """Move links from particular BucketDetail objects to GetBucketsOutput object to optimize TOON serialization.""" - buckets: list[BucketDetail] = [] - links: set[Link] = set() - for bucket in self.buckets: - links.update(bucket.links or []) - buckets.append(bucket.model_copy(update={'links': None})) - links.update(self.links) - - return self.model_copy(update={'buckets': buckets, 'links': sorted(links, key=lambda x: (x.type, x.title))}) - - -class TableColumnInfo(BaseModel): - name: str = Field(description='Plain name of the column.') - quoted_name: str = Field( - description='The properly quoted name of the column.', - validation_alias=AliasChoices('quotedName', 'quoted_name', 'quoted-name'), - serialization_alias='quotedName', - ) - database_native_type: str = Field(description='The native, backend-specific data type.') - nullable: bool = Field(description='Whether the column can contain null values.') - keboola_base_type: str | None = Field(default=None, description='The storage backend agnostic data type.') - description: str | None = Field(default=None, description='Description of the column.') - - -class TableSummary(BaseModel): - """Listing-level view of a table. - - Carries only the fields available when enumerating tables in a bucket (no warehouse - round-trip). Notably it does NOT declare ``fully_qualified_name`` or ``columns`` — those - require fetching a single table's detail. Emitting a ``null`` FQN here would be misleading - (per the query_data queryability rule, ``null`` means "not queryable"), so the field is - simply absent from summaries. See ``TableDetail`` for the full per-table view. - """ - - id: str = Field(description='Unique identifier for the table.') - name: str = Field(description='Name of the table.') - display_name: str = Field( - description='The display name of the table.', - validation_alias=AliasChoices('displayName', 'display_name', 'display-name'), - serialization_alias='displayName', - ) - description: str | None = Field(default=None, description='Description of the table.') - primary_key: list[str] | None = Field( - default=None, - description='List of primary key columns.', - validation_alias=AliasChoices('primaryKey', 'primary_key', 'primary-key'), - serialization_alias='primaryKey', - ) - created: str | None = Field(default=None, description='Creation timestamp of the table.') - updated: str | None = Field(default=None, description='Timestamp of the most recent change to the table.') - rows_count: int | None = Field( - default=None, - description='Number of rows in the table.', - validation_alias=AliasChoices('rowsCount', 'rows_count', 'rows-count'), - serialization_alias='rowsCount', - ) - data_size_bytes: int | None = Field( - default=None, - description='Total data size of the table in bytes.', - validation_alias=AliasChoices('dataSizeBytes', 'data_size_bytes', 'data-size-bytes'), - serialization_alias='dataSizeBytes', - ) - links: list[Link] | None = Field(default=None, description='The links relevant to the table.') - source_project: str | None = Field( - default=None, description='The source Keboola project of the linked table, None otherwise.' - ) - - # these are internal fields not meant to be exposed to LLMs - branch_id: str | None = Field(default=None, exclude=True, description='The ID of the branch the bucket belongs to.') - prod_id: str = Field(default='', exclude=True, description='The ID of the production branch bucket.') - - @model_validator(mode='before') - @classmethod - def set_updated(cls, values: dict[str, Any]) -> dict[str, Any]: - if not values.get('updated'): - values['updated'] = _max_timestamp( - values.get('lastChangeDate'), - values.get('lastImportDate'), - ) - return values - - @model_validator(mode='before') - @classmethod - def set_description(cls, values: dict[str, Any]) -> dict[str, Any]: - # KBC.description metadata holds the curated, user-editable description and must win over the - # legacy top-level `description` field, which is auto-generated and stale for linked tables. - description = ( - get_metadata_property(values.get('metadata', []), MetadataField.DESCRIPTION) - or get_metadata_property(get_nested(values, 'sourceTable.metadata', default=[]), MetadataField.DESCRIPTION) - or values.get('description') - ) - values['description'] = description or None - return values - - @model_validator(mode='before') - @classmethod - def set_branch_id(cls, values: dict[str, Any]) -> dict[str, Any]: - branch_id = get_metadata_property(values.get('metadata', []), MetadataField.FAKE_DEVELOPMENT_BRANCH) - if branch_id: - values['branch_id'] = branch_id - values['prod_id'] = values['id'].replace(f'c-{branch_id}-', 'c-') - else: - values['branch_id'] = None - values['prod_id'] = values['id'] - return values - - @model_validator(mode='before') - @classmethod - def set_source_project(cls, values: dict[str, Any]) -> dict[str, Any]: - if source_project_raw := cast(dict[str, Any], get_nested(values, 'sourceTable.project')): - values['source_project'] = f'{source_project_raw["name"]} (ID: {source_project_raw["id"]})' - return values - - @field_serializer('primary_key') - # Serialize the primary key as a string so the whole table is serialized - # as tabular data in Toon format. - def serialize_primary_key(self, primary_key: list[str] | None) -> str | None: - return '|'.join(primary_key) if primary_key else None - - -class TableDetail(TableSummary): - """Full per-table view returned when fetching a specific table by ID. - - Extends ``TableSummary`` with the fields that require a dedicated table-detail fetch - (and, for the FQN, resolving the warehouse backend path): column definitions, the fully - qualified database name, usage references, and create/update lineage. - """ - - columns: list[TableColumnInfo] | None = Field( - default=None, - description='List of column information including database identifiers.', - ) - fully_qualified_name: str | None = Field( - default=None, - description='Fully qualified name of the table.', - validation_alias=AliasChoices('fullyQualifiedName', 'fully_qualified_name', 'fully-qualified-name'), - serialization_alias='fullyQualifiedName', - ) - used_by: list[ComponentUsageReference] | None = Field( - default=None, description='The components / transformations that use the table.' - ) - created_by: ComponentUsageReference | None = Field( - default=None, description='Configuration that created the table (component/config ID and timestamp).' - ) - last_updated_by: ComponentUsageReference | None = Field( - default=None, description='Configuration that last updated the table (component/config ID and timestamp).' - ) - - def with_lineage_metadata(self, values: dict[str, Any]) -> 'TableDetail': - metadata = values.get('metadata', []) - if not metadata or not isinstance(metadata, list): - return self - last_updated_by = get_last_updated_by(metadata) - return self.model_copy( - update={ - 'created_by': get_created_by(metadata), - 'last_updated_by': last_updated_by, - 'updated': _max_timestamp( - self.updated, - last_updated_by.timestamp if last_updated_by else None, - ), - } - ) - - -class GetTablesOutput(BaseModel): - # SerializeAsAny so detail instances serialize with their full field set (columns, - # fullyQualifiedName, ...) while summaries (from bucket listing) serialize as the - # TableSummary subset — omitting fully_qualified_name rather than emitting it as null. - tables: list[SerializeAsAny[TableSummary]] = Field(description='List of tables.') - links: list[Link] = Field(description='Links relevant to the table listing.') - tables_not_found: list[str] | None = Field(default=None, description='List of table IDs that were not found.') - - def pack_links(self) -> 'GetTablesOutput': - """Move links from particular table objects to GetTablesOutput object to optimize TOON serialization.""" - tables: list[TableDetail | TableSummary] = [] - links: set[Link] = set() - for table in self.tables: - links.update(table.links or []) - tables.append(table.model_copy(update={'links': None})) - links.update(self.links) - - return self.model_copy(update={'tables': tables, 'links': sorted(links, key=lambda x: (x.type, x.title))}) - - -class UpdateItemResult(BaseModel): - item_id: str = Field(description='The storage item identifier that was updated.') - success: bool = Field(description='Whether the update succeeded.') - error: str | None = Field(default=None, description='Error message if the update failed.') - timestamp: datetime | None = Field(default=None, description='Timestamp of the update if successful.') - - -class UpdateDescriptionsOutput(BaseModel): - results: list[UpdateItemResult] = Field(description='Results for each update attempt.') - total_processed: int = Field(description='Total number of items processed.') - successful: int = Field(description='Number of successful updates.') - failed: int = Field(description='Number of failed updates.') - - -class DescriptionUpdate(BaseModel): - """Structured update describing a storage item and its new description.""" - - item_id: str = Field( - description='Storage item name: "bucket_id", "bucket_id.table_id", "bucket_id.table_id.column_name"' - ) - description: str = Field(description='New description to set for the storage item.') - - -class StorageItemId(BaseModel): - """Represents a parsed storage item ID.""" - - item_type: Literal['bucket', 'table', 'column'] = Field(description='Type of storage item.') - bucket_id: str | None = Field(default=None, description='Bucket identifier.') - table_id: str | None = Field(default=None, description='Table identifier.') - column_name: str | None = Field(default=None, description='Column name.') - - -class DescriptionUpdateGroups(BaseModel): - """Groups description updates by type.""" - - bucket_updates: dict[str, str] = Field(description='Bucket description updates by bucket ID.') - table_updates: dict[str, str] = Field(description='Table description updates by table ID.') - column_updates_by_table: dict[str, dict[str, str]] = Field(description='Column updates by table ID.') - - -async def _find_buckets(client: KeboolaClient, bucket_id: str) -> tuple[BucketDetail | None, BucketDetail | None]: - prod_raw, dev_raw = await merged_bucket_detail(client, bucket_id) - - prod_bucket: BucketDetail | None = None - dev_bucket: BucketDetail | None = None - - if prod_raw: - bucket = BucketDetail.model_validate(prod_raw).with_lineage_metadata(prod_raw) - if not bucket.branch_id: - prod_bucket = bucket - elif bucket.branch_id == client.branch_id: - dev_bucket = bucket - - if dev_raw: - bucket = BucketDetail.model_validate(dev_raw).with_lineage_metadata(dev_raw) - if bucket.branch_id == client.branch_id: - dev_bucket = bucket - elif not bucket.branch_id and not prod_bucket: - prod_bucket = bucket - - # Legacy: if user passed a dev-style bucket_id, try to find the prod version - if not prod_bucket and client.branch_id and f'.c-{client.branch_id}-' in bucket_id: - from keboola_mcp_server.tools.storage_helpers import _safe_bucket_detail - - prod_id = bucket_id.replace(f'c-{client.branch_id}-', 'c-') - if raw := await _safe_bucket_detail(client, prod_id, branch_id='default'): - bucket = BucketDetail.model_validate(raw).with_lineage_metadata(raw) - if not bucket.branch_id: - prod_bucket = bucket - - return prod_bucket, dev_bucket - - -async def _combine_buckets( - client: KeboolaClient, - links_manager: ProjectLinksManager | None, - prod_bucket: BucketDetail | None, - dev_bucket: BucketDetail | None, - storage_branches: bool = False, -) -> BucketDetail: - def _links(_id: str, name: str) -> list[Link] | None: - if links_manager: - return [links_manager.get_bucket_detail_link(_id, name)] - else: - return None - - if prod_bucket and dev_bucket: - # generate a URL link to the dev bucket but with the prod bucket's name - links = _links(dev_bucket.id, prod_bucket.name or prod_bucket.id) - bucket = prod_bucket.shade_by(dev_bucket, client.branch_id, links or [], storage_branches=storage_branches) - elif prod_bucket: - links = _links(prod_bucket.id, prod_bucket.name or prod_bucket.id) - bucket = prod_bucket.model_copy(update={'links': links}) - elif dev_bucket: - links = _links(dev_bucket.id, dev_bucket.name or dev_bucket.id) - bucket = dev_bucket.model_copy(update={'id': dev_bucket.prod_id, 'branch_id': None, 'links': links}) - else: - raise ValueError('No buckets specified.') - - return bucket - - -@tool_errors() -async def get_buckets( - ctx: Context, bucket_ids: Annotated[Sequence[str], Field(description='Filter by specific bucket IDs.')] = tuple() -) -> GetBucketsOutput: - """ - Lists buckets or retrieves full details of specific buckets, including descriptions, - lineage references (created/updated by), and links. - - WHEN NOT TO USE: - - Do NOT call with `bucket_ids=[]` just to find a bucket by name. Use `search` with - item_types=["bucket"] instead. - - Only use `bucket_ids=[]` when you need a complete inventory of all buckets in the project. - - EXAMPLES: - - `bucket_ids=[]` → summaries of all buckets in the project - - `bucket_ids=["id1", ...]` → full details of the buckets with the specified IDs - """ - client = KeboolaClient.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - if bucket_ids: - has_sb = await has_storage_branches(client) - - async def _fetch_bucket_detail(bucket_id: str) -> BucketDetail | str: - prod_bucket, dev_bucket = await _find_buckets(client, bucket_id) - if prod_bucket or dev_bucket: - return await _combine_buckets(client, links_manager, prod_bucket, dev_bucket, storage_branches=has_sb) - else: - return bucket_id - - results = await process_concurrently(bucket_ids, _fetch_bucket_detail) - buckets: list[BucketDetail] = [] - missing_ids: list[str] = [] - - for bucket_detail_or_id in unwrap_results(results, 'Failed to fetch one or more buckets'): - if isinstance(bucket_detail_or_id, BucketDetail): - buckets.append(bucket_detail_or_id) - elif isinstance(bucket_detail_or_id, str): - missing_ids.append(bucket_detail_or_id) - - output = GetBucketsOutput( - buckets=buckets, - buckets_not_found=missing_ids if missing_ids else None, - links=[links_manager.get_bucket_dashboard_link()], - ) - - else: - output = await _list_buckets(client, links_manager) - - return output.pack_links() - - -async def _list_buckets(client: KeboolaClient, links_manager: ProjectLinksManager) -> GetBucketsOutput: - """Retrieves information about all buckets in the project.""" - has_sb = await has_storage_branches(client) - raw_bucket_data = await merged_bucket_list(client, include=['metadata', 'linkedBuckets']) - - # group buckets by their ID as it would appear on the production branch - buckets_by_prod_id: dict[str, list[BucketDetail]] = defaultdict(list) - for raw in raw_bucket_data: - bucket = BucketDetail.model_validate(raw) - if bucket.branch_id and bucket.branch_id != client.branch_id: - # a dev branch bucket from a different branch - continue - buckets_by_prod_id[bucket.prod_id].append(bucket) - - buckets: list[BucketDetail] = [] - for prod_id, group in buckets_by_prod_id.items(): - prod_bucket: BucketDetail | None = None - dev_buckets: list[BucketDetail] = [] - for b in group: - if b.branch_id: - dev_buckets.append(b) - else: - prod_bucket = b - - if not prod_bucket and not dev_buckets: - # should not happen - raise Exception(f'No buckets in the group: prod_id={prod_id}') - - else: - bucket = await _combine_buckets( - client, links_manager, prod_bucket, next(iter(dev_buckets), None), storage_branches=has_sb - ) - buckets.append(bucket) - - # Count buckets by stage (only count input, derive output) - total_count = len(buckets) - input_count = sum(1 for bucket in buckets if bucket.stage == 'in') - output_count = total_count - input_count - - bucket_counts = BucketCounts(total_buckets=total_count, input_buckets=input_count, output_buckets=output_count) - - return GetBucketsOutput( - buckets=buckets, bucket_counts=bucket_counts, links=[links_manager.get_bucket_dashboard_link()] - ) - - -@tool_errors() -async def get_tables( - ctx: Context, - bucket_ids: Annotated[Sequence[str], Field(description='Filter by specific bucket IDs.')] = tuple(), - table_ids: Annotated[Sequence[str], Field(description='Filter by specific table IDs.')] = tuple(), - include_usage: Annotated[ - bool, - Field(description=('Show components / transformations where each table is used.')), - ] = False, -) -> GetTablesOutput: - """ - Lists tables in buckets or retrieves full details of specific tables, including fully qualified database name, - column definitions, lineage references (created/updated by) and links. - - WHEN NOT TO USE: - - Do NOT list tables across buckets just to find a table by name. Use `search` with - item_types=["table"] instead — it also matches column names and descriptions. - - Only use `bucket_ids` listing when you need all tables in specific known buckets. - - RETURNS: - - With `bucket_ids`: Summaries of tables (ID, name, description, primary key). - - With `table_ids`: Full details including columns, data types, and fully qualified database names. - - With `table_ids` and `include_usage`: Full details plus components / transformations that use the tables - in their input / output mappings. Use only when explicitly needed or evident from context; usage calculation - might be demanding in big projects. - - COLUMN DATA TYPES: - - database_native_type: The actual type in the storage backend (Snowflake, BigQuery, etc.) - with precision, scale, and other implementation details - - keboola_base_type: Standardized type indicating the semantic data type. May not always be - available. When present, it reveals the actual type of data stored in the column - for example, - a column with database_native_type VARCHAR might have keboola_base_type INTEGER, indicating - it stores integer values despite being stored as text in the backend. - - QUERYABILITY RULE: - - A table is directly queryable via query_data tool only if fullyQualifiedName is present and non-null - in the response. - - If fullyQualifiedName is absent or null (e.g. for linked/alias tables from other projects), - the table cannot be queried via SQL from this workspace. - - Do not attempt to construct or guess the FQN — it will not work. In that case, - inform the user of the limitation immediately. - - EXAMPLES: - - `bucket_ids=["id1", ...]` → summary info of the tables in the buckets with the specified IDs - - `table_ids=["id1", ...]` → detailed info of the tables specified by their IDs - - `bucket_ids=[]` and `table_ids=[]` → empty list; you have to specify at least one filter - - """ - client = KeboolaClient.from_state(ctx.session.state) - workspace_manager = WorkspaceManager.from_state(ctx.session.state) - links_manager = await ProjectLinksManager.from_client(client) - - tables_by_id: dict[str, TableDetail | TableSummary] = {} - missing_ids: list[str] = [] - - if bucket_ids: - for table in await _list_tables(bucket_ids, client, links_manager): - tables_by_id[table.id] = table - - if table_ids: - - async def _fetch_table_detail(_table_id: str) -> TableDetail | str: - if _table := await _get_table(_table_id, client, workspace_manager, links_manager): - return _table - else: - return _table_id - - # Touch the WorkspaceManager to initialize the workspace before launching the concurrent tasks - # to prevent race condition and initializing multiple database backend workspaces. - _ = await workspace_manager.get_workspace_id() - results = await process_concurrently(table_ids, _fetch_table_detail) - - for table_detail_or_id in unwrap_results(results, 'Failed to fetch one or more tables'): - if isinstance(table_detail_or_id, TableDetail): - tables_by_id[table_detail_or_id.id] = table_detail_or_id - elif isinstance(table_detail_or_id, str): - missing_ids.append(table_detail_or_id) - - # Add the component usage to the table detail - if include_usage: - prod_ids_to_ids = {table.prod_id: table.id for table in tables_by_id.values()} - usage_by_ids = await find_id_usage( - client, - list(prod_ids_to_ids.keys()), - ['configuration', 'configuration-row', 'transformation'], - ['storage.input', 'storage.output'], - ) - # Initialize the used_by list for all tables to avoid None values which could confuse the model. - # Usage only applies to full table details; summaries (from bucket listing) carry no used_by. - for table in tables_by_id.values(): - if isinstance(table, TableDetail): - table.used_by = [] - for id_usage in usage_by_ids: - table_id = prod_ids_to_ids.get(id_usage.target_id) - if table_id and isinstance(table := tables_by_id[table_id], TableDetail): - table.used_by = id_usage.usage_references - elif not table_id: - LOG.error(f'Target ID has changed during searching for usage: prod_id={id_usage.target_id}.') - - return GetTablesOutput( - tables=list(tables_by_id.values()), - tables_not_found=missing_ids if missing_ids else None, - links=[links_manager.get_bucket_dashboard_link()], - ).pack_links() - - -async def _get_table( - table_id: str, - client: KeboolaClient, - workspace_manager: WorkspaceManager, - links_manager: ProjectLinksManager, -) -> TableDetail | None: - prod_table, dev_table = await merged_table_detail(client, table_id) - - # Validate metadata: prod should not have branch metadata, dev should match our branch - if prod_table: - branch_id = get_metadata_property(prod_table.get('metadata', []), MetadataField.FAKE_DEVELOPMENT_BRANCH) - if branch_id: - prod_table = None - - if dev_table: - branch_id = get_metadata_property(dev_table.get('metadata', []), MetadataField.FAKE_DEVELOPMENT_BRANCH) - if branch_id != client.branch_id: - dev_table = None - - raw_table = dev_table or prod_table - if not raw_table: - return None - - raw_columns = cast(list[str], raw_table.get('columns', [])) - raw_column_metadata = cast(dict[str, list[dict[str, Any]]], raw_table.get('columnMetadata', {})) - raw_source_column_metadata = cast( - dict[str, list[dict[str, Any]]], get_nested(raw_table, 'sourceTable.columnMetadata', default={}) - ) - - sql_dialect = await workspace_manager.get_sql_dialect() - db_table_info = await workspace_manager.get_table_info(raw_table) - - column_info = [] - for col_name in raw_columns: - col_meta = raw_column_metadata.get(col_name, []) - source_col_meta = raw_source_column_metadata.get(col_name, []) - - description: str | None = get_metadata_property(col_meta, MetadataField.DESCRIPTION) - if not description: - description = get_metadata_property(source_col_meta, MetadataField.DESCRIPTION) - - base_type: str | None = get_metadata_property( - col_meta, MetadataField.DATATYPE_BASETYPE, preferred_providers=['user'] - ) - if not base_type: - base_type = get_metadata_property( - source_col_meta, MetadataField.DATATYPE_BASETYPE, preferred_providers=['user'] - ) - - native_type: str | None = get_metadata_property(col_meta, MetadataField.DATATYPE_TYPE) - if not native_type: - native_type = get_metadata_property(source_col_meta, MetadataField.DATATYPE_TYPE) - - nullable_str: str | None = get_metadata_property(col_meta, MetadataField.DATATYPE_NULLABLE) - if not nullable_str: - nullable_str = get_metadata_property(source_col_meta, MetadataField.DATATYPE_NULLABLE) - - if native_type is None: - native_type = 'STRING' if sql_dialect == 'BigQuery' else 'VARCHAR' - LOG.warning( - f'No KBC.datatype.type in columnMetadata: ' - f'col_name={col_name}, sql_dialect={sql_dialect}, table_id={table_id}' - ) - - # KBC.datatype.nullable is stored as '1'/'true' for nullable, '0'/'false' (or absent) for non-nullable - nullable = str(nullable_str).lower() in ('1', 'true') if nullable_str is not None else False - - column_info.append( - TableColumnInfo( - name=col_name, - quoted_name=await workspace_manager.get_quoted_name(col_name), - database_native_type=native_type, - nullable=nullable, - keboola_base_type=base_type, - description=description, - ) - ) - - bucket_info = cast(dict[str, Any], raw_table.get('bucket', {})) - bucket_id = cast(str, bucket_info.get('id', '')) - - table_name = cast(str, raw_table.get('name', '')) - links = [links_manager.get_table_detail_link(bucket_id, table_name)] - table = TableDetail.model_validate( - raw_table - | { - 'columns': column_info, - 'fully_qualified_name': db_table_info.fqn.identifier if db_table_info else None, - 'links': links, - } - ).with_lineage_metadata(raw_table) - return table.model_copy(update={'id': table.prod_id, 'branch_id': None}) - - -async def _list_tables( - bucket_ids: Sequence[str], - client: KeboolaClient, - links_manager: ProjectLinksManager, -) -> Iterable[TableSummary]: - """Retrieves all tables in a specific bucket with their basic (summary) information. - - Listing does not resolve the warehouse FQN or column details, so this returns - ``TableSummary`` objects, which omit ``fully_qualified_name`` and ``columns`` rather than - emitting them as null. Fetch a specific table by ID (``_get_table``) for the full detail. - """ - has_sb = await has_storage_branches(client) - tables_by_prod_id: dict[str, TableSummary] = {} - sapi_includes = ['metadata', 'columnMetadata', 'sourceMetadata', 'sourceColumnMetadata'] - - for bucket_id in bucket_ids: - prod_bucket, dev_bucket = await _find_buckets(client, bucket_id) - - if prod_bucket: - raw_table_data = await client.storage_client.bucket_table_list( - prod_bucket.id, include=sapi_includes, branch_id='default' - ) - for raw in raw_table_data: - table_name = cast(str, raw.get('name', '')) - table = TableSummary.model_validate( - raw | {'links': [links_manager.get_table_detail_link(prod_bucket.id, table_name)]} - ) - assert table.id == table.prod_id, f'Table ID mismatch: {table.id} != {table.prod_id}' - tables_by_prod_id[table.id] = table - - if dev_bucket: - dev_branch_id = client.branch_id if has_sb else 'default' - raw_table_data = await client.storage_client.bucket_table_list( - dev_bucket.id, include=sapi_includes, branch_id=dev_branch_id - ) - for raw in raw_table_data: - table = TableSummary.model_validate(raw) - tables_by_prod_id[table.prod_id] = table.model_copy( - update={ - 'id': table.prod_id, - 'branch_id': None, - 'links': [links_manager.get_table_detail_link(dev_bucket.id, table.name)], - } - ) - - return tables_by_prod_id.values() - - -def _parse_item_id(item_id: str) -> StorageItemId: - """ - Parse an item_id string to extract item type and identifiers. - - :param item_id: Item ID (e.g., "in.c-bucket", "in.c-bucket.table", "in.c-bucket.table.column") - :return: StorageItemId object with structured data - """ - if not item_id.startswith(('in.', 'out.')): - raise ValueError(f'Invalid item_id format: {item_id} - must start with in. or out.') - - parts = item_id.split('.') - - if len(parts) == BUCKET_ID_PARTS: - return StorageItemId(item_type='bucket', bucket_id=item_id) - elif len(parts) == TABLE_ID_PARTS: - bucket_id = f'{parts[0]}.{parts[1]}' - return StorageItemId(item_type='table', bucket_id=bucket_id, table_id=item_id) - elif len(parts) == COLUMN_ID_PARTS: - bucket_id = f'{parts[0]}.{parts[1]}' - table_id = f'{parts[0]}.{parts[1]}.{parts[2]}' - return StorageItemId(item_type='column', bucket_id=bucket_id, table_id=table_id, column_name=parts[3]) - else: - raise ValueError(f'Invalid item_id format: {item_id}') - - -def _group_updates_by_type(updates: list[DescriptionUpdate]) -> DescriptionUpdateGroups: - """Group updates by type for efficient processing.""" - bucket_updates: dict[str, str] = {} - table_updates: dict[str, str] = {} - column_updates_by_table: dict[str, dict[str, str]] = defaultdict(dict) - - for update in updates: - parsed = _parse_item_id(update.item_id) - - if parsed.item_type == 'bucket': - bucket_updates[parsed.bucket_id] = update.description - elif parsed.item_type == 'table': - table_updates[parsed.table_id] = update.description - elif parsed.item_type == 'column': - column_updates_by_table[parsed.table_id][parsed.column_name] = update.description - - return DescriptionUpdateGroups( - bucket_updates=bucket_updates, - table_updates=table_updates, - column_updates_by_table=dict(column_updates_by_table), - ) - - -async def _update_bucket_description(client: KeboolaClient, bucket_id: str, description: str) -> UpdateItemResult: - """Update a bucket description.""" - try: - response = await client.storage_client.bucket_metadata_update( - bucket_id=bucket_id, - metadata={MetadataField.DESCRIPTION: description}, - ) - description_entry = next(entry for entry in response if entry.get('key') == MetadataField.DESCRIPTION) - return UpdateItemResult(item_id=bucket_id, success=True, timestamp=description_entry['timestamp']) - except Exception as e: - return UpdateItemResult(item_id=bucket_id, success=False, error=str(e)) - - -async def _update_table_description(client: KeboolaClient, table_id: str, description: str) -> UpdateItemResult: - """Update a table description.""" - try: - response = await client.storage_client.table_metadata_update( - table_id=table_id, - metadata={MetadataField.DESCRIPTION: description}, - columns_metadata={}, - ) - raw_metadata = cast(list[JsonDict], response.get('metadata', [])) - description_entry = next(entry for entry in raw_metadata if entry.get('key') == MetadataField.DESCRIPTION) - return UpdateItemResult(item_id=table_id, success=True, timestamp=description_entry['timestamp']) - except Exception as e: - return UpdateItemResult(item_id=table_id, success=False, error=str(e)) - - -async def _update_column_descriptions( - client: KeboolaClient, table_id: str, column_updates: dict[str, str] -) -> list[UpdateItemResult]: - """Update multiple column descriptions for a single table.""" - try: - columns_metadata = { - column_name: [{'key': MetadataField.DESCRIPTION, 'value': description, 'columnName': column_name}] - for column_name, description in column_updates.items() - } - - response = await client.storage_client.table_metadata_update( - table_id=table_id, - columns_metadata=columns_metadata, - ) - - column_metadata = cast(dict[str, list[JsonDict]], response.get('columnsMetadata', {})) - results = [] - - for column_name in column_updates.keys(): - try: - description_entry = next( - entry - for entry in column_metadata.get(column_name, []) - if entry.get('key') == MetadataField.DESCRIPTION - ) - results.append( - UpdateItemResult( - item_id=f'{table_id}.{column_name}', success=True, timestamp=description_entry['timestamp'] - ) - ) - except Exception as e: - results.append(UpdateItemResult(item_id=f'{table_id}.{column_name}', success=False, error=str(e))) - - return results - except Exception as e: - # If the entire table update fails, mark all columns as failed - return [ - UpdateItemResult(item_id=f'{table_id}.{column_name}', success=False, error=str(e)) - for column_name in column_updates.keys() - ] - - -@tool_errors() -async def update_descriptions( - ctx: Context, - updates: Annotated[ - list[DescriptionUpdate], - Field( - description='List of DescriptionUpdate objects with storage item_id and new description. ' - 'Examples: "bucket_id", "bucket_id.table_id", "bucket_id.table_id.column_name"' - ), - ], -) -> UpdateDescriptionsOutput: - """Updates the description for a Keboola storage item. - - This tool supports three item types, inferred from the provided item_id: - - - bucket: item_id = "in.c-bucket" - - table: item_id = "in.c-bucket.table" - - column: item_id = "in.c-bucket.table.column" - - Usage examples (payload uses a list of DescriptionUpdate objects): - - Update a bucket: - updates=[DescriptionUpdate(item_id="in.c-my-bucket", description="New bucket description")] - - Update a table: - updates=[DescriptionUpdate(item_id="in.c-my-bucket.my-table", description="New table description")] - - Update a column: - updates=[DescriptionUpdate(item_id="in.c-my-bucket.my-table.my_column", description="New column description")] - """ - client = KeboolaClient.from_state(ctx.session.state) - results: list[UpdateItemResult] = [] - valid_updates: list[DescriptionUpdate] = [] - - # Handle invalid item_ids first and filter valid ones - for update in updates: - try: - _parse_item_id(update.item_id) - valid_updates.append(update) - except ValueError as e: - results.append( - UpdateItemResult(item_id=update.item_id, success=False, error=f'Invalid item_id format: {e}') - ) - - # Process valid updates - grouped_updates = _group_updates_by_type(valid_updates) - for bucket_id, description in grouped_updates.bucket_updates.items(): - result = await _update_bucket_description(client, bucket_id, description) - results.append(result) - - for table_id, description in grouped_updates.table_updates.items(): - result = await _update_table_description(client, table_id, description) - results.append(result) - - for table_id, column_updates in grouped_updates.column_updates_by_table.items(): - table_results = await _update_column_descriptions(client, table_id, column_updates) - results.extend(table_results) - - successful = sum(1 for r in results if r.success) - failed = len(results) - successful - - return UpdateDescriptionsOutput(results=results, total_processed=len(results), successful=successful, failed=failed) diff --git a/src/keboola_mcp_server/tools/storage/usage.py b/src/keboola_mcp_server/tools/storage/usage.py deleted file mode 100644 index 81bb796cb..000000000 --- a/src/keboola_mcp_server/tools/storage/usage.py +++ /dev/null @@ -1,182 +0,0 @@ -from collections import defaultdict -from typing import Mapping, Optional, Sequence, cast - -from pydantic import BaseModel, Field - -from keboola_mcp_server.clients.base import JsonStruct -from keboola_mcp_server.clients.client import ( - KeboolaClient, - get_metadata_property, -) -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.tools.search import ( - SearchComponentItemType, - SearchItemType, - SearchSpec, - fetch_configurations, -) -from keboola_mcp_server.utils import parse_iso_timestamp - - -class ComponentUsageReference(BaseModel): - component_id: str = Field(description='The ID of the component.') - configuration_id: str = Field(description='The ID of the configuration.') - configuration_row_id: str | None = Field(default=None, description='The ID of the configuration row.') - configuration_name: str | None = Field(default=None, description='The name of the configuration.') - used_in: str | None = Field(default=None, description='The dot-separated path within the configuration.') - timestamp: str | None = Field(default=None, description='The timestamp of the usage.') - - -class UsageById(BaseModel): - target_id: str - usage_references: list[ComponentUsageReference] - - -async def find_id_usage( - client: KeboolaClient, - target_ids: Sequence[str], - item_types: Optional[Sequence[SearchComponentItemType]] = None, - scopes: Sequence[str] = tuple(), -) -> list[UsageById]: - """ - Finds component configurations (including rows) that reference any of the target IDs in the specified configuration - scopes. - - :param client: The Keboola client to use. - :param target_ids: The IDs to search for. - :param item_types: Item types to search for. Only component configuration item types are supported. - :param scopes: Dot-separated keys to search in the configuration. - :return: A list of UsageById objects. - """ - - if not target_ids: - return [] - - spec = SearchSpec( - patterns=target_ids, - # Casting SearchComponentItemType to SearchItemType since it is a subset of SearchItemType. - item_types=cast(Sequence[SearchItemType], item_types) or tuple(), - search_scopes=scopes, - pattern_mode='literal', - search_type='config-based', - return_all_matched_patterns=True, - ) - - search_hits = await fetch_configurations(client, spec) - - # group usage references by pattern = target_id - output: dict[str, list[ComponentUsageReference]] = defaultdict(list) - for search_hit in search_hits: - for match in search_hit.matches: - for target_id in match.patterns: - output[target_id].append( - # TODO: Consider whether adding configuration description is useful, it could overload context. - ComponentUsageReference( - component_id=search_hit.component_id, - configuration_id=search_hit.configuration_id, - configuration_row_id=search_hit.configuration_row_id, - configuration_name=search_hit.name, - used_in=match.scope, - timestamp=search_hit.updated, - ) - ) - return [ - UsageById(target_id=target_id, usage_references=usage_references) - for target_id, usage_references in output.items() - ] - - -def get_created_by( - metadata: Sequence[Mapping[str, JsonStruct]] | Mapping[str, JsonStruct], -) -> ComponentUsageReference | None: - """ - Gets the created by reference from the metadata. - :param metadata: The metadata to search in. - :return: The created by reference. - """ - metadata_items = _coerce_metadata_list(metadata) - component_id = get_metadata_property(metadata_items, MetadataField.CREATED_BY_COMPONENT_ID) - configuration_id = get_metadata_property(metadata_items, MetadataField.CREATED_BY_CONFIGURATION_ID) - configuration_row_id = get_metadata_property(metadata_items, MetadataField.CREATED_BY_CONFIGURATION_ROW_ID) - if component_id is None or configuration_id is None: - return None - timestamp = _get_latest_metadata_timestamp( - metadata_items, - [ - MetadataField.CREATED_BY_COMPONENT_ID, - MetadataField.CREATED_BY_CONFIGURATION_ID, - MetadataField.CREATED_BY_CONFIGURATION_ROW_ID, - ], - ) - return ComponentUsageReference( - component_id=str(component_id), - configuration_id=str(configuration_id), - configuration_row_id=str(configuration_row_id) if configuration_row_id else None, - configuration_scope=None, - timestamp=timestamp, - ) - - -def get_last_updated_by( - metadata: Sequence[Mapping[str, JsonStruct]] | Mapping[str, JsonStruct], -) -> ComponentUsageReference | None: - """ - Gets the last updated by reference from the metadata. - :param metadata: The metadata to search in. - :return: The last updated by reference. - """ - metadata_items = _coerce_metadata_list(metadata) - component_id = get_metadata_property(metadata_items, MetadataField.UPDATED_BY_COMPONENT_ID) - configuration_id = get_metadata_property(metadata_items, MetadataField.UPDATED_BY_CONFIGURATION_ID) - configuration_row_id = get_metadata_property(metadata_items, MetadataField.UPDATED_BY_CONFIGURATION_ROW_ID) - if component_id is None or configuration_id is None: - return None - timestamp = _get_latest_metadata_timestamp( - metadata_items, - [ - MetadataField.UPDATED_BY_COMPONENT_ID, - MetadataField.UPDATED_BY_CONFIGURATION_ID, - MetadataField.UPDATED_BY_CONFIGURATION_ROW_ID, - ], - ) - return ComponentUsageReference( - component_id=str(component_id), - configuration_id=str(configuration_id), - configuration_row_id=str(configuration_row_id) if configuration_row_id else None, - configuration_scope=None, - timestamp=timestamp, - ) - - -def _coerce_metadata_list( - metadata: Sequence[Mapping[str, JsonStruct]] | Mapping[str, JsonStruct], -) -> list[Mapping[str, JsonStruct]]: - if isinstance(metadata, Mapping): - metadata_value = metadata.get('metadata') - if isinstance(metadata_value, list): - return [item for item in metadata_value if isinstance(item, Mapping)] - if 'key' in metadata and 'value' in metadata: - return [metadata] - return [] - return [item for item in metadata if isinstance(item, Mapping)] - - -def _get_latest_metadata_timestamp(metadata: list[Mapping[str, JsonStruct]], keys: Sequence[str]) -> str | None: - latest_ts = None - latest_raw: str | None = None - for item in metadata: - if item.get('key') not in keys: - continue - raw_ts = item.get('timestamp') - if raw_ts is None: - continue - if not isinstance(raw_ts, str): - continue - try: - parsed = parse_iso_timestamp(raw_ts) - except ValueError: - continue - if latest_ts is None or parsed > latest_ts: - latest_ts = parsed - latest_raw = raw_ts - return latest_raw diff --git a/src/keboola_mcp_server/tools/storage_helpers.py b/src/keboola_mcp_server/tools/storage_helpers.py deleted file mode 100644 index d5a023c31..000000000 --- a/src/keboola_mcp_server/tools/storage_helpers.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Branch-aware fetch helpers for storage objects. - -These helpers handle the dual-fetch + merge pattern needed when working on dev branches: -- On the default/production branch: single fetch from the default endpoint -- On a dev branch with storage-branches feature: parallel fetch from both default and branch endpoints, - merged so that branch data wins on ID collision -- On a dev branch without storage-branches (legacy): single fetch from the default endpoint - (legacy branches embed branch data in the default endpoint) - -All callers (storage tools, search, etc.) should use these helpers instead of calling -the storage client bucket/table methods directly. -""" - -import asyncio -import logging -from typing import Any - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import KeboolaClient, get_metadata_property -from keboola_mcp_server.config import MetadataField - -LOG = logging.getLogger(__name__) - -STORAGE_BRANCHES_FEATURE = 'storage-branches' - - -async def has_storage_branches(client: KeboolaClient) -> bool: - """Checks if the project has the storage-branches feature enabled and client is on a dev branch.""" - return client.branch_id is not None and await client.has_feature(STORAGE_BRANCHES_FEATURE) - - -async def merged_bucket_list(client: KeboolaClient, **kwargs: Any) -> list[JsonDict]: - """ - List all buckets visible from the current branch context. - - Returns production buckets merged with the current branch's buckets (branch wins on ID collision). - On the default branch or legacy branches, returns production data from the default endpoint. - """ - if await has_storage_branches(client): - prod_data, branch_data = await asyncio.gather( - client.storage_client.bucket_list(branch_id='default', **kwargs), - client.storage_client.bucket_list(branch_id=client.branch_id, **kwargs), - ) - return _merge_by_id(prod_data, branch_data) - else: - raw = await client.storage_client.bucket_list(branch_id='default', **kwargs) - return _filter_current_branch(raw, client.branch_id) - - -async def merged_bucket_table_list(client: KeboolaClient, bucket_id: str, **kwargs: Any) -> list[JsonDict]: - """ - List all tables in a bucket, merging production and branch data. - - For storage-branches: fetches from both default and branch endpoints for the given bucket_id. - For legacy branches: fetches from the default endpoint (which includes branched data). - """ - if await has_storage_branches(client): - prod_data, branch_data = await asyncio.gather( - client.storage_client.bucket_table_list(bucket_id, branch_id='default', **kwargs), - client.storage_client.bucket_table_list(bucket_id, branch_id=client.branch_id, **kwargs), - ) - return _merge_by_id(prod_data, branch_data) - else: - raw = await client.storage_client.bucket_table_list(bucket_id, branch_id='default', **kwargs) - return _filter_current_branch(raw, client.branch_id) - - -async def merged_bucket_detail(client: KeboolaClient, bucket_id: str) -> tuple[JsonDict | None, JsonDict | None]: - """ - Fetch production and branch versions of a bucket. - - Returns (prod_raw, dev_raw) tuple. Either may be None if not found. - """ - if await has_storage_branches(client): - prod_raw, dev_raw = await asyncio.gather( - _safe_bucket_detail(client, bucket_id, branch_id='default'), - _safe_bucket_detail(client, bucket_id, branch_id=client.branch_id), - ) - return prod_raw, dev_raw - else: - # Legacy: both prod and dev are accessible from the default endpoint - prod_raw = await _safe_bucket_detail(client, bucket_id, branch_id='default') - dev_raw = None - if client.branch_id: - if f'c-{client.branch_id}-' in bucket_id: - dev_id = bucket_id - else: - dev_id = bucket_id.replace('c-', f'c-{client.branch_id}-') - dev_raw = await _safe_bucket_detail(client, dev_id, branch_id='default') - return prod_raw, dev_raw - - -async def merged_table_detail(client: KeboolaClient, table_id: str) -> tuple[JsonDict | None, JsonDict | None]: - """ - Fetch production and branch versions of a table. - - Returns (prod_raw, dev_raw) tuple. Either may be None if not found. - """ - if await has_storage_branches(client): - prod_raw, dev_raw = await asyncio.gather( - _safe_table_detail(client, table_id, branch_id='default'), - _safe_table_detail(client, table_id, branch_id=client.branch_id), - ) - return prod_raw, dev_raw - else: - prod_raw = await _safe_table_detail(client, table_id, branch_id='default') - dev_raw = None - if client.branch_id: - if f'c-{client.branch_id}-' in table_id: - dev_id = table_id - else: - dev_id = table_id.replace('c-', f'c-{client.branch_id}-') - dev_raw = await _safe_table_detail(client, dev_id, branch_id='default') - return prod_raw, dev_raw - - -def _filter_current_branch(items: list[JsonDict], branch_id: str | None) -> list[JsonDict]: - """Filter out items belonging to other dev branches (legacy mode). - - Keeps items that are either production (no branch metadata) or belong to the current branch. - When branch_id is None (production), returns all items without branch metadata. - """ - result = [] - for item in items: - item_branch = get_metadata_property(item.get('metadata', []), MetadataField.FAKE_DEVELOPMENT_BRANCH) - if not item_branch or item_branch == branch_id: - result.append(item) - return result - - -def _merge_by_id(prod_data: list[JsonDict], branch_data: list[JsonDict]) -> list[JsonDict]: - """Merge production and branch data lists. Branch wins on ID collision.""" - branch_ids = {item.get('id') for item in branch_data} - merged = list(branch_data) - for item in prod_data: - if item.get('id') not in branch_ids: - merged.append(item) - return merged - - -async def _safe_bucket_detail(client: KeboolaClient, bucket_id: str, **kwargs: Any) -> JsonDict | None: - """Fetch bucket detail, returning None on 404.""" - import httpx - - try: - return await client.storage_client.bucket_detail(bucket_id, **kwargs) - except httpx.HTTPStatusError as e: - if e.response.status_code == 404: - return None - raise - - -async def _safe_table_detail(client: KeboolaClient, table_id: str, **kwargs: Any) -> JsonDict | None: - """Fetch table detail, returning None on 404.""" - import httpx - - try: - return await client.storage_client.table_detail(table_id, **kwargs) - except httpx.HTTPStatusError as e: - if e.response.status_code == 404: - return None - raise diff --git a/src/keboola_mcp_server/tools/validation.py b/src/keboola_mcp_server/tools/validation.py deleted file mode 100644 index 6c9d0fee0..000000000 --- a/src/keboola_mcp_server/tools/validation.py +++ /dev/null @@ -1,635 +0,0 @@ -""" -Validator functions for Component Configuration data that are generated by agents. -""" - -import json -import logging -from dataclasses import dataclass -from enum import Enum -from importlib import resources -from typing import Callable, Optional, cast - -import jsonschema -import jsonschema.validators - -from keboola_mcp_server.clients.base import ( - JsonDict, - JsonPrimitive, - JsonStruct, -) -from keboola_mcp_server.clients.client import ORCHESTRATOR_COMPONENT_ID, FlowType, KeboolaClient -from keboola_mcp_server.tools.components.model import Component -from keboola_mcp_server.tools.components.utils import ( - BIGQUERY_TRANSFORMATION_ID, - SNOWFLAKE_TRANSFORMATION_ID, - fetch_component, -) - -LOG = logging.getLogger(__name__) - -ValidateFunction = Callable[[JsonDict, JsonDict], None] - -RESOURCES = 'keboola_mcp_server.resources' - - -class ConfigurationSchemaResources(str, Enum): - STORAGE = 'storage-schema.json' - LEGACY_FLOW = 'flow-schema.json' - - -@dataclass(frozen=True) -class ValidationContext: - component_id: str - configuration_id: str | None = None - configuration_row_id: str | None = None - scope: str | None = None - - def __str__(self) -> str: - str_repr = f'component_id={self.component_id}' - if self.configuration_id: - str_repr += f', configuration_id={self.configuration_id}' - if self.configuration_row_id: - str_repr += f', configuration_row_id={self.configuration_row_id}' - if self.scope: - str_repr += f', scope={self.scope}' - return str_repr - - -class RecoverableValidationError(jsonschema.ValidationError): - """ - An instance was invalid under a provided schema using a recoverable message for the Agent. - """ - - def __init__( # noqa: B042 - self, - *args, - initial_message: Optional[str] = None, - validation_context: ValidationContext | None = None, - **kwargs, - ): - super().__init__(*args, **kwargs) - self.initial_message = initial_message - self.validation_context = validation_context - - @classmethod - def create_from_values( - cls, - other: jsonschema.ValidationError, - initial_message: Optional[str] = None, - validation_context: ValidationContext | None = None, - ): - return cls(**other._contents(), initial_message=initial_message, validation_context=validation_context) - - def __str__(self) -> str: - """ - Build a compact string representation showing only the violated schema constraint, - not the entire schema object. - """ - str_repr = f'{self.message}\n' - # Show only the violated keyword and its value, not the full schema - if isinstance(self.validator, str) and self.validator_value is not None: - schema_path = ''.join(f'[{p!r}]' for p in self.absolute_schema_path) - str_repr += f'Failed validating {self.validator!r} in schema{schema_path}:\n' - str_repr += f' {json.dumps({self.validator: self.validator_value}, indent=2)}\n' - # Show the path and value of the failing instance - if self.absolute_path: - instance_path = ''.join(f'[{p!r}]' for p in self.absolute_path) - str_repr += f'On instance{instance_path}:\n' - str_repr += f' {json.dumps(self.instance, indent=4)}\n' - if self.initial_message: - str_repr += f'{self.initial_message}\n' - if self.validation_context: - str_repr += f'Validation component context: {str(self.validation_context)}\n' - # When a required-field violation occurs in parameters scope, surface ALL required fields so - # the agent can populate every missing field in a single retry instead of fixing one at a time. - # Only emitted for parameters scope — for storage/flow the hint wording does not apply. - if ( - self.validator == 'required' - and isinstance(self.validator_value, list) - and self.validation_context is not None - and self.validation_context.scope == 'parameters' - ): - required_fields = ', '.join(f'`{f}`' for f in self.validator_value) - str_repr += ( - f'HINT: Ensure ALL of the following required fields are present in `parameters`: {required_fields}. ' - f'Call `get_components` to retrieve the full schema and `get_config_examples` for real-world examples.' - f'\n' - ) - return str_repr.rstrip() - - -class KeboolaParametersValidator: - """ - We use this validator to load parameters' schema that has been fetched from AI service for a given component ID and - to validate the parameters configuration (json data) received from the Agent against the loaded schema. - - A custom JSON Schema validator that handles UI elements and schema normalization: - 1. Ignores 'button' type (UI-only construct) - 2. Normalizes schema by: - - Converting boolean 'required' flags to proper list format (propagating the required flag up) - - Ensuring 'properties' is a dictionary if it is an empty list - """ - - @classmethod - def validate(cls, instance: JsonDict, schema: JsonDict) -> None: - """ - Validate the json data instance against the schema. - :param instance: The json data to validate - :param schema: The schema to validate against - """ - sanitized_schema = cls.sanitize_schema(schema) - base_validator = jsonschema.validators.validator_for(sanitized_schema) - keboola_validator = jsonschema.validators.extend( - base_validator, type_checker=base_validator.TYPE_CHECKER.redefine('button', cls.check_button_type) - ) - return keboola_validator(sanitized_schema).validate(instance) - - @staticmethod - def check_button_type(checker: jsonschema.TypeChecker, instance: object) -> bool: - """ - Dummy button type checker. - We accept button as a type since it is a UI construct and not a data type. - :returns: True if instance is a dict with a type field with value 'button', False otherwise - """ - # TODO: We can add a custom pydantic model or json schema for validating button type instances. - return isinstance(instance, dict) and 'button' == instance.get('type', None) - - @staticmethod - def sanitize_schema(schema: JsonDict) -> JsonDict: - """ - Normalize a JSON schema in place and return the same schema object. - - The normalization currently: - - removes empty ``enum`` arrays, because ``"enum": []`` would otherwise make the - schema reject every value; - - converts non-list ``required`` values into the standard list form and, for - boolean-like required flags, propagates the requirement to the parent schema; - - normalizes ``properties`` from an empty list to an empty dict when needed. - - These rules are applied recursively to nested schema nodes, including structures - reachable through ``properties`` and other schema-composition/container keywords - handled by this sanitizer. - """ - - def _sanitize_node( - schema: JsonStruct | JsonPrimitive, - ) -> tuple[JsonStruct | JsonPrimitive, Optional[bool]]: - - # default returns the element of a schema if we are at the bottom of the tree (not a dict) - if not isinstance(schema, dict): - return schema, False - - # Strip empty enum arrays - "enum": [] means no value is valid, which is - # a side effect of dynamically populated UI schemas with no options available. - if 'enum' in schema and isinstance(schema['enum'], list) and len(schema['enum']) == 0: - del schema['enum'] - - is_current_required = None - required = schema.get('required', []) - if not isinstance(required, list): - # Convert required field to empty list, and set is_current_required to True/False if the required - # field is set to true/false and propagate the required flag up to the parent's required list - is_current_required = str(required).lower() == 'true' - required = [] - - if (properties := schema.get('properties')) is not None: - if properties == []: - properties = {} # convert empty list to empty dict to avoid AttributeError in jsonschema - elif not isinstance(properties, dict): - # Invalid schema - properties must be a dictionary. SchemaError will be caught and logged - # in _validate_json_against_schema but the validation will succeed since we cant use invalid schema - raise jsonschema.SchemaError(f'properties must be a dictionary, got {type(properties)}') - - for property_name, subschema in properties.items(): - # we recursively sanitize the subschemas within the properties - properties[property_name], is_child_required = _sanitize_node(subschema) - # if is_child_required is None, do not propagate - the child has required field correctly set - if is_child_required is True and property_name not in required: - required.append(property_name) - elif is_child_required is False and property_name in required: - required.remove(property_name) - schema['properties'] = properties - - if required: - schema['required'] = list(required) - else: - schema.pop('required', None) - - # Recurse into 'items' - if 'items' in schema: - items = schema['items'] - if isinstance(items, dict): - schema['items'], _ = _sanitize_node(items) - elif isinstance(items, list): - schema['items'] = [_sanitize_node(item)[0] if isinstance(item, dict) else item for item in items] - - # Recurse into schema-list keywords (allOf, anyOf, oneOf) - for keyword in ('allOf', 'anyOf', 'oneOf'): - if keyword in schema and isinstance(schema[keyword], list): - schema[keyword] = [_sanitize_node(s)[0] if isinstance(s, dict) else s for s in schema[keyword]] - - # Recurse into single-schema keywords - for keyword in ('not', 'if', 'then', 'else'): - if keyword in schema and isinstance(schema[keyword], dict): - schema[keyword], _ = _sanitize_node(schema[keyword]) - - # Recurse into additionalProperties (when it's a schema, not a boolean) - if 'additionalProperties' in schema and isinstance(schema['additionalProperties'], dict): - schema['additionalProperties'], _ = _sanitize_node(schema['additionalProperties']) - - # Recurse into patternProperties - if 'patternProperties' in schema and isinstance(schema['patternProperties'], dict): - for pattern, subschema in schema['patternProperties'].items(): - if isinstance(subschema, dict): - schema['patternProperties'][pattern], _ = _sanitize_node(subschema) - - # Recurse into definitions / $defs - for keyword in ('definitions', '$defs'): - if keyword in schema and isinstance(schema[keyword], dict): - for name, subschema in schema[keyword].items(): - if isinstance(subschema, dict): - schema[keyword][name], _ = _sanitize_node(subschema) - - return schema, is_current_required - - sanitized_schema = cast(JsonDict, _sanitize_node(schema)[0]) - return sanitized_schema - - -def validate_storage_configuration_against_schema( - storage: JsonDict, - initial_message: Optional[str] = None, - validation_context: ValidationContext | None = None, -) -> JsonDict: - """Validate the storage configuration using jsonschema. - :param storage: The storage configuration to validate - :param initial_message: The initial message to include in the error message - :returns: The validated storage configuration (json data as the input) if the validation succeeds - """ - schema = _load_schema(ConfigurationSchemaResources.STORAGE) - _validate_json_against_schema( - json_data=storage, - schema=schema, - initial_message=initial_message, - validation_context=validation_context, - ) - return storage - - -def _validate_parameters_configuration_against_schema( - parameters: JsonDict, - schema: JsonDict, - initial_message: Optional[str] = None, - validation_context: ValidationContext | None = None, -) -> JsonDict: - """ - Validate the parameters configuration using jsonschema. - :parameters: json data to validate - :schema: json schema to validate against (root or row parameter configuration schema) - :initial_message: initial message to include in the error message - :returns: The validated parameters configuration (json data as the input) if the validation succeeds - """ - _validate_json_against_schema( - json_data=parameters, - schema=schema, - initial_message=initial_message, - validate_fn=KeboolaParametersValidator.validate, - validation_context=validation_context, - ) - return parameters - - -def validate_flow_configuration_against_schema( - flow: JsonDict, - flow_type: FlowType, - schema: Optional[JsonDict] = None, - initial_message: Optional[str] = None, - validation_context: ValidationContext | None = None, -) -> JsonDict: - """ - Validate the flow configuration using jsonschema. - :flow: json data to validate - :flow_type: selects the bundled schema only when ``schema`` is None; it must then be - ORCHESTRATOR_COMPONENT_ID (legacy). When ``schema`` is provided, that schema is - authoritative and ``flow_type`` is not used for schema selection. - :schema: explicit schema to validate against; required for conditional flows (resolved live). - When None, only the bundled legacy orchestrator schema is available. - :initial_message: initial message to include in the error message - :returns: The validated flow configuration - """ - if schema is None: - if flow_type != ORCHESTRATOR_COMPONENT_ID: - raise ValueError( - f'No schema provided for flow type "{flow_type}". The conditional flow schema must be ' - f'resolved from the Developer Portal via resolve_flow_schema() and passed explicitly.' - ) - schema = _load_schema(ConfigurationSchemaResources.LEGACY_FLOW) - _validate_json_against_schema( - json_data=flow, - schema=schema, - initial_message=initial_message, - validation_context=validation_context, - ) - return flow - - -def _validate_json_against_schema( - json_data: JsonDict, - schema: JsonDict, - initial_message: Optional[str] = None, - validate_fn: Optional[ValidateFunction] = None, - validation_context: ValidationContext | None = None, -): - """Validate JSON data against the provided schema.""" - try: - validate_fn = validate_fn or jsonschema.validate - validate_fn(json_data, schema) - except jsonschema.ValidationError as e: - raise RecoverableValidationError.create_from_values( - e, initial_message=initial_message, validation_context=validation_context - ) - except jsonschema.SchemaError as e: - LOG.exception( - f'The validation schema is not valid: {e}\n' - f'initial_message: {initial_message}\n' - f'schema: {schema}\n' - f'json_data: {json_data}' - ) - # this is not an Agent error, the schema is not valid and we are unable to validate the json - # hence we continue with as if it was valid - return - - -def _load_schema(json_schema_name: ConfigurationSchemaResources) -> JsonDict: - files = resources.files(RESOURCES) - schema_file = files / json_schema_name.value - with schema_file.open('r', encoding='utf-8') as f: - return json.load(f) - - -STORAGE_VALIDATION_INITIAL_MESSAGE = 'The provided storage configuration input does not follow the storage schema.\n' -ROOT_PARAMETERS_VALIDATION_INITIAL_MESSAGE = ( - 'The provided Root parameters configuration input does not follow the Root parameter json schema for component ' - 'id: {component_id}.\n' -) -ROW_PARAMETERS_VALIDATION_INITIAL_MESSAGE = ( - 'The provided Row parameters configuration input does not follow the Row parameter json schema for component ' - 'id: {component_id}.\n' -) - - -def validate_root_storage_configuration( - storage: Optional[JsonDict], - component: 'Component', - initial_message: Optional[str] = None, - *, - configuration_id: str | None = None, -) -> JsonDict: - """ - Utility function to validate the root storage configuration. - """ - return _validate_storage_configuration( - storage, - component, - initial_message, - is_row_storage=False, - configuration_id=configuration_id, - ) - - -def validate_row_storage_configuration( - storage: Optional[JsonDict], - component: 'Component', - initial_message: Optional[str] = None, - *, - configuration_id: str | None = None, - configuration_row_id: str | None = None, -) -> JsonDict: - """ - Utility function to validate the row storage configuration. - """ - return _validate_storage_configuration( - storage, - component, - initial_message, - is_row_storage=True, - configuration_id=configuration_id, - configuration_row_id=configuration_row_id, - ) - - -def _validate_storage_configuration( - storage: Optional[JsonDict], - component: 'Component', - initial_message: Optional[str] = None, - *, - is_row_storage: bool = False, - configuration_id: str | None = None, - configuration_row_id: str | None = None, -) -> JsonDict: - """ - Validates the storage configuration and checks if it is necessary for the component. - :param storage: The storage configuration to validate received from the agent. - :param component: The component for which the storage is provided - :param initial_message: The initial message to include in the error message. - :param is_row_storage: Whether the provided storage is for a row configuration. (False for root, True for row) - :param configuration_id: The configuration ID to include in the error message. - :param configuration_row_id: The configuration row ID to include in the error message. - :return: The contents of the 'storage' key from the validated configuration, - or an empty dict if no storage is provided. - """ - # As expected by the storage schema, we normalize storage to {'storage': storage | {} | None} - # since the agent bot can input storage as {'storage': storage} or just storage - storage_cfg = cast(Optional[JsonDict], storage.get('storage', storage) if storage else {}) - - # If storage is None, we set it to an empty dict - if storage_cfg is None: - LOG.warning( - f'No "storage" configuration provided for component {component.component_id} of type ' - f'{component.component_type}.' - ) - storage_cfg = {} - # Only for SQL transformations - storage must contain either input or output mappings - if component.component_id in [SNOWFLAKE_TRANSFORMATION_ID, BIGQUERY_TRANSFORMATION_ID]: - if not storage_cfg.get('input') and not storage_cfg.get('output'): - raise ValueError( - f'The "storage" must contain either "input" or "output" mappings in the configuration of the SQL ' - f'transformation "{component.component_id}".' - ) - # For row-based writers - ROOT must have an empty storage, ROW must have non-empty input in storage - if component.component_type == 'writer' and component.capabilities.is_row_based: - if not is_row_storage and storage_cfg != {}: - # ROOT storage is not empty but the writer is row-based - this is not allowed - raise ValueError( - 'The "storage" must be empty for root configuration of the writer component ' - f'"{component.component_id}" since it is row-based. In this case, storage should only be defined ' - 'in its outgoing row configurations.' - ) - elif is_row_storage and not storage_cfg.get('input'): - # ROW storage does not contain input configuration for row-based writer - this is not allowed - raise ValueError( - f'The "storage" must contain "input" mappings for the row configuration of the writer component ' - f'"{component.component_id}".' - ) - # Only for non-row-based writers - ROOT must have non-empty input in storage - if component.component_type == 'writer' and not component.capabilities.is_row_based: - if is_row_storage: - LOG.warning( - f'Validating "storage" for row configuration of non-row-based writer {component.component_id} is not ' - 'semantically correct. Possible cause: agent error or wrong component flag. Proceeding with validation.' - ) - if not storage_cfg.get('input'): - # ROOT storage does not contain input configuration for non-row-based writer - this is not allowed - # (We also can get here when bot tries to create a row config for non-row-based writer {both require input}) - raise ValueError( - f'The "storage" must contain "input" mappings for the root configuration of the writer component ' - f'"{component.component_id}".' - ) - - initial_message = (initial_message or '') + '\n' - initial_message += STORAGE_VALIDATION_INITIAL_MESSAGE - normalized_storage = cast(JsonDict, {'storage': storage_cfg}) - validation_context = ValidationContext( - component_id=component.component_id, - configuration_id=configuration_id, - configuration_row_id=configuration_row_id, - scope='storage', - ) - normalized_storage = validate_storage_configuration_against_schema( - normalized_storage, - initial_message, - validation_context=validation_context, - ) - return cast(JsonDict, normalized_storage['storage']) - - -def validate_root_parameters_configuration( - parameters: JsonDict, - component: Component, - initial_message: Optional[str] = None, - *, - configuration_id: str | None = None, -) -> JsonDict: - """ - Utility function to validate the root parameters configuration. - :param parameters: The parameters of the configuration to validate - :param component: The component for which the configuration is provided - :param initial_message: The initial message to include in the error message - :return: The contents of the 'parameters' key from the validated configuration - """ - initial_message = (initial_message or '') + '\n' - initial_message += ROOT_PARAMETERS_VALIDATION_INITIAL_MESSAGE.format(component_id=component.component_id) - return _validate_parameters_configuration( - parameters, - component.configuration_schema, - component.component_id, - initial_message, - configuration_id=configuration_id, - ) - - -def validate_row_parameters_configuration( - parameters: JsonDict, - component: Component, - initial_message: Optional[str] = None, - *, - configuration_id: str | None = None, - configuration_row_id: str | None = None, -) -> JsonDict: - """ - Utility function to validate the row parameters configuration. - :param parameters: The parameters of the configuration to validate - :param component: The component for which the configuration is provided - :param initial_message: The initial message to include in the error message - :return: The contents of the 'parameters' key from the validated configuration - """ - initial_message = (initial_message or '') + '\n' - initial_message += ROW_PARAMETERS_VALIDATION_INITIAL_MESSAGE.format(component_id=component.component_id) - return _validate_parameters_configuration( - parameters, - component.configuration_row_schema, - component.component_id, - initial_message, - configuration_id=configuration_id, - configuration_row_id=configuration_row_id, - ) - - -async def validate_processors_configuration( - client: KeboolaClient, - processors: list[JsonDict], - initial_message: Optional[str] = None, -) -> list[JsonDict]: - """ - Validates the configuration of a list of processors against their respective configuration - schemas. - Skips validation for processors without a configuration schema or those using - a configuration schema from the template. - - :param client: An instance of KeboolaClient used to fetch processor information. - :param processors: A list of processor definitions containing their parameters and other details. - :param initial_message: An optional string providing an initial context for validation messages. - :return: The list of processors after validation. - """ - - for processor in processors: - processor_id = cast(JsonDict, processor['definition'])['component'] - processor_info = await fetch_component(client, processor_id) - - # the vast majority of processors do not have a configuration schema, so we skip them - if not processor_info.configuration_schema: - continue - # some processors use configuration schema from the template, so we skip them - if 'print_hello' in processor_info.configuration_schema.get('required', []): - continue - - validation_context = ValidationContext( - component_id=processor_id, - ) - _validate_json_against_schema( - json_data=processor['parameters'], - schema=processor_info.configuration_schema, - initial_message=f'{initial_message}\nThe configuration of "{processor_id}" processor is not valid.', - validation_context=validation_context, - ) - - return processors - - -def _validate_parameters_configuration( - parameters: JsonDict, - schema: Optional[JsonDict], - component_id: str, - initial_message: Optional[str] = None, - configuration_id: str | None = None, - configuration_row_id: str | None = None, -) -> JsonDict: - """ - Utility function to validate the parameters configuration. - :param parameters: The parameters configuration to validate - :param schema: The schema to validate against - :param component_id: The ID of the component - :param initial_message: The initial message to include in the error message - :return: The contents of the 'parameters' key from the validated configuration - """ - # As expected by the component parameter schema, we use only the parameters configurations without the "parameters" - # key since the agent bot can input parameters as {'parameters': parameters} or just parameters - expected_parameters = cast(JsonDict, parameters.get('parameters', parameters)) - - if not schema: - LOG.warning(f'No schema provided for component {component_id}, skipping validation.') - return expected_parameters - - validation_context = ValidationContext( - component_id=component_id, - configuration_id=configuration_id, - configuration_row_id=configuration_row_id, - scope='parameters', - ) - expected_parameters = _validate_parameters_configuration_against_schema( - expected_parameters, - schema, - initial_message, - validation_context=validation_context, - ) - return expected_parameters diff --git a/src/keboola_mcp_server/utils.py b/src/keboola_mcp_server/utils.py deleted file mode 100644 index b237034d3..000000000 --- a/src/keboola_mcp_server/utils.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Shared utility helpers for the Keboola MCP server.""" - -import re -from datetime import datetime - - -def parse_iso_timestamp(ts: str) -> datetime: - """Parse an ISO 8601 timestamp string into a datetime object. - - Handles both ``Z`` and numeric timezone offsets in ``+HHMM`` form - (as returned by the Keboola Storage API), which Python ≤ 3.10's - ``datetime.fromisoformat`` does not accept without normalization. - """ - normalized = re.sub(r'([+-]\d{2})(\d{2})$', r'\1:\2', ts.replace('Z', '+00:00')) - return datetime.fromisoformat(normalized) diff --git a/src/keboola_mcp_server/workspace.py b/src/keboola_mcp_server/workspace.py deleted file mode 100644 index 9bab97f51..000000000 --- a/src/keboola_mcp_server/workspace.py +++ /dev/null @@ -1,823 +0,0 @@ -import abc -import asyncio -import json -import logging -import re -import time -import uuid -from typing import Any, Awaitable, Callable, Literal, Mapping, Sequence, cast -from urllib.parse import urlunparse - -from httpx import HTTPStatusError -from pydantic import Field -from pydantic.dataclasses import dataclass - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.query import QueryServiceClient -from keboola_mcp_server.tools.storage_helpers import has_storage_branches - -LOG = logging.getLogger(__name__) - - -@dataclass(frozen=True) -class JobSubmittedInfo: - """Information surfaced to the tool layer the moment a backend job becomes addressable. - - This fires immediately after Query Service returns a `queryJobId`. The tool layer turns - this into an MCP `notifications/progress` so clients (e.g. Kai, Claude Code) can record - the handle and use it to cancel out-of-band by POSTing to `cancellation_url` themselves, - regardless of which MCP replica the cancel lands on. - """ - - job_id: str - cancellation_url: str | None - backend: str - - -# Async callback invoked exactly once per execute_query, immediately after the backend -# returns a job handle. Callbacks are best-effort: any exception raised inside is suppressed -# so a failed progress notification cannot abort the underlying query. -JobSubmittedCallback = Callable[[JobSubmittedInfo], Awaitable[None]] - - -def get_backend_path(table: Mapping[str, Any]) -> list[str] | None: - """Extracts the backendPath from a table's bucket info if available.""" - bucket = table.get('bucket') - if isinstance(bucket, dict): - backend_path = bucket.get('backendPath') - if isinstance(backend_path, list): - return backend_path - return None - - -@dataclass(frozen=True) -class TableFqn: - """The properly quoted parts of a fully qualified table name.""" - - # TODO: refactor this and probably use just a simple string - # Snowflake FQNs are database.schema.table. BigQuery has no cross-project access, so the - # database tier is meaningless there — `db_name` is empty and the FQN is just dataset.table. - db_name: str # database (Snowflake); empty for BigQuery - schema_name: str # schema (Snowflake); dataset (BigQuery) - table_name: str - quote_char: str = '' - - @property - def identifier(self) -> str: - """Returns the properly quoted database identifier.""" - return '.'.join( - f'{self.quote_char}{n}{self.quote_char}' for n in [self.db_name, self.schema_name, self.table_name] if n - ) - - def __repr__(self) -> str: - return self.identifier - - def __str__(self) -> str: - return self.__repr__() - - -@dataclass(frozen=True) -class DbColumnInfo: - name: str - quoted_name: str - native_type: str - nullable: bool - - -@dataclass(frozen=True) -class DbTableInfo: - id: str - fqn: TableFqn - columns: Mapping[str, DbColumnInfo] - - -QueryStatus = Literal['ok', 'error'] -SqlSelectDataRow = Mapping[str, Any] - - -@dataclass(frozen=True) -class SqlSelectData: - columns: Sequence[str] = Field(description='Names of the columns returned from SQL select.') - rows: Sequence[SqlSelectDataRow] = Field( - description='Selected rows, each row is a dictionary of column: value pairs.' - ) - - -@dataclass(frozen=True) -class QueryResult: - status: QueryStatus = Field(description='Status of running the SQL query.') - data: SqlSelectData | None = Field(default=None, description='Data selected by the SQL SELECT query.') - message: str | None = Field( - default=None, description='Either an error message or the information from non-SELECT queries.' - ) - - @property - def is_ok(self) -> bool: - return self.status == 'ok' - - @property - def is_error(self) -> bool: - return not self.is_ok - - -class _Workspace(abc.ABC): - _QUERY_TIMEOUT = 300.0 # 5 minutes - _CANCELLATION_TIMEOUT = 30.0 # 30 seconds to wait for cancellation - _SELECTED_ROWS_MSG = 'Returning {rows} of {total} selected rows.' - _PAGE_SIZE = 1_000 - - def __init__(self, workspace_id: int, client: KeboolaClient) -> None: - self._workspace_id = workspace_id - self._client = client - self._qsclient: QueryServiceClient | None = None - - @property - def id(self) -> int: - return self._workspace_id - - @abc.abstractmethod - def get_sql_dialect(self) -> str: - pass - - @abc.abstractmethod - def get_quoted_name(self, name: str) -> str: - pass - - @abc.abstractmethod - async def get_table_info(self, table: Mapping[str, Any]) -> DbTableInfo | None: - # TODO: use a pydantic class for the 'table' param - pass - - async def _cancel_job_with_timeout(self, job_id: str, reason: str) -> tuple[bool, bool]: - """ - Cancel a query job and poll until cancellation is confirmed. - - :param job_id: The query job ID to cancel. - :param reason: The reason for cancellation (used in cancel request and logging). - :return: Tuple of (cancellation_confirmed, query_completed). - cancellation_confirmed: True if cancellation was confirmed (or query completed), - False if it failed or timed out. - query_completed: True if query completed successfully during cancellation polling, - False otherwise. - """ - try: - await self._qsclient.cancel_job(job_id, reason=reason) - LOG.info(f'Query cancellation requested: job_id={job_id}') - - # Poll for cancellation confirmation - cancel_start = time.perf_counter() - while True: - job_status = await self._qsclient.get_job_status(job_id) - if 'status' not in job_status: - LOG.warning(f'Query status response missing "status" field: job_id={job_id}') - return (False, False) - status = job_status['status'] - - if status == 'completed': - LOG.info(f'Query completed successfully during cancellation attempt: job_id={job_id}') - return (True, True) # Cancellation confirmed, query completed - elif status in ['failed', 'canceled', 'cancelled']: - LOG.info(f'Query job cancellation confirmed: job_id={job_id}, status={status}') - return (True, False) # Cancellation confirmed, query not completed - - if time.perf_counter() - cancel_start > self._CANCELLATION_TIMEOUT: - LOG.info( - f'Query cancellation polling timed out after {self._CANCELLATION_TIMEOUT}s: ' - f'job_id={job_id}, status={status}' - ) - return (False, False) - - await asyncio.sleep(0.5) # Poll every 500ms - - except HTTPStatusError as e: - LOG.error( - f'HTTP error during query cancellation: job_id={job_id}, ' - f'status_code={e.response.status_code}, error={e}' - ) - return (False, False) - except Exception: - LOG.exception(f'Unexpected error during query cancellation: job_id={job_id}') - return (False, False) - - async def execute_query( - self, - sql_query: str, - *, - max_rows: int | None = None, - max_chars: int | None = None, - on_job_submitted: JobSubmittedCallback | None = None, - ) -> QueryResult: - """ - Runs a given SQL query through the Query Service. - - The Query Service is backend-agnostic; the SQL itself must follow the dialect of the - workspace backend (see :meth:`get_sql_dialect` / :meth:`get_quoted_name`). - - :param sql_query: The SQL query to be executed. - :param max_rows: The maximum number of rows to fetch from the query results. If None, no limit is applied. - :param max_chars: The maximum number of chars to fetch from the query results. If None, no limit is applied. - :param on_job_submitted: Optional async callback invoked with the backend job handle as soon as the job is - registered with the Query Service. Exceptions raised inside the callback are suppressed so a failed - notification cannot abort the query. - :return: The result of the executed query. - """ - if max_rows is not None and max_rows <= 0: - raise ValueError('The "max_rows" must be a positive integer or None.') - if max_chars is not None and max_chars <= 0: - raise ValueError('The "max_chars" must be a positive integer or None.') - - if not self._qsclient: - self._qsclient = await self._create_qs_client() - - ts_start = time.perf_counter() - job_id = await self._qsclient.submit_job(statements=[sql_query], workspace_id=str(self.id)) - # The job is now registered with Query Service, so everything from here on must run under - # the CancelledError handler below: if the client cancels while we are still in the - # `on_job_submitted` callback (e.g. emitting the progress notification), we must still - # propagate the cancel to the backend rather than leak a running QS job. - try: - if on_job_submitted is not None: - info = JobSubmittedInfo( - job_id=job_id, - cancellation_url=self._qsclient.build_cancel_url(job_id), - backend=self.get_sql_dialect().lower(), - ) - # Best-effort: a failed progress notification must not kill the running query. - # CancelledError is `BaseException` since Python 3.8, so `except Exception` already - # lets it propagate on the supported Python (>=3.10). The explicit branch below - # documents intent and re-raises so the outer CancelledError handler can cancel the - # backend job; it also guards against a future refactor that might widen the catch - # to `BaseException` and silently swallow cancellation. - try: - await on_job_submitted(info) - except asyncio.CancelledError: - raise - except Exception as exc: - LOG.warning(f'on_job_submitted callback raised for job_id={job_id}: {exc!r} — continuing') - while (job_status := await self._qsclient.get_job_status(job_id)) and job_status['status'] not in [ - 'completed', - 'failed', - 'canceled', - 'cancelled', - ]: - await asyncio.sleep(1) - elapsed_time = time.perf_counter() - ts_start - if elapsed_time > self._QUERY_TIMEOUT: - # Cancel the query before raising timeout error. Inline the reason (rather than - # binding a `reason` local) so it can't be mistaken for an in-scope variable by - # the `except asyncio.CancelledError` handler below, which uses its own reason. - cancellation_confirmed, query_completed = await self._cancel_job_with_timeout( - job_id, f'Query timeout exceeded after {elapsed_time:.2f} seconds' - ) - - # If query completed during cancellation, fetch and return results - if query_completed: - LOG.info(f'Query completed during cancellation polling, returning results: job_id={job_id}') - # Break out of the polling loop to fetch results below - job_status = await self._qsclient.get_job_status(job_id) - break - - # Query did not complete - raise timeout error - if cancellation_confirmed: - raise RuntimeError( - f'Query execution timed out after {elapsed_time:.2f} seconds. ' - f'The query has been cancelled: job_id={job_id}' - ) - else: - raise RuntimeError( - f'Query execution timed out after {elapsed_time:.2f} seconds. ' - f'Cancellation was attempted but could not be confirmed. ' - f'The query may still be running on the server: job_id={job_id}' - ) - except asyncio.CancelledError: - # Client (e.g. MCP `notifications/cancelled`) cancelled the in-flight tool call. - # Propagate the cancel to the backend so the query doesn't keep scanning data. - # `asyncio.shield` keeps the cancel HTTP call alive even though our own task - # is being cancelled; without it the request would be torn down immediately. - LOG.info(f'Query cancelled by client: job_id={job_id}') - try: - await asyncio.shield(self._cancel_job_with_timeout(job_id, reason='Client cancelled the request')) - except asyncio.CancelledError: - # Outer scope was cancelled again while the shielded cancel was still running; - # we did our best — let the original CancelledError propagate below. - pass - raise - - # Short-circuit when the poll loop exited because the job was cancelled out-of-band - # (e.g. the user clicked STOP and the kai-agent backend POSTed - # `POST /api/v1/queries/{job_id}/cancel` directly to Query Service, or the in-flight - # request itself was aborted with notifications/cancelled). Going through the results - # fetch path here surfaces QS's generic "Job is still running or not completed yet" - # message, which is misleading — we already know the job reached a terminal CANCELLED - # state. Return a clear cancel result instead and skip the results fetch entirely. - terminal_status = job_status['status'] - if terminal_status in ('canceled', 'cancelled'): - LOG.info(f'Query was cancelled (terminal status={terminal_status}): job_id={job_id}') - return QueryResult(status='error', data=None, message='Query was cancelled') - - statement_id = cast(list[JsonDict], job_status['statements'])[0]['id'] - - # Fetch results with pagination - all_rows: list[list[Any]] = [] - all_rows_chars: int = 0 - columns: list[str] = [] - offset = 0 - page_size = self._PAGE_SIZE - message: str | None = None - total_query_rows: int | None = None - - while True: - if max_rows is not None: - remaining = max_rows - len(all_rows) - if remaining <= 0: - break - rows_to_fetch = min(page_size, remaining) - else: - rows_to_fetch = page_size - - results = await self._qsclient.get_job_results( - job_id, - statement_id, - offset=offset, - limit=max(rows_to_fetch, 100), # QueryService requires 100 - 10_000 - ) - - # Store message, total_query_rows and columns from the first response - if offset == 0: - status = results['status'] - message = results['message'] - total_query_rows = results.get('numberOfRows') - - if status in ['failed', 'canceled', 'cancelled']: - return QueryResult(status='error', data=None, message=self._format_error_message(message)) - elif status != 'completed': - raise ValueError(f'Unexpected query status: {status}') - - columns = [col['name'] for col in cast(list[JsonDict], results['columns'])] - - page_data = cast(list[list[Any]], results.get('data', [])) - if not page_data: - break - - page_data = page_data[:rows_to_fetch] - char_limit_reached = False - if max_chars is not None: - for row in page_data: - chars = sum(len(str(v)) for v in row if v is not None) - if all_rows_chars + chars <= max_chars: - all_rows.append(row) - all_rows_chars += chars - else: - # The first row that does not fit ends pagination so that the result - # is a contiguous prefix; we must not skip this row and then append - # later smaller rows that happen to fit. - char_limit_reached = True - break - else: - all_rows.extend(page_data) - - if len(page_data) < rows_to_fetch: - break - - if max_rows is not None and len(all_rows) >= max_rows: - break - - if char_limit_reached or (max_chars is not None and all_rows_chars >= max_chars): - break - - offset += len(page_data) - - rows = [{col_name: value for col_name, value in zip(columns, row)} for row in all_rows] - - if columns: - message = ' '.join( - filter(None, [message, self._SELECTED_ROWS_MSG.format(rows=len(rows), total=total_query_rows)]) - ) - query_result = QueryResult(status='ok', data=SqlSelectData(columns=columns, rows=rows), message=message) - else: - query_result = QueryResult(status='ok', message=message) - - return query_result - - async def get_branch_id(self) -> str: - if not self._qsclient: - self._qsclient = await self._create_qs_client() - return self._qsclient.branch_id - - async def _create_qs_client(self) -> QueryServiceClient: - """ - Creates a QueryServiceClient for the workspace. - - Note: Currently, QueryServiceClient is not cached and sessions are not used, so bearer token - expiration is not an issue. If sessions and caching are reintroduced in the future, token - expiration handling will need to be considered. - """ - real_branch_id = self._client.branch_id - if not real_branch_id: - for branch in await self._client.storage_client.branches_list(): - if (is_default := branch.get('isDefault')) and isinstance(is_default, bool) and is_default: - real_branch_id = branch['id'] - break - if not real_branch_id: - raise RuntimeError('Cannot determine the default branch ID') - - # Prefer bearer token over storage token for Query Service - token = f'Bearer {self._client.bearer_token}' if self._client.bearer_token else self._client.token - - return QueryServiceClient.create( - root_url=urlunparse(('https', f'query.{self._client.hostname_suffix}', '', '', '', '')), - branch_id=real_branch_id, - token=token, - headers=self._client.headers, - ) - - def _format_error_message(self, message: str | None) -> str | None: - """ - Normalizes a failed-query error message returned by the Query Service into a clean, - human-readable string. The base implementation passes the message through unchanged; - backends whose Query Service responses wrap the error may override this. - """ - return message - - @classmethod - def _dump(cls, json_data: Mapping[str, Any]) -> str: - return json.dumps(json_data, ensure_ascii=False, separators=(',', ':')) - - -class _SnowflakeWorkspace(_Workspace): - def __init__(self, workspace_id: int, schema: str, client: KeboolaClient): - super().__init__(workspace_id, client) - self._schema = schema # default schema created for the workspace - - def get_sql_dialect(self) -> str: - return 'Snowflake' - - def get_quoted_name(self, name: str) -> str: - return f'"{name}"' # wrap name in double quotes - - async def get_table_info(self, table: Mapping[str, Any]) -> DbTableInfo | None: - table_id = table['id'] - - # The table's own bucket backendPath resolves to the database + schema where the table - # physically lives. For a linked bucket — including a materialized alias shared from another - # project — Storage propagates that backendPath onto the linked table itself, so the FQN is - # directly queryable from this workspace. - bp = get_backend_path(table) - if not bp or len(bp) < 2: - LOG.warning(f'No backendPath available for table {table_id}, cannot construct FQN') - return None - - return DbTableInfo( - id=table_id, - fqn=TableFqn(bp[0], bp[1], table['name'], quote_char='"'), - columns={}, - ) - - -class _BigQueryWorkspace(_Workspace): - # The Query Service surfaces BigQuery errors as a serialized error object, e.g. - # {Location: "query"; Message: "Syntax error: Unexpected identifier ..."; Reason: "invalidQuery"} - # Extract the human-readable `Message: "..."` part so the error reads like Snowflake's plain text. - _BQ_ERROR_MESSAGE_RE = re.compile(r'Message:\s*"((?:[^"\\]|\\.)*)"') - - def __init__(self, workspace_id: int, dataset_id: str, project_id: str, client: KeboolaClient): - super().__init__(workspace_id, client) - self._dataset_id = dataset_id # default dataset created for the workspace - self._project_id = project_id - - def get_sql_dialect(self) -> str: - return 'BigQuery' - - def get_quoted_name(self, name: str) -> str: - return f'`{name}`' # wrap name in back tick - - async def get_table_info(self, table: Mapping[str, Any]) -> DbTableInfo | None: - table_id = table['id'] - - # BigQuery has no cross-project data sharing: a table that is an alias in its source project - # (sourceTable.isAlias) is not materialized into this project's dataset and cannot be queried - # from this workspace. Materialized aliases are a Snowflake-only capability. - if table.get('sourceTable', {}).get('isAlias'): - return None - - bp = get_backend_path(table) - if not bp: - LOG.warning(f'No backendPath available for table {table_id}, cannot construct FQN') - return None - - # BigQuery backendPath[0] is the dataset name; normalize separators for BQ dataset naming. - # BigQuery has no cross-project access, so the FQN is dataset.table with no project/database - # tier (db_name is left empty) — see editor-service SapiDataProvider::parseBackendPath. - schema_name = bp[0].replace('.', '_').replace('-', '_') - table_name = table['name'] - - return DbTableInfo( - id=table_id, - fqn=TableFqn(db_name='', schema_name=schema_name, table_name=table_name, quote_char='`'), - columns={}, - ) - - def _format_error_message(self, message: str | None) -> str | None: - if message and (m := self._BQ_ERROR_MESSAGE_RE.search(message)): - return m.group(1).replace('\\"', '"') - return message - - -@dataclass(frozen=True) -class _WspInfo: - id: int - schema: str - backend: str - credentials: str | None # the backend credentials; it can contain serialized JSON data - readonly: bool | None - - @staticmethod - def from_sapi_info(sapi_wsp_info: Mapping[str, Any]) -> '_WspInfo': - _id = sapi_wsp_info.get('id') - backend = sapi_wsp_info.get('connection', {}).get('backend') - _schema = sapi_wsp_info.get('connection', {}).get('schema') - credentials = sapi_wsp_info.get('connection', {}).get('user') - readonly = sapi_wsp_info.get('readOnlyStorageAccess') - return _WspInfo(id=_id, schema=_schema, backend=backend, credentials=credentials, readonly=readonly) - - -class WorkspaceManager: - STATE_KEY = 'workspace_manager' - MCP_META_KEY = 'KBC.McpServer.v2.workspaceId' - MCP_WORKSPACE_COMPONENT_ID = 'keboola.mcp-server-tool' - - @classmethod - def from_state(cls, state: Mapping[str, Any]) -> 'WorkspaceManager': - instance = state[cls.STATE_KEY] - assert isinstance(instance, WorkspaceManager), f'Expected WorkspaceManager, got: {instance}' - return instance - - @classmethod - async def create(cls, client: KeboolaClient, workspace_schema: str | None = None) -> 'WorkspaceManager': - # On projects with the `storage-branches` feature, each dev branch needs its own - # workspace so the agent's queries (FQN paths, `query_data`) see that branch's - # table versions. The workspace ID is stored under the same metadata key but in - # the branch's own metadata, which is per-branch (`branch/{id}/metadata`). - # On legacy projects (no `storage-branches`) and on the default branch, fall back - # to the production-branch workspace shared by the whole project. - # `has_storage_branches` already requires `branch_id is not None`, so the default - # branch always takes the prod-client path. - if await has_storage_branches(client): - return cls(client, workspace_schema) - prod_client = await client.with_branch_id(None) - return cls(prod_client, workspace_schema) - - def __init__(self, client: KeboolaClient, workspace_schema: str | None = None): - """ - Initializes the WorkspaceManager. - - :param client: The KeboolaClient bound to the branch whose workspace this manager - owns. On default-branch or legacy-project paths this is the production-branch - client; on a `storage-branches` project bound to a dev branch this is the - dev-branch client (see :meth:`create`). - :param workspace_schema: The schema of the workspace to use. - """ - self._client = client - self._workspace_schema = workspace_schema - self._workspace: _Workspace | None = None - self._table_info_cache: dict[str, DbTableInfo] = {} - - async def _find_ws_by_schema(self, schema: str) -> _WspInfo | None: - """Finds the workspace info by its schema.""" - - for sapi_wsp_info in await self._client.storage_client.workspace_list(): - assert isinstance(sapi_wsp_info, dict) - wi = _WspInfo.from_sapi_info(sapi_wsp_info) # type: ignore[attr-defined] - if wi.id and wi.backend and wi.schema and wi.schema == schema: - return wi - - return None - - async def _find_ws_by_id(self, workspace_id: str | int) -> _WspInfo | None: - """Finds the workspace info by its ID.""" - - try: - sapi_wsp_info = await self._client.storage_client.workspace_detail(workspace_id) - assert isinstance(sapi_wsp_info, dict) - wi = _WspInfo.from_sapi_info(sapi_wsp_info) # type: ignore[attr-defined] - - if wi.id and wi.backend and wi.schema: - return wi - else: - raise ValueError(f'Invalid workspace info: {sapi_wsp_info}') - - except HTTPStatusError as e: - if e.response.status_code == 404: - return None - else: - raise e - - async def _find_ws_in_branch(self) -> _WspInfo | None: - """Finds the workspace info in the current branch.""" - - meta_key = self.MCP_META_KEY - metadata = await self._client.storage_client.branch_metadata_get() - for m in metadata: - if m.get('key') == meta_key and (raw_value := m.get('value')): - if (info := await self._find_ws_by_id(raw_value)) and info.readonly: - return info - - return None - - async def _create_ws(self, *, timeout_sec: float = 300.0) -> _WspInfo | None: - """ - Creates a new workspace under a component configuration and returns its info. - - The workspace is created under the MCP_WORKSPACE_COMPONENT_ID component so that - it is correctly attributed for billing. This method creates the configuration, - creates the workspace under it, and cleans up the configuration on failure. - - :param timeout_sec: The number of seconds to wait for the workspace creation job to finish. - :return: The workspace info if the workspace was created successfully, None otherwise. - """ - - # Verify token before creating workspace to ensure it has proper permissions - token_info = await self._client.storage_client.verify_token() - - # Check for defaultBackend parameter in token info under owner object - owner_info = token_info.get('owner', {}) - default_backend = owner_info.get('defaultBackend') - - if default_backend == 'snowflake': - login_type = 'snowflake-person-sso' - elif default_backend == 'bigquery': - login_type = 'default' - else: - raise ValueError(f'Unexpected default backend: {default_backend}') - - component_id = self.MCP_WORKSPACE_COMPONENT_ID - config_name = f'mcp-workspace-{uuid.uuid4().hex[:8]}' - config_resp = await self._client.storage_client.configuration_create( - component_id=component_id, - name=config_name, - description='Auto-created by MCP server for workspace billing.', - configuration={}, - ) - config_id = str(config_resp['id']) - - try: - resp = await self._client.storage_client.workspace_create_for_config( - component_id=component_id, - config_id=config_id, - login_type=login_type, - backend=default_backend, - async_run=True, - read_only_storage_access=True, - ) - except Exception: - try: - await self._client.storage_client.configuration_delete(component_id, config_id) - except Exception as cleanup_err: - LOG.warning( - f'Failed to clean up configuration {component_id}/{config_id} ' - f'after workspace creation failure: {cleanup_err}', - exc_info=True, - ) - raise - - assert 'id' in resp, f'Expected job ID in response: {resp}' - assert isinstance(resp['id'], int) - - job_id = resp['id'] - start_ts = time.perf_counter() - LOG.info(f'Requested new workspace: job_id={job_id}, timeout={timeout_sec:.2f} seconds') - - while True: - job_info = await self._client.storage_client.job_detail(job_id) - job_status = job_info['status'] - - duration = time.perf_counter() - start_ts - LOG.info( - f'Job info: job_id={job_id}, status={job_status}, ' - f'duration={duration:.2f} seconds, timeout={timeout_sec:.2f} seconds' - ) - - if job_info['status'] == 'success': - assert 'results' in job_info, f'Expected `results` in job info: {job_info}' - job_results = job_info['results'] - assert isinstance(job_results, dict) - assert 'id' in job_results, f'Expected `id` in `results` in job info: {job_info}' - assert isinstance(job_results['id'], int) - - workspace_id = job_results['id'] - LOG.info(f'Created workspace: {workspace_id}') - return await self._find_ws_by_id(workspace_id) - - elif duration > timeout_sec: - LOG.info(f'Workspace creation timed out after {duration:.2f} seconds.') - return None - - else: - remaining_time = max(0.0, timeout_sec - duration) - await asyncio.sleep(min(5.0, remaining_time)) - - def _init_workspace(self, info: _WspInfo) -> _Workspace: - """Creates a new `Workspace` instance based on the workspace info.""" - - if info.backend == 'snowflake': - return _SnowflakeWorkspace(workspace_id=info.id, schema=info.schema, client=self._client) - - elif info.backend == 'bigquery': - credentials = json.loads(info.credentials or '{}') - if project_id := credentials.get('project_id'): - return _BigQueryWorkspace( - workspace_id=info.id, - dataset_id=info.schema, - project_id=project_id, - client=self._client, - ) - - else: - raise ValueError(f'No credentials or no project ID in workspace: {info.schema}') - - else: - raise ValueError(f'Unexpected backend type "{info.backend}" in workspace: {info.schema}') - - async def _get_workspace(self) -> _Workspace: - if self._workspace: - return self._workspace - - if self._workspace_schema: - # use the workspace that was explicitly requested - # this workspace must never be written to the default branch metadata - LOG.info(f'Looking up workspace by schema: {self._workspace_schema}') - if info := await self._find_ws_by_schema(self._workspace_schema): - LOG.info(f'Found workspace: {info}') - self._workspace = self._init_workspace(info) - return self._workspace - else: - raise ValueError( - f'No Keboola workspace found or the workspace has no read-only storage access: ' - f'workspace_schema={self._workspace_schema}' - ) - - LOG.info('Looking up workspace in the default branch.') - if info := await self._find_ws_in_branch(): - # use the workspace that has already been created by the MCP server and noted to the branch - LOG.info(f'Found workspace: {info}') - self._workspace = self._init_workspace(info) - return self._workspace - - # create a new workspace and note its ID to the branch - LOG.info('Creating workspace in the default branch.') - if info := await self._create_ws(): - # All tokens share the same read-only workspace - # Race conditions during initialization are acceptable (last-write-wins) - meta = await self._client.storage_client.branch_metadata_update({self.MCP_META_KEY: info.id}) - LOG.info(f'Set metadata in the default branch: {meta}') - # use the newly created workspace - self._workspace = self._init_workspace(info) - return self._workspace - else: - raise ValueError('Failed to initialize Keboola Workspace.') - - async def execute_query( - self, - sql_query: str, - *, - max_rows: int | None = None, - max_chars: int | None = None, - on_job_submitted: JobSubmittedCallback | None = None, - ) -> QueryResult: - workspace = await self._get_workspace() - return await workspace.execute_query( - sql_query, - max_rows=max_rows, - max_chars=max_chars, - on_job_submitted=on_job_submitted, - ) - - async def get_table_info(self, table: Mapping[str, Any]) -> DbTableInfo | None: - # Whether an alias table is queryable depends on the backend (Snowflake materializes aliases - # from linked buckets, BigQuery does not), so each workspace implementation makes that call. - table_id = table['id'] - if table_id in self._table_info_cache: - return self._table_info_cache[table_id] - - workspace = await self._get_workspace() - if info := await workspace.get_table_info(table): - self._table_info_cache[table_id] = info - - return info - - async def get_quoted_name(self, name: str) -> str: - workspace = await self._get_workspace() - return workspace.get_quoted_name(name) - - async def get_sql_dialect(self) -> str: - workspace = await self._get_workspace() - return workspace.get_sql_dialect() - - async def get_workspace_id(self) -> int: - workspace = await self._get_workspace() - return workspace.id - - async def get_branch_id(self) -> str: - workspace = await self._get_workspace() - return await workspace.get_branch_id() diff --git a/src/links.ts b/src/links.ts new file mode 100644 index 000000000..1e8b2fa15 --- /dev/null +++ b/src/links.ts @@ -0,0 +1,268 @@ +import { + CONDITIONAL_FLOW_COMPONENT_ID, + DATA_APP_COMPONENT_ID, + FLOW_TYPES, + type FlowType, +} from '@/constants'; + +/** + * UI / docs links surfaced to the user alongside tool results. Faithful port of + * the Python `links.ProjectLinksManager` — pure URL building from the project's + * base URL, project id, and (optional) dev-branch id. + */ +export type UrlType = 'ui-detail' | 'ui-dashboard' | 'docs'; + +export type Link = { + type: UrlType; + title: string; + url: string; +}; + +const detail = (title: string, url: string): Link => ({ type: 'ui-detail', title, url }); +const dashboard = (title: string, url: string): Link => ({ type: 'ui-dashboard', title, url }); +const docs = (title: string, url: string): Link => ({ type: 'docs', title, url }); + +const FLOW_DOCUMENTATION_URL = 'https://help.keboola.com/flows/'; + +const isFlowType = (componentId: string | undefined): componentId is FlowType => + componentId !== undefined && (FLOW_TYPES as readonly string[]).includes(componentId); + +const isDataAppComponent = (componentId: string | undefined): boolean => + componentId === DATA_APP_COMPONENT_ID; + +const isTransformationComponent = (componentId: string): boolean => + Boolean(componentId) && componentId.includes('transformation'); + +export class ProjectLinksManager { + private readonly baseUrl: string; + private readonly projectId: string; + private readonly branchId: string | undefined; + + constructor(options: { baseUrl: string; projectId: string; branchId?: string }) { + this.baseUrl = options.baseUrl; + this.projectId = options.projectId; + this.branchId = options.branchId; + } + + private url(path: string): string { + const parts = [this.baseUrl, 'admin/projects', this.projectId]; + if (this.branchId) { + parts.push('branch', this.branchId); + } + parts.push(path); + return parts.join('/'); + } + + private flowPath(flowType: FlowType): string { + return flowType === CONDITIONAL_FLOW_COMPONENT_ID ? 'flows-v2' : 'flows'; + } + + /** Most relevant links for a Keboola object from mutually-exclusive identifiers. */ + getLinks(opts: { + bucketId?: string; + tableId?: string; + componentId?: string; + configurationId?: string; + name?: string; + }): Link[] { + const { bucketId, tableId, componentId, configurationId, name } = opts; + if (componentId && configurationId) { + return [this.getComponentConfigLink(componentId, configurationId, name ?? '')]; + } + if (componentId) { + return [this.getConfigDashboardLink(componentId, name ?? '')]; + } + if (tableId) { + return [this.getTableDetailLinkFromTableId(tableId)]; + } + if (bucketId) { + return [this.getBucketDetailLink(bucketId, name ?? bucketId)]; + } + return []; + } + + // --- Project --- + getProjectDetailLink(): Link { + return detail('Project Dashboard', this.url('')); + } + + getProjectLinks(): Link[] { + return [this.getProjectDetailLink()]; + } + + // --- Flows --- + getFlowDetailLink(flowId: string | number, flowName: string, flowType: FlowType): Link { + return detail(`Flow: ${flowName}`, this.url(`${this.flowPath(flowType)}/${flowId}`)); + } + + getFlowsDashboardLink(flowType: FlowType): Link { + const label = flowType === CONDITIONAL_FLOW_COMPONENT_ID ? 'Conditional Flows' : 'Flows'; + return dashboard(`${label} in the project`, this.url(this.flowPath(flowType))); + } + + getFlowsDocsLink(): Link { + return docs('Documentation for Keboola Flows', FLOW_DOCUMENTATION_URL); + } + + getFlowLinks(flowId: string | number, flowName: string, flowType: FlowType): Link[] { + return [ + this.getFlowDetailLink(flowId, flowName, flowType), + this.getFlowsDashboardLink(flowType), + this.getFlowsDocsLink(), + ]; + } + + // --- Schedulers --- + getSchedulerDetailLink(flowId: string | number, flowType: FlowType): Link { + return detail('Schedules', this.url(`${this.flowPath(flowType)}/${flowId}/schedules`)); + } + + // --- Components --- + getComponentConfigLink( + componentId: string, + configurationId: string, + configurationName: string, + ): Link { + if (isTransformationComponent(componentId)) { + return this.getTransformationConfigLink(componentId, configurationId, configurationName); + } + if (isDataAppComponent(componentId)) { + return this.getDataAppConfigLink(configurationId, configurationName, false); + } + if (isFlowType(componentId)) { + return this.getFlowDetailLink(configurationId, configurationName, componentId); + } + return detail( + `Configuration: ${configurationName}`, + this.url(`components/${componentId}/${configurationId}`), + ); + } + + getConfigDashboardLink(componentId: string, componentName: string | undefined): Link { + const label = componentName ? componentName : `Component "${componentId}"`; + return dashboard(`${label} Configurations Dashboard`, this.url(`components/${componentId}`)); + } + + getUsedComponentsLink(): Link { + return dashboard('Used Components Dashboard', this.url('components/configurations')); + } + + getConfigurationLinks( + componentId: string, + configurationId: string, + configurationName: string, + ): Link[] { + return [ + this.getComponentConfigLink(componentId, configurationId, configurationName), + this.getConfigDashboardLink(componentId, undefined), + ]; + } + + // --- Data Apps --- + getDataAppConfigLink( + configurationId: string, + configurationName: string, + usesBasicAuthentication: boolean, + ): Link { + const title = usesBasicAuthentication + ? `Data App Configuration (To see password, click on "OPEN DATA APP"): ${configurationName}` + : `Data App Configuration: ${configurationName}`; + return detail(title, this.url(`data-apps/${configurationId}`)); + } + + getDataAppDashboardLink(): Link { + return dashboard('Data Apps in the project', this.url('data-apps')); + } + + getDataAppDeploymentLink(deploymentLink: string): Link { + return detail('Data App Deployment', deploymentLink); + } + + getDataAppLinks( + configurationId: string, + configurationName: string, + deploymentLink?: string, + usesBasicAuthentication = false, + ): Link[] { + const links = [ + this.getDataAppConfigLink(configurationId, configurationName, usesBasicAuthentication), + this.getDataAppDashboardLink(), + ]; + if (deploymentLink) { + links.push(this.getDataAppDeploymentLink(deploymentLink)); + } + return links; + } + + // --- Transformations --- + getTransformationsDashboardLink(): Link { + return dashboard('Transformations dashboard', this.url('transformations-v2')); + } + + getTransformationConfigLink( + transformationType: string, + transformationId: string, + transformationName: string, + ): Link { + return detail( + `Transformation: ${transformationName}`, + this.url(`transformations-v2/${transformationType}/${transformationId}`), + ); + } + + getTransformationLinks( + transformationType: string, + transformationId: string, + transformationName: string, + ): Link[] { + return [ + this.getTransformationConfigLink(transformationType, transformationId, transformationName), + this.getTransformationsDashboardLink(), + ]; + } + + // --- Jobs --- + getJobDetailLink(jobId: string): Link { + return detail(`Job: ${jobId}`, this.url(`queue/${jobId}`)); + } + + getJobsDashboardLink(): Link { + return dashboard('Jobs in the project', this.url('queue')); + } + + getJobLinks(jobId: string): Link[] { + return [this.getJobDetailLink(jobId), this.getJobsDashboardLink()]; + } + + // --- Buckets --- + getBucketDetailLink(bucketId: string, bucketName: string): Link { + return detail(`Bucket: ${bucketName}`, this.url(`storage/${bucketId}`)); + } + + getBucketDashboardLink(): Link { + return dashboard('Buckets in the project', this.url('storage')); + } + + getBucketLinks(bucketId: string, bucketName: string): Link[] { + return [this.getBucketDetailLink(bucketId, bucketName), this.getBucketDashboardLink()]; + } + + // --- Tables --- + getTableDetailLink(bucketId: string, tableName: string): Link { + return detail(`Table: ${tableName}`, this.url(`storage/${bucketId}/table/${tableName}`)); + } + + getTableDetailLinkFromTableId(tableId: string): Link { + const segments = tableId.split('.'); + const tableName = segments[segments.length - 1]!; + const bucketId = segments.slice(0, -1).join('.'); + return this.getTableDetailLink(bucketId, tableName); + } + + getTableLinks(bucketId: string, bucketName: string, tableName: string): Link[] { + return [ + this.getTableDetailLink(bucketId, tableName), + this.getBucketDetailLink(bucketId, bucketName), + ]; + } +} diff --git a/src/logger.ts b/src/logger.ts new file mode 100644 index 000000000..917f5485d --- /dev/null +++ b/src/logger.ts @@ -0,0 +1,16 @@ +import pino from 'pino'; + +// Python log levels (DEBUG/INFO/...) map onto pino's lowercase levels. +const LEVELS: Record = { + DEBUG: 'debug', + INFO: 'info', + WARNING: 'warn', + ERROR: 'error', + CRITICAL: 'fatal', +}; + +export const createLogger = (level = 'INFO') => + // Logs go to stderr (fd 2) so stdout stays clean for the stdio JSON-RPC transport. + pino({ level: LEVELS[level.toUpperCase()] ?? 'info' }, pino.destination(2)); + +export const logger = createLogger(process.env.LOG_LEVEL); diff --git a/src/mcp/authorization.ts b/src/mcp/authorization.ts new file mode 100644 index 000000000..5093e664e --- /dev/null +++ b/src/mcp/authorization.ts @@ -0,0 +1,67 @@ +/** + * Header-based tool authorization, ported 1:1 from the Python + * `ToolAuthorizationMiddleware` (`authorization.py`). + * + * Authorization is configured via HTTP headers, surfaced onto the per-request Config + * (see `config.ts`): + * - `X-Allowed-Tools`: comma-separated allow list of tool names + * - `X-Disallowed-Tools`: comma-separated deny list (removed from the allowed set) + * - `X-Read-Only-Mode`: "true"/"1"/"yes" restricts to tools with `readOnlyHint=true` + * + * These headers are intended to be injected by infrastructure/proxy layers rather than + * set directly by end clients. + */ + +/** Parsed header-authorization configuration for a request. */ +export type AuthorizationConfig = { + /** Allow list, or `null` when no `X-Allowed-Tools` restriction is present. */ + allowedTools: Set | null; + /** Deny list, or `null` when no `X-Disallowed-Tools` restriction is present. */ + disallowedTools: Set | null; + /** Whether `X-Read-Only-Mode` is enabled. */ + readOnlyMode: boolean; +}; + +const READ_ONLY_TRUTHY = new Set(['true', '1', 'yes']); + +const parseToolSet = (raw: string | undefined): Set | null => { + if (!raw) return null; + const parsed = new Set( + raw + .split(',') + .map((t) => t.trim()) + .filter((t) => t.length > 0), + ); + return parsed.size > 0 ? parsed : null; +}; + +/** Builds the authorization config from the raw header values. */ +export const parseAuthorizationConfig = (raw: { + allowedTools?: string; + disallowedTools?: string; + readOnlyMode?: string; +}): AuthorizationConfig => ({ + allowedTools: parseToolSet(raw.allowedTools), + disallowedTools: parseToolSet(raw.disallowedTools), + readOnlyMode: READ_ONLY_TRUTHY.has((raw.readOnlyMode ?? '').toLowerCase()), +}); + +/** Whether any header-authorization filter is active. */ +export const hasAuthorizationFilters = (config: AuthorizationConfig): boolean => + config.allowedTools !== null || config.disallowedTools !== null || config.readOnlyMode; + +/** + * Header-based authorization decision for a single tool. Ported 1:1 from + * `_is_tool_name_authorized`: disallow list first, then read-only mode, then allow list. + */ +export const isToolNameAuthorized = ( + toolName: string, + isReadOnly: boolean, + config: AuthorizationConfig, +): boolean => { + const { allowedTools, disallowedTools, readOnlyMode } = config; + if (disallowedTools && disallowedTools.has(toolName)) return false; + if (readOnlyMode && !isReadOnly) return false; + if (allowedTools !== null && !allowedTools.has(toolName)) return false; + return true; +}; diff --git a/src/mcp/errors.ts b/src/mcp/errors.ts new file mode 100644 index 000000000..f48a8906b --- /dev/null +++ b/src/mcp/errors.ts @@ -0,0 +1,81 @@ +import type { ZodError, ZodIssue } from 'zod'; + +/** + * Validation-error formatting — port of `keboola_mcp_server.errors` + * (`_format_validation_errors`, `prettify_validation_error`, `ValidationErrorMiddleware`). + * + * The Python server used Pydantic + FastMCP middleware to catch a `ValidationError` + * raised during tool-argument validation and re-render it with explicit field locations + * so both humans and LLMs can see exactly which fields are missing or invalid. The + * TypeScript server validates with Zod, so the faithful equivalent formats a `ZodError`. + * + * The recovery-hint / per-exception logging behavior of the Python `tool_errors` + * decorator already lives in `@/mcp/tool` (`registerTool`); this module only adds the + * validation-error prettifier the Python `errors` module also provided. + */ + +export type FormattedValidationError = { + field: string; + message: string; + extra: Record; +}; + +export type FormattedValidationErrors = { + errors: FormattedValidationError[]; +}; + +/** + * Formats Zod validation issues into a structured object — port of + * `_format_validation_errors`. `field` is the dotted location path, `message` is the + * human-readable message, and `extra` carries every remaining issue field (e.g. the + * error `code`) as strings, matching the Python `extra` dict. + */ +export const formatValidationErrors = (issues: ZodIssue[]): FormattedValidationErrors => { + const errors: FormattedValidationError[] = issues.map((issue) => { + const extra: Record = {}; + for (const [key, value] of Object.entries(issue)) { + if (key === 'path' || key === 'message') continue; + extra[key] = typeof value === 'string' ? value : JSON.stringify(value); + } + return { + field: (issue.path ?? []).map((p) => String(p)).join('.'), + message: issue.message ?? 'Validation error', + extra, + }; + }); + return { errors }; +}; + +/** Renders the structured errors as a YAML-compatible block (no YAML dependency needed). */ +const toYaml = (formatted: FormattedValidationErrors): string => { + const lines: string[] = ['errors:']; + for (const err of formatted.errors) { + lines.push(`- field: ${err.field}`); + lines.push(` message: ${err.message}`); + const extraKeys = Object.keys(err.extra); + if (extraKeys.length === 0) { + lines.push(' extra: {}'); + } else { + lines.push(' extra:'); + for (const key of extraKeys) { + lines.push(` ${key}: ${err.extra[key]}`); + } + } + } + return `${lines.join('\n')}\n`; +}; + +/** + * Formats a Zod validation error into a human- and LLM-readable string — port of + * `prettify_validation_error`. Produces the same + * `Found N validation error(s) for ` header followed by the structured body. + * + * @param error The Zod validation error to format. + * @param modelName The name of the validated model/tool (Pydantic carried this on the + * error's `title`; Zod does not, so callers pass it explicitly). + */ +export const prettifyValidationError = (error: ZodError, modelName = 'unknown'): string => { + const issues = error.issues ?? []; + const header = `Found ${issues.length} validation error(s) for ${modelName}`; + return `${header}\n${toYaml(formatValidationErrors(issues))}`; +}; diff --git a/src/mcp/filtering.ts b/src/mcp/filtering.ts new file mode 100644 index 000000000..f1b4dbded --- /dev/null +++ b/src/mcp/filtering.ts @@ -0,0 +1,211 @@ +/** + * Project / role / branch tool gating, ported 1:1 from the Python + * `ToolsFilteringMiddleware` (`mcp.py`). + * + * The MCP TypeScript SDK has no FastMCP-style middleware, so the gating is applied + * by wrapping the low-level `tools/list` and `tools/call` request handlers after the + * tools are registered (see `wrapToolGating` in `server.ts`). This module is the + * single source of truth for the project-feature / token-role / branch rules — the + * same `authorizeToolCall` decision is used for both discovery (list) and execution + * (call). + */ + +export const SEMANTIC_TOOLING_FEATURE = 'mcp-semantic-tooling'; + +export const SEMANTIC_TOOL_NAMES = new Set([ + 'search_semantic_context', + 'get_semantic_context', + 'get_semantic_schema', + 'validate_semantic_query', +]); + +/** + * Data app tools are supported only in the main/production branch. This single set is + * the source of truth for both the list filter and the call guard — keeping them in + * sync is what prevents a new (possibly destructive) data app tool from leaking onto + * non-main branches. + */ +export const DATA_APP_BRANCH_GATED_TOOLS = new Set([ + 'modify_streamlit_data_app', + 'modify_python_js_data_app', + 'create_python_js_data_app_git_credential', + 'get_data_apps', + 'deploy_data_app', + 'delete_python_js_data_app_draft', +]); + +export const MODIFY_FLOW_TOOL_NAME = 'modify_flow'; +export const UPDATE_FLOW_TOOL_NAME = 'update_flow'; + +/** + * Tools served by the pgvector docs-search index. They are available only when the MCP + * has a configured, reachable index; otherwise they are filtered out of discovery and + * denied on call (RFC: feature_spec/docs-search-pgvector/, point 5). Nothing else is + * affected by the index being absent. + */ +export const DOCS_INDEX_TOOL_NAMES = new Set(['docs_query', 'find_component_id']); + +/** Token info as returned by the Storage API `tokens/verify` endpoint (loosely typed). */ +export type TokenInfo = Record; + +/** Minimal tool shape the gating needs (name + read-only hint). */ +export type GatedTool = { + name: string; + readOnly: boolean; +}; + +/** Whether the tool belongs to semantic tooling (name-based; TS tools carry no tags). */ +export const isSemanticToolName = (name: string): boolean => SEMANTIC_TOOL_NAMES.has(name); + +const asRecord = (value: unknown): Record | undefined => + value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; + +export const getProjectFeatures = (tokenInfo: TokenInfo): Set => { + const owner = asRecord(tokenInfo.owner); + const features = owner?.features; + if (!Array.isArray(features)) return new Set(); + return new Set(features.filter((f): f is string => typeof f === 'string' && f.length > 0)); +}; + +export const getTokenRole = (tokenInfo: TokenInfo): string => { + const admin = asRecord(tokenInfo.admin); + const role = admin?.role; + return typeof role === 'string' ? role : ''; +}; + +/** + * Context for a gating decision. `isMainBranch` is `branchId === undefined`; + * `isOauth` is whether a bearer token is present. + */ +export type GatingContext = { + tokenRole: string; + features: Set; + isOauth: boolean; + isMainBranch: boolean; + /** Whether a docs-search index is configured + reachable (gates the docs tools). */ + docsIndexAvailable: boolean; +}; + +/** + * Filters a tool list for `tools/list`. Ported from `on_list_tools`. Branch is + * always treated as main during discovery (the caller passes `isMainBranch=true`), + * matching the Python behavior of forcing `branch_id=None` for list requests. + */ +export const filterToolsList = (tools: GatedTool[], ctx: GatingContext): GatedTool[] => { + const { features, isOauth, isMainBranch } = ctx; + const tokenRole = ctx.tokenRole.toLowerCase(); + let result = tools; + + if (features.has('hide-conditional-flows')) { + result = result.filter((t) => t.name !== 'create_conditional_flow'); + } else { + result = result.filter((t) => t.name !== 'create_flow'); + } + + // Show modify_flow to admin/share or OAuth users; update_flow to everyone else. + if (tokenRole === 'admin' || tokenRole === 'share' || isOauth) { + result = result.filter((t) => t.name !== UPDATE_FLOW_TOOL_NAME); + } else { + result = result.filter((t) => t.name !== MODIFY_FLOW_TOOL_NAME); + } + + if (!isMainBranch) { + result = result.filter((t) => !DATA_APP_BRANCH_GATED_TOOLS.has(t.name)); + } + + if (tokenRole === 'readonly') { + result = result.filter((t) => t.readOnly); + } + + if (!features.has(SEMANTIC_TOOLING_FEATURE)) { + result = result.filter((t) => !isSemanticToolName(t.name)); + } + + if (!ctx.docsIndexAvailable) { + result = result.filter((t) => !DOCS_INDEX_TOOL_NAMES.has(t.name)); + } + + return result; +}; + +/** + * Decides whether a call to `toolName` is allowed. Ported 1:1 from + * `authorize_tool_call`. Returns a denial message, or `null` if allowed. + */ +export const authorizeToolCall = (params: { + toolName: string; + isReadOnly: boolean; + isSemantic: boolean; + tokenRole: string; + features: Set; + isOauth: boolean; + isMainBranch: boolean; + docsIndexAvailable: boolean; +}): string | null => { + const { toolName, isReadOnly, isSemantic, features, isOauth, isMainBranch } = params; + const tokenRole = params.tokenRole.toLowerCase(); + + if (!params.docsIndexAvailable && DOCS_INDEX_TOOL_NAMES.has(toolName)) { + return ( + `The tool "${toolName}" is not available: the Keboola documentation index is not ` + + 'configured or reachable. Contact your administrator to enable documentation search.' + ); + } + + if (tokenRole === 'readonly' && !isReadOnly) { + return ( + `Access denied: The tool "${toolName}" requires write permissions. ` + + `Your current role (${tokenRole}) only allows read-only operations. ` + + `Contact your administrator to request write access.` + ); + } + + if (!features.has(SEMANTIC_TOOLING_FEATURE) && isSemantic) { + return ( + `The tool "${toolName}" is not available in this project. ` + + 'Please ask Keboola support to enable "Semantic Layer Tooling" feature.' + ); + } + + if (features.has('hide-conditional-flows')) { + if (toolName === 'create_conditional_flow') { + return ( + 'The "create_conditional_flow" tool is not available in this project. ' + + 'Please ask Keboola support to enable "Conditional Flows" feature ' + + 'or use "create_flow" tool instead.' + ); + } + } else { + if (toolName === 'create_flow') { + return ( + 'The "create_flow" tool is not available in this project. ' + + 'This project uses "Conditional Flows", ' + + 'please use "create_conditional_flow" tool instead.' + ); + } + } + + if (tokenRole === 'admin' || tokenRole === 'share' || isOauth) { + if (toolName === UPDATE_FLOW_TOOL_NAME) { + return ( + 'The "update_flow" tool is not available for admin/OAuth tokens. ' + + `Use "${MODIFY_FLOW_TOOL_NAME}" to manage schedules instead.` + ); + } + } else { + if (toolName === MODIFY_FLOW_TOOL_NAME) { + return ( + `The "${MODIFY_FLOW_TOOL_NAME}" tool is not available for this token. ` + + `Use "${UPDATE_FLOW_TOOL_NAME}" to update flow configuration instead.` + ); + } + } + + if (DATA_APP_BRANCH_GATED_TOOLS.has(toolName) && !isMainBranch) { + return 'Data apps are supported only in the main production branch.'; + } + + return null; +}; diff --git a/src/mcp/tool.ts b/src/mcp/tool.ts new file mode 100644 index 000000000..2573392d4 --- /dev/null +++ b/src/mcp/tool.ts @@ -0,0 +1,81 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import type { CallToolResult, ToolAnnotations } from '@modelcontextprotocol/sdk/types.js'; +import type { z, ZodRawShape } from 'zod'; + +import { logger } from '@/logger'; +import { type ToolSerializer, toonSerializeCompact } from '@/serialize'; + +/** + * Declarative tool registration shared by every Keboola MCP tool. Wraps the SDK's + * `registerTool` to (1) serialize the handler's structured result to TOON text and + * (2) catch errors, append an optional recovery hint, and return an MCP error + * result — the parity equivalent of the Python `serializer=` + `tool_errors()`. + */ +export type ToolDefinition = { + name: string; + title?: string; + description: string; + inputSchema?: Shape; + annotations?: ToolAnnotations; + /** Output encoder; defaults to compact TOON (nulls dropped). */ + serializer?: ToolSerializer; + /** Recovery hint appended to error messages, to guide the model on failure. */ + recovery?: string; + handler: (args: z.infer>) => Promise | unknown; +}; + +/** + * Human-useful message from a tool error. `@keboola/api-client`'s `ApiError.message` is + * only the HTTP status text (e.g. "Bad Request") — the real reason and the support + * exception id live on `error.data` (`{ error, message, exceptionId }`). Surface them so + * the client sees "Bad Request: Invalid access token (exception ID: …)" instead of an + * opaque status. (Our raw client already composes this into its message, so it has no + * `.data` and passes through unchanged.) + */ +export const describeToolError = (error: unknown): string => { + const base = error instanceof Error ? error.message : String(error); + const data = (error as { data?: unknown }).data; + if (!data || typeof data !== 'object') return base; + const d = data as Record; + const detail = [d.error, d.message].find( + (v): v is string => typeof v === 'string' && v.trim().length > 0, + ); + let msg = detail && detail.trim() !== base ? `${base}: ${detail.trim()}` : base; + if (typeof d.exceptionId === 'string' && d.exceptionId.length > 0) { + msg += ` (exception ID: ${d.exceptionId})`; + } + return msg; +}; + +export const registerTool = ( + server: McpServer, + def: ToolDefinition, +): void => { + const serialize = def.serializer ?? toonSerializeCompact; + + server.registerTool( + def.name, + { + title: def.title, + description: def.description, + inputSchema: def.inputSchema ?? ({} as Shape), + annotations: def.annotations, + }, + // The SDK's registerTool overloads don't infer cleanly through a generic + // wrapper; the handler is fully typed at the ToolDefinition boundary, so we + // cast just this internal bridge callback. + (async (args: z.infer>): Promise => { + try { + const result = await def.handler(args); + // String results pass through verbatim (parity with FastMCP); objects are TOON-encoded. + const text = typeof result === 'string' ? result : serialize(result); + return { content: [{ type: 'text', text }] }; + } catch (error) { + const base = describeToolError(error); + const text = def.recovery ? `${base}\nRecovery: ${def.recovery}` : base; + logger.error({ err: error, tool: def.name }, `MCP tool "${def.name}" call failed`); + return { content: [{ type: 'text', text }], isError: true }; + } + }) as never, + ); +}; diff --git a/src/oauth.ts b/src/oauth.ts new file mode 100644 index 000000000..1369b13fd --- /dev/null +++ b/src/oauth.ts @@ -0,0 +1,717 @@ +/** + * OAuth provider for the MCP server. + * + * Port of the Python `keboola_mcp_server.oauth.SimpleOAuthProvider`. It implements a + * proxying OAuth 2.1 authorization server that delegates the actual authentication to + * the Keboola OAuth server. To avoid any persistence, all transient state (authorization + * request state, MCP authorization codes, MCP access/refresh tokens) is carried inside + * signed + gzipped JWS blobs that the server hands back to the client and later decodes. + * + * The provider also mints an extra Storage API (SAPI) token alongside every access token + * because some Keboola services (AI Service, Jobs Queue) still require the + * `X-StorageAPI-Token` header instead of `Authorization: Bearer `. + */ + +import { CompactSign, compactVerify } from 'jose'; +import { gunzipSync, gzipSync } from 'node:zlib'; + +import { logger } from '@/logger'; + +const OAUTH_LOG_ALL = Boolean(process.env.KEBOOLA_MCP_SERVER_OAUTH_LOG_ALL); + +const RE_LOCALHOST = /^(localhost|127\.0\.0\.1|\[::1]|::1)$/i; + +/** + * Whitelisted redirect-URI hosts, keyed by URL scheme. Mirrors the Python `_ALLOWED_DOMAINS`. + * Custom schemes (e.g. `cursor://`) require a handler registered in the browser and are used + * to redirect to a locally running app. + */ +const ALLOWED_DOMAINS: Record = { + https: [ + // Any keboola.com/dev subdomain EXCEPT user-deployable data-app subdomains, which live under a + // '*.hub..keboola.com' host. A free-trial user can deploy a data app whose '/callback' + // would otherwise capture the OAuth code, so we reject any host with a 'hub' DNS label (RISK-76). + /^(?!(?:.*\.)?hub\.).+\.keboola\.(com|dev)$/i, + /^(.*\.)?chatgpt\.com$/i, + /^(.*\.)?claude\.ai$/i, + /^librechat\.glami-ml\.com$/i, // no subdomains allowed + /^(.*\.)?make\.com$/i, + /^api\.devin\.ai$/i, // devin.ai API domain + /^cloud\.onyx\.app$/i, // onyx.app OAuth callback + /^global\.consent\.azure-apim\.net$/i, // Azure APIM consent domain + /^n8n\.groupondev\.com$/i, + /^n8n-business\.groupondev\.com$/i, + /^n8n-merchant\.groupondev\.com$/i, + /^n8n-llm-traffic\.groupondev\.com$/i, + /^n8n-finance\.groupondev\.com$/i, + /^n8n-playground\.groupondev\.com$/i, + /^n8n-staging\.groupondev\.com$/i, + ], + http: [RE_LOCALHOST], + cursor: [/^(anysphere\.cursor-retrieval|anysphere\.cursor-mcp)$/i], +}; + +/** Logs sensitive information only when `KEBOOLA_MCP_SERVER_OAUTH_LOG_ALL` is set. */ +const logDebug = (msg: string): void => { + if (OAUTH_LOG_ALL) { + logger.debug(msg); + } +}; + +/** Raised when a redirect URI is missing or not on the whitelist. */ +export class InvalidRedirectUriError extends Error {} + +/** Raised for HTTP-level errors during the OAuth flow; carries an HTTP status code. */ +export class OAuthHttpError extends Error { + constructor( + readonly status: number, + message: string, + ) { + super(message); + this.name = 'OAuthHttpError'; + } +} + +/** Mirrors the Python `mcp.server.auth.provider.AccessToken`. */ +export type AccessToken = { + token: string; + client_id: string; + scopes: string[]; + expires_at: number | null; + resource?: string | null; +}; + +/** Mirrors the Python `RefreshToken`. */ +export type RefreshToken = { + token: string; + client_id: string; + scopes: string[]; + expires_at: number | null; +}; + +/** Access token wrapping the delegate OAuth token plus the extra SAPI token. */ +export type ProxyAccessToken = AccessToken & { + delegate: AccessToken; + // Created by the MCP server for calling AI Service and Jobs Queue, which do not yet + // support 'Authorization: Bearer '. + sapi_token: string; +}; + +/** Refresh token wrapping the delegate OAuth refresh token. */ +export type ProxyRefreshToken = RefreshToken & { + delegate: RefreshToken; +}; + +/** Authorization code carrying the OAuth tokens, mirroring `_ExtendedAuthorizationCode`. */ +export type ExtendedAuthorizationCode = { + code: string; + scopes: string[]; + expires_at: number | null; + client_id: string; + code_challenge: string | null; + redirect_uri: string; + redirect_uri_provided_explicitly: boolean; + oauth_access_token: AccessToken; + oauth_refresh_token: RefreshToken; +}; + +/** The authorization parameters sent by the downstream MCP OAuth client (e.g. claude.ai). */ +export type AuthorizationParams = { + state?: string | null; + scopes?: string[] | null; + codeChallenge: string; + redirectUri: string; + redirectUriProvidedExplicitly: boolean; +}; + +/** OAuth token response, mirroring `mcp.shared.auth.OAuthToken`. */ +export type OAuthToken = { + access_token: string; + refresh_token: string; + token_type: 'Bearer'; + expires_in: number; + scope: string; +}; + +type ParsedRedirectUri = { + scheme: string; + host: string; + port: number | null; +}; + +/** + * Parses a redirect URI into scheme/host/port without normalizing away custom schemes. + * The host is the authority for hierarchical URIs (`scheme://host[:port]/...`) or, for + * opaque custom schemes such as `cursor:anysphere.cursor-mcp/...`, the part before the path. + */ +const parseRedirectUri = (uri: string): ParsedRedirectUri | null => { + const match = /^([a-zA-Z][a-zA-Z0-9+.-]*):\/\/([^/?#]*)(?:[/?#]|$)/.exec(uri); + if (!match) { + // Opaque form: scheme:authority/path (e.g. cursor:anysphere.cursor-mcp/cb) + const opaque = /^([a-zA-Z][a-zA-Z0-9+.-]*):([^/?#]*)(?:[/?#]|$)/.exec(uri); + if (!opaque) { + return null; + } + return { scheme: opaque[1]!.toLowerCase(), host: opaque[2] ?? '', port: null }; + } + + const scheme = match[1]!.toLowerCase(); + const authority = match[2] ?? ''; + // IPv6 literal: [::1]:port + const v6 = /^(\[[^\]]*])(?::(\d+))?$/.exec(authority); + if (v6) { + return { scheme, host: v6[1]!, port: v6[2] ? Number(v6[2]) : null }; + } + const lastColon = authority.lastIndexOf(':'); + if (lastColon !== -1 && /^\d+$/.test(authority.slice(lastColon + 1))) { + return { + scheme, + host: authority.slice(0, lastColon), + port: Number(authority.slice(lastColon + 1)), + }; + } + return { scheme, host: authority, port: null }; +}; + +/** + * Validates a redirect URI against the whitelist. Mirrors `_OAuthClientInformationFull.validate_redirect_uri`. + * + * Because there is no persistent client registry, we require the client to send its redirect URI in the + * authorization request and discard every URI whose scheme/host is not whitelisted. + * + * @returns the redirect URI unchanged when valid. + * @throws InvalidRedirectUriError when missing, scheme-less, or not whitelisted. + */ +export const validateRedirectUri = (redirectUri: string | null | undefined): string => { + if (!redirectUri) { + logger.warn('[validateRedirectUri] No redirect_uri specified.'); + throw new InvalidRedirectUriError('The redirect_uri must be specified.'); + } + + const parsed = parseRedirectUri(redirectUri); + if (!parsed || !parsed.scheme) { + logger.warn(`[validateRedirectUri] No scheme in redirect_uri: ${redirectUri}`); + throw new InvalidRedirectUriError(`Invalid redirect_uri: ${redirectUri}`); + } + + const allowedDomains = ALLOWED_DOMAINS[parsed.scheme]; + if (allowedDomains) { + if (!allowedDomains.some((p) => p.test(parsed.host) && fullMatch(p, parsed.host))) { + logger.warn(`[validateRedirectUri] Unknown domain in redirect_uri: ${redirectUri}`); + throw new InvalidRedirectUriError(`Invalid redirect_uri: ${redirectUri}`); + } + } else { + logger.warn(`[validateRedirectUri] Forbidden scheme in redirect_uri: ${redirectUri}`); + throw new InvalidRedirectUriError(`Invalid redirect_uri: ${redirectUri}`); + } + + logger.info(`[validateRedirectUri] Accepted redirect_uri: ${redirectUri}]`); + return redirectUri; +}; + +/** Python's `re.fullmatch` semantics: the pattern must match the entire string. */ +const fullMatch = (pattern: RegExp, value: string): boolean => { + const m = pattern.exec(value); + return m !== null && m[0] === value; +}; + +/** Appends params to a base URI's query string, skipping null/undefined. Mirrors `construct_redirect_uri`. */ +export const constructRedirectUri = ( + base: string, + params: Record, +): string => { + const hashIdx = base.indexOf('#'); + const fragment = hashIdx === -1 ? '' : base.slice(hashIdx); + const withoutFragment = hashIdx === -1 ? base : base.slice(0, hashIdx); + + const queryIdx = withoutFragment.indexOf('?'); + const head = queryIdx === -1 ? withoutFragment : withoutFragment.slice(0, queryIdx); + const existing = queryIdx === -1 ? '' : withoutFragment.slice(queryIdx + 1); + + const search = new URLSearchParams(existing); + for (const [key, value] of Object.entries(params)) { + if (value !== null && value !== undefined) { + search.append(key, value); + } + } + + const query = search.toString(); + return `${head}${query ? `?${query}` : ''}${fragment}`; +}; + +/** Joins a path onto a base URL the way Python's `urljoin(base, path)` does for absolute paths. */ +const urljoin = (base: string, path: string): string => new URL(path, base).toString(); + +export type SimpleOAuthProviderOptions = { + storageApiUrl: string; + mcpServerUrl: string; + callbackEndpoint: string; + clientId: string; + clientSecret: string; + serverUrl: string; + scope: string; + jwtSecret?: string; +}; + +const ceilToHour = (seconds: number): number => Math.ceil(seconds / 3600) * 3600; + +const randomHex = (bytes: number): string => { + const arr = new Uint8Array(bytes); + globalThis.crypto.getRandomValues(arr); + return Array.from(arr, (b) => b.toString(16).padStart(2, '0')).join(''); +}; + +const nowSeconds = (): number => Date.now() / 1000; + +/** + * Proxying OAuth provider. Port of `SimpleOAuthProvider`. + * + * Dynamic Client Registration is supported but never persisted: `getClient`/`registerClient` + * are effectively no-ops, redirect-URI and scope validation are relaxed (we instead whitelist + * redirect URIs at authorize time), and all state travels inside signed JWS blobs. + */ +export class SimpleOAuthProvider { + private readonly sapiTokensUrl: string; + private readonly mcpCallbackUrl: string; + private readonly oauthClientId: string; + private readonly oauthClientSecret: string; + private readonly oauthServerAuthUrl: string; + private readonly oauthServerTokenUrl: string; + private readonly oauthScope: string; + private readonly jwtSecret: Uint8Array; + + constructor(opts: SimpleOAuthProviderOptions) { + this.sapiTokensUrl = urljoin(opts.storageApiUrl, '/v2/storage/tokens'); + this.mcpCallbackUrl = urljoin(opts.mcpServerUrl, opts.callbackEndpoint); + this.oauthClientId = opts.clientId; + this.oauthClientSecret = opts.clientSecret; + this.oauthServerAuthUrl = urljoin(opts.serverUrl, '/oauth/authorize'); + this.oauthServerTokenUrl = urljoin(opts.serverUrl, '/oauth/token'); + this.oauthScope = opts.scope; + this.jwtSecret = new TextEncoder().encode(opts.jwtSecret || randomHex(32)); + } + + /** + * Creates the URL that redirects to the OAuth server for authorization. The state parameter + * is a signed JWS carrying all authorization parameters and expiring after 5 minutes. + */ + async authorize(clientId: string, params: AuthorizationParams): Promise { + const scopes = params.scopes ?? []; + const state = { + redirect_uri: params.redirectUri, + redirect_uri_provided_explicitly: String(params.redirectUriProvidedExplicitly), + // the scopes sent by the MCP server's OAuth client (e.g. claude.ai) + scopes, + code_challenge: params.codeChallenge, + state: params.state ?? null, + client_id: clientId, + expires_at: nowSeconds() + 5 * 60, // 5 minutes from now + }; + const stateJwt = await this.encode(state); + + const urlParams: Record = { + client_id: this.oauthClientId, + response_type: 'code', + redirect_uri: this.mcpCallbackUrl, + state: stateJwt, + // send no scopes to Keboola OAuth server and let it use its own default scope + }; + + return constructRedirectUri(this.oauthServerAuthUrl, urlParams); + } + + /** + * Handles the callback from the OAuth server: validates the state, exchanges the code with the + * OAuth server, and returns the redirect URL back to the downstream AI assistant OAuth client. + */ + async handleOAuthCallback(code: string, state: string): Promise { + let stateData: Record | undefined; + try { + stateData = await this.decode(state); + } catch { + logDebug(`[handleOAuthCallback] Invalid state: ${state}`); + throw new OAuthHttpError(400, 'Invalid state parameter'); + } + + if (!stateData) { + throw new OAuthHttpError(400, 'Invalid state parameter'); + } + + if ((stateData['expires_at'] as number) < nowSeconds()) { + logDebug(`[handleOAuthCallback] Expired state`); + throw new OAuthHttpError(400, 'Invalid state parameter'); + } + + const response = await this.fetchJson(this.oauthServerTokenUrl, { + client_id: this.oauthClientId, + client_secret: this.oauthClientSecret, + code, + grant_type: 'authorization_code', + // Keboola OAuth server requires the redirect_uri; the GitHub one does not. + redirect_uri: this.mcpCallbackUrl, + }); + + if (response.status !== 200) { + logger.error( + `[handleOAuthCallback] Failed to exchange code for token, OAuth server response: ` + + `status=${response.status}, text=${response.text}`, + ); + throw new OAuthHttpError( + 400, + `Failed to exchange code for token: status=${response.status}, text=${response.text}`, + ); + } + + const data = response.json as Record; + if ('error' in data) { + logger.error( + `[handleOAuthCallback] Error when exchanging code for token: data=${JSON.stringify(data)}`, + ); + throw new OAuthHttpError(400, String(data['error_description'] ?? data['error'])); + } + + const redirectUri = stateData['redirect_uri'] as string; + const scopes = stateData['scopes'] as string[]; + const [accessToken, refreshToken] = this.readOauthTokens(data, scopes); + + const authCode = { + code: `mcp_${randomHex(16)}`, + client_id: stateData['client_id'], + redirect_uri: redirectUri, + redirect_uri_provided_explicitly: stateData['redirect_uri_provided_explicitly'] === 'True', + expires_at: Math.trunc(nowSeconds() + 5 * 60), // 5 minutes from now + scopes, + code_challenge: stateData['code_challenge'], + oauth_access_token: accessToken, + oauth_refresh_token: refreshToken, + }; + const authCodeJwt = await this.encode(authCode); + + return constructRedirectUri(redirectUri, { + code: authCodeJwt, + state: stateData['state'] as string | null, + code_challenge: stateData['code_challenge'] as string | null, + }); + } + + /** + * Decodes + validates a JWS authorization code, returning the `ExtendedAuthorizationCode` + * or `null` when the code is invalid. (Expiry is logged but not enforced here, matching Python.) + */ + async loadAuthorizationCode( + authorizationCode: string, + ): Promise { + let raw: Record; + try { + raw = await this.decode(authorizationCode); + } catch { + logDebug(`[loadAuthorizationCode] Invalid authorization_code: ${authorizationCode}`); + return null; + } + + const authCode = raw as unknown as ExtendedAuthorizationCode; + const now = nowSeconds(); + if (authCode.expires_at && authCode.expires_at < now) { + logger.info( + `[loadAuthorizationCode] Expired authorization code: expires_at=${authCode.expires_at}, now=${now}`, + ); + } + return authCode; + } + + /** + * Swaps an authorization code for fresh MCP access + refresh tokens and mints a SAPI token. + * Mirrors `exchange_authorization_code`. + */ + async exchangeAuthorizationCode( + clientId: string, + authorizationCode: ExtendedAuthorizationCode, + ): Promise { + const expiresIn = Math.max( + 0, + Math.trunc((authorizationCode.oauth_access_token.expires_at ?? 0) - nowSeconds()), + ); + const sapiToken = await this.createSapiToken( + authorizationCode.oauth_access_token.token, + ceilToHour(expiresIn * 2), // twice as much as the access token's time out + ); + + const accessToken: ProxyAccessToken = { + token: `mcp_${randomHex(32)}`, + client_id: clientId, + scopes: authorizationCode.scopes, + expires_at: authorizationCode.oauth_access_token.expires_at, + delegate: authorizationCode.oauth_access_token, + sapi_token: sapiToken, + }; + const accessTokenJwt = await this.encode(accessToken); + + const refreshToken: ProxyRefreshToken = { + token: `mcp_${randomHex(32)}`, + client_id: clientId, + scopes: authorizationCode.scopes, + expires_at: authorizationCode.oauth_refresh_token.expires_at, + delegate: authorizationCode.oauth_refresh_token, + }; + const refreshTokenJwt = await this.encode(refreshToken); + + return { + access_token: accessTokenJwt, + refresh_token: refreshTokenJwt, + token_type: 'Bearer', + expires_in: expiresIn, + scope: accessToken.scopes.join(' '), + }; + } + + /** Decodes + validates a JWS access token. Returns `null` when invalid. */ + async loadAccessToken(token: string): Promise { + let raw: Record; + try { + raw = await this.decode(token); + } catch { + logDebug(`[loadAccessToken] Invalid token: ${token}`); + return null; + } + + const proxyToken = raw as unknown as ProxyAccessToken; + const now = nowSeconds(); + if (proxyToken.expires_at && proxyToken.expires_at < now) { + logger.info( + `[loadAccessToken] Expired access token: expires_at=${proxyToken.expires_at}, now=${now}`, + ); + } + return proxyToken; + } + + /** Decodes + validates a JWS refresh token. Returns `null` when invalid. */ + async loadRefreshToken(refreshToken: string): Promise { + let raw: Record; + try { + raw = await this.decode(refreshToken); + } catch { + logDebug(`[loadRefreshToken] Invalid token: ${refreshToken}`); + return null; + } + + const proxyToken = raw as unknown as ProxyRefreshToken; + const now = nowSeconds(); + if (proxyToken.expires_at && proxyToken.expires_at < now) { + logger.info( + `[loadRefreshToken] Expired refresh token: expires_at=${proxyToken.expires_at}, now=${now}`, + ); + } + return proxyToken; + } + + /** + * Swaps a refresh token for fresh MCP access + refresh tokens and mints a SAPI token. + * Mirrors `exchange_refresh_token`. + */ + async exchangeRefreshToken( + clientId: string, + refreshToken: ProxyRefreshToken, + scopes?: string[], + ): Promise { + const response = await this.fetchJson(this.oauthServerTokenUrl, { + client_id: this.oauthClientId, + client_secret: this.oauthClientSecret, + grant_type: 'refresh_token', + refresh_token: refreshToken.delegate.token, + }); + + if (response.status !== 200) { + logger.error( + `[exchangeRefreshToken] Failed to refresh token, OAuth server response: ` + + `status=${response.status}, text=${response.text}`, + ); + throw new OAuthHttpError( + 400, + `Failed to refresh token: status=${response.status}, text=${response.text}`, + ); + } + + const data = response.json as Record; + if ('error' in data) { + logger.error( + `[exchangeRefreshToken] Error when refreshing token: data=${JSON.stringify(data)}`, + ); + throw new OAuthHttpError(400, String(data['error_description'] ?? data['error'])); + } + + const [oauthAccessToken, oauthRefreshToken] = this.readOauthTokens( + data, + scopes && scopes.length > 0 ? scopes : refreshToken.scopes, + ); + const expiresIn = Math.max(0, Math.trunc((oauthAccessToken.expires_at ?? 0) - nowSeconds())); + const sapiToken = await this.createSapiToken(oauthAccessToken.token, ceilToHour(expiresIn * 2)); + + const accessToken: ProxyAccessToken = { + token: `mcp_${randomHex(32)}`, + client_id: clientId, + scopes: oauthAccessToken.scopes, + expires_at: oauthAccessToken.expires_at, + delegate: oauthAccessToken, + sapi_token: sapiToken, + }; + const accessTokenJwt = await this.encode(accessToken); + + const newRefreshToken: ProxyRefreshToken = { + token: `mcp_${randomHex(32)}`, + client_id: clientId, + scopes: oauthRefreshToken.scopes, + expires_at: oauthRefreshToken.expires_at, + delegate: oauthRefreshToken, + }; + const refreshTokenJwt = await this.encode(newRefreshToken); + + return { + access_token: accessTokenJwt, + refresh_token: refreshTokenJwt, + token_type: 'Bearer', + expires_in: Math.max(0, Math.trunc((accessToken.expires_at ?? 0) - nowSeconds())), + scope: accessToken.scopes.join(' '), + }; + } + + /** No-op: tokens are not stored, so there is nothing to revoke. */ + async revokeToken(token: string, tokenTypeHint?: string): Promise { + logDebug(`[revokeToken] token=${token}, token_type_hint=${tokenTypeHint ?? ''}`); + } + + /** + * Reads the access + refresh tokens from the OAuth server response. Mirrors `_read_oauth_tokens`. + * The refresh token lifetime is derived from the access token's: roughly one week by default. + */ + readOauthTokens(data: Record, scopes: string[]): [AccessToken, RefreshToken] { + const expiresIn = Number(data['expires_in']); // seconds + if (expiresIn <= 0) { + logger.error( + `[readOauthTokens] Received already expired token: data=${JSON.stringify(data)}`, + ); + throw new OAuthHttpError(400, 'The original OAuth access token has already expired.'); + } + + const currentTime = Math.trunc(nowSeconds()); + + const accessToken: AccessToken = { + token: data['access_token'] as string, + client_id: this.oauthClientId, + scopes, + // slightly different from 'expires_at' kept by the OAuth server + expires_at: currentTime + expiresIn, + }; + const refreshToken: RefreshToken = { + token: data['refresh_token'] as string, + client_id: this.oauthClientId, + scopes, + // The expires_in refers to the access token; there is no way to know when the refresh + // token expires. Keboola issues refresh tokens lasting ~1 month and access tokens ~1 hour. + // We derive the refresh-token lifespan from the access-token lifespan, ~1 week by default. + expires_at: currentTime + ceilToHour(Math.min(168 * expiresIn, 168 * 3600)), + }; + + return [accessToken, refreshToken]; + } + + /** Creates a Storage API token for services that do not yet support bearer tokens. */ + async createSapiToken(oauthAccessToken: string, expiresIn: number): Promise { + const res = await fetch(this.sapiTokensUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + Authorization: `Bearer ${oauthAccessToken}`, + }, + body: JSON.stringify({ + description: 'Created by the MCP server.', + expiresIn, + canReadAllFileUploads: true, + canManageBuckets: true, + }), + }); + + if (res.status !== 200) { + const text = await res.text(); + logger.error( + `[createSapiToken] Failed to create Storage API token, Storage API response: ` + + `status=${res.status}, text=${text}`, + ); + throw new OAuthHttpError( + res.status, + `Failed to create Storage API token: status=${res.status}, text=${text}`, + ); + } + + const data = (await res.json()) as { token: string }; + return data.token; + } + + /** POSTs a form-urlencoded body and returns status + parsed JSON (or raw text). */ + private async fetchJson( + url: string, + form: Record, + ): Promise<{ status: number; json: unknown; text: string }> { + const res = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Accept: 'application/json', + }, + body: new URLSearchParams(form).toString(), + redirect: 'follow', + }); + const text = await res.text(); + let json: unknown = {}; + try { + json = JSON.parse(text); + } catch { + json = {}; + } + return { status: res.status, json, text }; + } + + /** + * Encodes a value as a compact JWS: JSON → UTF-8 → gzip → HS256-signed JWS. + * Mirrors the Python `_encode` (gzip payload signed via `jwt.api_jws.encode`). + */ + async encode(data: unknown, key?: string): Promise { + const jsonBytes = new TextEncoder().encode(JSON.stringify(data)); + const gz = gzipSync(jsonBytes); + const signingKey = key ? new TextEncoder().encode(key) : this.jwtSecret; + return new CompactSign(gz).setProtectedHeader({ alg: 'HS256', typ: 'JWT' }).sign(signingKey); + } + + /** Inverse of {@link encode}. Throws when the signature is invalid. Mirrors `_decode`. */ + async decode(data: string, key?: string): Promise> { + const signingKey = key ? new TextEncoder().encode(key) : this.jwtSecret; + const { payload } = await compactVerify(data, signingKey, { algorithms: ['HS256'] }); + const jsonStr = gunzipSync(Buffer.from(payload)).toString('utf-8'); + return JSON.parse(jsonStr) as Record; + } +} + +/** Builds the OAuth authorization-server metadata document (`.well-known`). */ +export const buildAuthorizationServerMetadata = (mcpServerUrl: string): Record => { + const issuer = new URL(mcpServerUrl).origin; + return { + issuer, + authorization_endpoint: `${issuer}/authorize`, + token_endpoint: `${issuer}/token`, + registration_endpoint: `${issuer}/register`, + response_types_supported: ['code'], + response_modes_supported: ['query'], + grant_types_supported: ['authorization_code', 'refresh_token'], + token_endpoint_auth_methods_supported: ['client_secret_post', 'none'], + code_challenge_methods_supported: ['S256'], + }; +}; + +/** Builds the OAuth protected-resource metadata document (`.well-known`). */ +export const buildProtectedResourceMetadata = (mcpServerUrl: string): Record => { + const issuer = new URL(mcpServerUrl).origin; + return { + resource: issuer, + authorization_servers: [issuer], + }; +}; diff --git a/src/preview.ts b/src/preview.ts new file mode 100644 index 000000000..116b366c4 --- /dev/null +++ b/src/preview.ts @@ -0,0 +1,500 @@ +/** + * `/preview/configuration` custom HTTP endpoint — a faithful port of the Python + * `preview.py` (`preview_config_diff`). + * + * It simulates a config-changing MCP tool ("update_config", "update_config_row", …) + * WITHOUT writing anything, returning the original vs. updated configuration so a UI + * can show a diff before the user commits. The route lives outside the MCP tool-call + * path, so it re-applies the exact same authorization the real tool call would get: + * + * 1. Header authorization (`X-Allowed-Tools` / `X-Disallowed-Tools` / `X-Read-Only-Mode`) + * — port of `ToolAuthorizationMiddleware`. + * 2. Project-feature / token-role / branch gating via {@link authorizeToolCall} — the + * AI-3438 hardening: without it a restricted caller could drive a write tool's + * preview (e.g. a data-app tool on a non-main branch, or any write tool with a + * read-only token). + * + * Everything runs against a READ-ONLY Keboola client: the raw clients are built with + * `readonly: true`, so any accidental non-GET request throws instead of mutating. + */ + +import type { ErrorObject } from 'ajv'; +// zod emits draft 2020-12 JSON Schema, so use the matching Ajv build. +import Ajv2020 from 'ajv/dist/2020.js'; +import { z } from 'zod'; + +import { createKeboolaClients, type KeboolaClients } from '@/clients/keboola'; +import { createRawClient } from '@/clients/raw'; +import { deriveServiceUrls } from '@/clients/urls'; +import type { Config } from '@/config'; +import { logger } from '@/logger'; +import { + hasAuthorizationFilters, + isToolNameAuthorized, + parseAuthorizationConfig, +} from '@/mcp/authorization'; +import { + authorizeToolCall, + getProjectFeatures, + getTokenRole, + isSemanticToolName, + type TokenInfo, +} from '@/mcp/filtering'; +import { createServer } from '@/server'; +import type { ConfigParamUpdate } from '@/tools/components'; +import { configPreviewInternals } from '@/tools/components'; + +type JsonDict = Record; + +/** Error that carries an HTTP status code, mirroring the Python JSONResponse status codes. */ +export class PreviewHttpError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message); + this.name = 'PreviewHttpError'; + } +} + +// --------------------------------------------------------------------------- +// Request / response shapes (port of the Pydantic models, including alias handling). +// --------------------------------------------------------------------------- + +/** Reads the first present alias from a record, mirroring Pydantic `AliasChoices`. */ +const pickAlias = (body: JsonDict, aliases: string[]): unknown => { + for (const alias of aliases) { + if (alias in body) return body[alias]; + } + return undefined; +}; + +export type PreviewConfigDiffRq = { + toolName: string; + toolParams: JsonDict; +}; + +/** Parses the POST body into the request model (port of `PreviewConfigDiffRq.model_validate`). */ +export const parsePreviewRequest = (raw: unknown): PreviewConfigDiffRq => { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + throw new PreviewHttpError('Request body must be a JSON object.', 400); + } + const body = raw as JsonDict; + const toolName = pickAlias(body, ['toolName', 'tool_name', 'tool-name', 'ToolName']); + const toolParams = pickAlias(body, ['toolParams', 'tool_params', 'tool-params', 'ToolParams']); + if (typeof toolName !== 'string' || toolName.length === 0) { + throw new PreviewHttpError('Field "toolName" is required and must be a string.', 400); + } + if (!toolParams || typeof toolParams !== 'object' || Array.isArray(toolParams)) { + throw new PreviewHttpError('Field "toolParams" is required and must be an object.', 400); + } + return { toolName, toolParams: toolParams as JsonDict }; +}; + +export type ConfigCoordinates = { + componentId?: string | null; + configurationId?: string | null; + configurationRowId?: string | null; +}; + +export type PreviewConfigDiffResp = { + coordinates: ConfigCoordinates; + originalConfig: JsonDict | null; + updatedConfig: JsonDict | null; + isValid: boolean; + validationErrors?: string[] | null; +}; + +const toStringOrNull = (value: unknown): string | null => + value === undefined || value === null ? null : String(value); + +/** + * Serializes the response with camelCase aliases and `exclude_none` semantics + * (drops null/undefined entries, like Python's `model_dump(by_alias=True, exclude_none=True)`). + */ +const serializeResponse = (resp: PreviewConfigDiffResp): JsonDict => { + const coordinates: JsonDict = {}; + if (resp.coordinates.componentId != null) coordinates.componentId = resp.coordinates.componentId; + if (resp.coordinates.configurationId != null) { + coordinates.configurationId = resp.coordinates.configurationId; + } + if (resp.coordinates.configurationRowId != null) { + coordinates.configurationRowId = resp.coordinates.configurationRowId; + } + + const out: JsonDict = { coordinates, isValid: resp.isValid }; + if (resp.originalConfig != null) out.originalConfig = resp.originalConfig; + if (resp.updatedConfig != null) out.updatedConfig = resp.updatedConfig; + if (resp.validationErrors != null) out.validationErrors = resp.validationErrors; + return out; +}; + +// --------------------------------------------------------------------------- +// Tool metadata lookup (input schema + read-only hint), parity with the Python +// `app.state.mcp_tools_input_schema` / `mcp_read_only_tools` built from list_tools(). +// --------------------------------------------------------------------------- + +type RegisteredTool = { + inputSchema?: z.ZodType; + annotations?: { readOnlyHint?: boolean }; +}; + +type ToolMetadata = { + /** JSON Schema (draft) for the tool input, or undefined when the tool has no schema. */ + inputSchema?: JsonDict; + isReadOnly: boolean; +}; + +/** + * Enumerates the registered tools (unfiltered, like Python's `list_tools(run_middleware=False)`) + * and returns the JSON Schema + read-only hint for the named tool. Building a server is the + * single source of truth for tool schemas; no per-tool schema is duplicated here. + */ +const lookupToolMetadata = (config: Config, toolName: string): ToolMetadata | undefined => { + const server = createServer(config) as unknown as { + _registeredTools: Record; + }; + const tool = server._registeredTools[toolName]; + if (!tool) return undefined; + // `io: 'input'` so fields with defaults stay optional (matching what the tool call + // actually accepts, and the schema the MCP SDK advertises in tools/list). + const inputSchema = tool.inputSchema + ? (z.toJSONSchema(tool.inputSchema, { io: 'input' }) as JsonDict) + : undefined; + return { inputSchema, isReadOnly: tool.annotations?.readOnlyHint === true }; +}; + +// --------------------------------------------------------------------------- +// Read-only client. +// --------------------------------------------------------------------------- + +/** + * Builds a Keboola client whose raw HTTP clients reject any non-GET request. The + * preview path only reads (config detail, component fetch) + validates in memory, so a + * read-only client is both sufficient and a safety net against accidental writes — + * mirroring the Python `create_session_state(..., readonly=True)`. + */ +const createReadOnlyClients = (config: Config): KeboolaClients => { + const clients = createKeboolaClients(config); + if (!config.storageApiUrl) { + throw new Error('Storage API URL is not configured.'); + } + const urls = deriveServiceUrls(config.storageApiUrl); + const token = config.storageToken!; + const storageToken = config.bearerToken ? `Bearer ${config.bearerToken}` : token; + return { + ...clients, + rawStorage: createRawClient({ + baseUrl: `${urls.storage}/v2/storage`, + token: storageToken, + readonly: true, + }), + rawQueue: createRawClient({ baseUrl: urls.queue, token, readonly: true }), + rawAi: createRawClient({ baseUrl: urls.ai, token, readonly: true }), + rawSyncActions: createRawClient({ baseUrl: urls.syncActions, token, readonly: true }), + }; +}; + +// --------------------------------------------------------------------------- +// Schema validation (port of `_validate_tool_params`, using ajv in place of jsonschema). +// --------------------------------------------------------------------------- + +const formatValidationError = (toolName: string, error: ErrorObject): string => { + // `instancePath` like "/parameter_updates/1" -> "parameter_updates.1" (parity with + // Python's '.'.join(e.path)). + const field = error.instancePath.replace(/^\//, '').replaceAll('/', '.'); + const message = error.message ?? 'is invalid'; + const detail = field ? `${field}: ${message}` : message; + return `Found 1 validation error for ${toolName}:\n${detail}`; +}; + +/** + * Validates raw tool params against the tool's JSON Schema. Returns the error message + * (already formatted) when invalid, or null when valid. Mirrors `_validate_tool_params`. + */ +const validateToolParams = ( + toolName: string, + toolParams: JsonDict, + schema: JsonDict, +): string | null => { + try { + const ajv = new Ajv2020({ allErrors: false, strict: false }); + const validate = ajv.compile(schema); + if (validate(toolParams)) return null; + const first = validate.errors?.[0]; + if (!first) return `Found 1 validation error for ${toolName}:\nunknown validation error`; + return formatValidationError(toolName, first); + } catch (error) { + logger.error({ err: error, tool: toolName }, '[preview] Invalid tool schema'); + return 'Internal error: Invalid tool schema'; + } +}; + +// --------------------------------------------------------------------------- +// Coordinate extraction (port of `_extract_coordinates`). +// --------------------------------------------------------------------------- + +const UPDATE_FLOW_TOOL_NAME = 'update_flow'; +const MODIFY_FLOW_TOOL_NAME = 'modify_flow'; +const DATA_APP_COMPONENT_ID = 'keboola.data-apps'; + +const extractCoordinates = (toolName: string, toolParams: JsonDict): ConfigCoordinates => { + switch (toolName) { + case 'update_config': + return { + componentId: toStringOrNull(toolParams.component_id), + configurationId: toStringOrNull(toolParams.configuration_id), + }; + case 'update_config_row': + return { + componentId: toStringOrNull(toolParams.component_id), + configurationId: toStringOrNull(toolParams.configuration_id), + configurationRowId: toStringOrNull(toolParams.configuration_row_id), + }; + case UPDATE_FLOW_TOOL_NAME: + case MODIFY_FLOW_TOOL_NAME: + return { + componentId: toStringOrNull(toolParams.flow_type), + configurationId: toStringOrNull(toolParams.configuration_id), + }; + case 'modify_streamlit_data_app': + return { + componentId: DATA_APP_COMPONENT_ID, + configurationId: toStringOrNull(toolParams.configuration_id), + }; + case 'update_sql_transformation': + // The Python endpoint resolves the component id from the workspace SQL dialect. + // The diff for this tool is not implemented in the TS port (see DIFF_IMPLEMENTED_TOOLS), + // so only the configuration id is surfaced here. + return { configurationId: toStringOrNull(toolParams.configuration_id) }; + default: + throw new PreviewHttpError(`Invalid tool name: "${toolName}"`, 400); + } +}; + +// --------------------------------------------------------------------------- +// Mutator dispatch (port of `_prepare_mutator` + the diff assembly). +// --------------------------------------------------------------------------- + +/** The set of config-changing tools the preview endpoint can diff. */ +const SUPPORTED_TOOLS = new Set([ + 'update_config', + 'update_config_row', + 'update_sql_transformation', + UPDATE_FLOW_TOOL_NAME, + MODIFY_FLOW_TOOL_NAME, + 'modify_streamlit_data_app', +]); + +/** Tools whose original/updated config diff is implemented in the TypeScript port. */ +const DIFF_IMPLEMENTED_TOOLS = new Set(['update_config', 'update_config_row']); + +/** + * Computes (originalConfig, newConfiguration) for the config tools whose mutation + * internals are factored out and safe to run read-only. Returns the original config + * record and the rebuilt `configuration` object. + */ +const computeConfigDiff = async ( + config: Config, + clients: KeboolaClients, + rq: PreviewConfigDiffRq, +): Promise<{ original: JsonDict; newConfiguration: JsonDict }> => { + const params = rq.toolParams; + const componentId = String(params.component_id ?? ''); + const configurationId = String(params.configuration_id ?? ''); + const parameterUpdates = + (params.parameter_updates as ConfigParamUpdate[] | null | undefined) ?? null; + const storage = (params.storage as JsonDict | null | undefined) ?? null; + const processorsBefore = (params.processors_before as JsonDict[] | null | undefined) ?? null; + const processorsAfter = (params.processors_after as JsonDict[] | null | undefined) ?? null; + + if (rq.toolName === 'update_config') { + const original = await configPreviewInternals.configurationDetail( + clients, + componentId, + configurationId, + ); + const newConfiguration = await configPreviewInternals.buildUpdatedConfigPayload({ + config, + clients, + componentId, + configurationId, + parameterUpdates, + storage, + processorsBefore, + processorsAfter, + isRow: false, + }); + return { original, newConfiguration }; + } + + // update_config_row + const configurationRowId = String(params.configuration_row_id ?? ''); + const original = await configPreviewInternals.configurationRowDetail( + clients, + componentId, + configurationId, + configurationRowId, + ); + const newConfiguration = await configPreviewInternals.buildUpdatedConfigPayload({ + config, + clients, + componentId, + configurationId, + configurationRowId, + parameterUpdates, + storage, + processorsBefore, + processorsAfter, + isRow: true, + }); + return { original, newConfiguration }; +}; + +/** + * Assembles the updated config from the original + new `configuration` and the + * top-level field overrides (name/description/changeDescription/isDisabled), mirroring + * the diff assembly block in `preview_config_diff`. + */ +const assembleUpdatedConfig = ( + original: JsonDict, + newConfiguration: JsonDict, + toolParams: JsonDict, +): JsonDict => { + const updated = structuredClone(original); + updated.configuration = newConfiguration; + if (toolParams.name) updated.name = toolParams.name; + if (toolParams.description) updated.description = toolParams.description; + if (toolParams.is_disabled !== undefined && toolParams.is_disabled !== null) { + updated.isDisabled = toolParams.is_disabled; + } + if (toolParams.change_description) updated.changeDescription = toolParams.change_description; + return updated; +}; + +// --------------------------------------------------------------------------- +// Main handler. +// --------------------------------------------------------------------------- + +/** + * Verifies the Storage token to read project features + admin role. Throwing here would + * surface as a 500; on failure we degrade to no-feature / no-role defaults so gating is + * still applied (and a misconfigured token simply fails the project gate). + */ +const verifyToken = async (clients: KeboolaClients): Promise => { + try { + return (await clients.storage.tokens.verify()) as TokenInfo; + } catch { + return {}; + } +}; + +/** + * Runs the full preview flow for an already-parsed request and returns the response + * payload, throwing {@link PreviewHttpError} for the 4xx cases. The Hono route is a thin + * wrapper around this. + */ +export const runPreviewConfigDiff = async ( + config: Config, + rq: PreviewConfigDiffRq, +): Promise => { + const meta = lookupToolMetadata(config, rq.toolName); + const isReadOnly = meta?.isReadOnly ?? false; + + // 1) Header authorization (port of ToolAuthorizationMiddleware). This route runs + // outside the MCP middleware chain, so enforce the same headers explicitly. + const auth = parseAuthorizationConfig({ + allowedTools: config.allowedTools, + disallowedTools: config.disallowedTools, + readOnlyMode: config.readOnlyMode, + }); + if (hasAuthorizationFilters(auth) && !isToolNameAuthorized(rq.toolName, isReadOnly, auth)) { + logger.info(`[preview] Tool authorization denied (headers): ${rq.toolName}`); + throw new PreviewHttpError(`The tool "${rq.toolName}" is not authorized for this client.`, 403); + } + + // Log only non-sensitive metadata; toolParams can carry user-supplied secrets. + logger.info( + `[preview] toolName=${rq.toolName} paramKeys=${Object.keys(rq.toolParams).sort().join(',')}`, + ); + + const clients = createReadOnlyClients(config); + + // 2) Project-feature / token-role / branch gating (the AI-3438 hardening) via the + // exact same authorizeToolCall decision a real MCP tool call uses. + const tokenInfo = await verifyToken(clients); + const denial = authorizeToolCall({ + toolName: rq.toolName, + isReadOnly, + isSemantic: isSemanticToolName(rq.toolName), + tokenRole: getTokenRole(tokenInfo), + features: getProjectFeatures(tokenInfo), + isOauth: Boolean(config.bearerToken), + isMainBranch: config.branchId === undefined, + // Preview only handles config-mutation tools, never the docs-search tools. + docsIndexAvailable: true, + }); + if (denial) { + logger.info(`[preview] Tool authorization denied (project/role/branch): ${rq.toolName}`); + throw new PreviewHttpError(denial, 403); + } + + const coordinates = extractCoordinates(rq.toolName, rq.toolParams); + + // 3) Validate the params against the tool's input schema. A schema failure returns a + // 200 with isValid=false (empty configs) — the KAI backend relies on that shape. + if (meta?.inputSchema) { + const validationError = validateToolParams(rq.toolName, rq.toolParams, meta.inputSchema); + if (validationError) { + return serializeResponse({ + coordinates, + originalConfig: {}, + updatedConfig: {}, + isValid: false, + validationErrors: [validationError], + }); + } + } else { + logger.warn(`[preview] No input schema found for tool "${rq.toolName}"`); + } + + if (!SUPPORTED_TOOLS.has(rq.toolName)) { + throw new PreviewHttpError(`Invalid tool name: "${rq.toolName}"`, 400); + } + + // The mutation internals for update_sql_transformation / update_flow / modify_flow / + // modify_streamlit_data_app are not factored out as reusable, write-free functions in + // the TypeScript port (they live inline in the tool handlers and perform the write), + // so their read-only diff is not yet available. Authorization, validation and + // coordinate extraction above still run for them. + if (!DIFF_IMPLEMENTED_TOOLS.has(rq.toolName)) { + throw new PreviewHttpError( + `Configuration diff preview for tool "${rq.toolName}" is not supported by this server.`, + 400, + ); + } + + // 4) Run the (read-only) mutator and assemble the diff. A value/validation error + // returns a 200 with isValid=false (parity with the Python except block). + try { + const { original, newConfiguration } = await computeConfigDiff(config, clients, rq); + const updatedConfig = assembleUpdatedConfig(original, newConfiguration, rq.toolParams); + return serializeResponse({ + coordinates, + originalConfig: original, + updatedConfig, + isValid: true, + validationErrors: null, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.info(`[preview] ${message}`); + return serializeResponse({ + coordinates, + originalConfig: {}, + updatedConfig: {}, + isValid: false, + validationErrors: [message], + }); + } +}; diff --git a/src/prompts.ts b/src/prompts.ts new file mode 100644 index 000000000..8ce3a221f --- /dev/null +++ b/src/prompts.ts @@ -0,0 +1,320 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; + +// Ported from prompts/keboola_prompts.py + prompts/add_prompts.py. Only the +// "one-click" prompts (no required parameters) are registered, matching +// add_keboola_prompts(). Each returns a single user message with fixed content; +// the prompt name is the function name and the description is its docstring. + +type OneClickPrompt = { + name: string; + description: string; + content: string; +}; + +const PROMPTS: OneClickPrompt[] = [ + { + name: 'analyze_project_structure', + description: + 'Generate a comprehensive analysis prompt for a Keboola project’s structure. ' + + 'This prompt analyzes the project’s components, data flow, buckets, tables, ' + + 'and configurations to provide insights into capabilities and applications.', + content: `Based on the components that are being used and the data available from all +of the buckets in the project, give me a high-level understanding of what is going on inside +of this project and the types of use cases that are being performed. + +**Analysis Requirements:** +Highlight the key functionalities being implemented, emphasizing the project's +capability to address specific problems or tasks. Explore the range of use cases the +project is designed for, detailing examples of real-world scenarios it can handle. Be sure to also include +the names of real example buckets, tables & configurations that are within the project. + +**Structure your output in the following format:** + +## High-level Summary +• Bullet-point summary of the activities and use cases being performed + +## Data Sources & Integrations +• List all data sources and external integrations +• Include specific extractor components and their configurations +• Mention connection types and data refresh patterns + +## Data Processing & Transformation +• Detail transformation workflows and SQL logic +• Highlight data cleaning, enrichment, and aggregation processes +• Include specific transformation component names and examples + +## Data Storage & Management +• Describe bucket organization and table structures +• Include real bucket and table names from the project +• Explain data retention and archival strategies + +## Use Cases +• Identify specific business use cases and scenarios +• Provide real-world examples the project can handle +• Connect technical capabilities to business outcomes + +Please provide a comprehensive analysis with specific examples and names from the actual project data.`, + }, + { + name: 'project_health_check', + description: + 'Generate a comprehensive health check analysis for the entire Keboola project. ' + + 'This one-click prompt analyzes project health, identifies issues, and provides recommendations.', + content: `Perform a comprehensive health check of this Keboola project and identify +any issues, risks, or optimization opportunities. + +**Health Check Areas:** + +## 1. Component Health +• Analyze all components for errors, warnings, or performance issues +• Check component configurations for best practices +• Identify unused or redundant components +• Review component update status and versions + +## 2. Data Quality Assessment +• Examine tables for data completeness and consistency +• Identify tables with potential data quality issues +• Check for empty tables or tables with unusual patterns +• Analyze data freshness and update frequencies + +## 3. Performance Analysis +• Identify slow-running transformations or jobs +• Check for resource-intensive operations +• Analyze job execution patterns and bottlenecks +• Review storage usage and optimization opportunities + +## 4. Security & Access Review +• Review bucket and table permissions +• Check for potential security vulnerabilities +• Analyze token usage and access patterns +• Identify overprivileged configurations + +## 5. Cost Optimization +• Identify cost optimization opportunities +• Review storage usage and retention policies +• Analyze job execution efficiency +• Suggest resource optimization strategies + +## 6. Recommendations +• Prioritized list of issues to address +• Quick wins for immediate improvement +• Long-term optimization strategies +• Best practices implementation suggestions + +Please provide specific findings with component and table names and actionable recommendations.`, + }, + { + name: 'data_quality_assessment', + description: + 'Generate a comprehensive data quality assessment for all project data. ' + + 'One-click analysis of data quality across all buckets and tables.', + content: `Conduct a comprehensive data quality assessment across all data in this Keboola project. + +**Data Quality Analysis:** + +## 1. Completeness Analysis +• Identify tables with missing or null values +• Calculate completeness percentages for key columns +• Flag tables with significant data gaps +• Analyze data volume trends and anomalies + +## 2. Consistency Checks +• Check for data format inconsistencies +• Identify duplicate records across tables +• Analyze referential integrity between related tables +• Flag inconsistent naming conventions + +## 3. Accuracy Assessment +• Identify potential data accuracy issues +• Check for outliers and anomalous values +• Analyze data validation patterns +• Review data transformation logic for accuracy + +## 4. Timeliness Evaluation +• Assess data freshness across all tables +• Identify stale or outdated data +• Review data update frequencies +• Flag tables with irregular update patterns + +## 5. Data Profiling Summary +• Statistical overview of each table +• Data type distribution and usage +• Value distribution analysis +• Schema evolution and changes + +## 6. Quality Scores & Recommendations +• Overall quality score for each table +• Prioritized list of data quality issues +• Specific improvement recommendations +• Data governance suggestions + +Please analyze the actual project data and provide specific findings with table names, +metrics, and actionable recommendations.`, + }, + { + name: 'component_usage_summary', + description: + 'Generate a comprehensive summary of all components and their usage patterns. ' + + 'One-click overview of project components, configurations, and usage analytics.', + content: `Generate a comprehensive summary of all components in this Keboola +project, their configurations, and usage patterns. + +**Component Analysis:** + +## 1. Component Inventory +• Complete list of all components by type (extractors, transformations, writers) +• Component versions and update status +• Configuration count per component +• Active vs inactive component status + +## 2. Usage Analytics +• Job execution frequency per component +• Success/failure rates and reliability metrics +• Resource consumption patterns +• Peak usage times and scheduling analysis + +## 3. Configuration Analysis +• Number of configurations per component +• Configuration complexity and parameter usage +• Shared vs component-specific configurations +• Configuration change history and evolution + +## 4. Data Flow Mapping +• Input and output relationships between components +• Data dependencies and lineage +• Critical path analysis in data pipelines +• Component interdependency mapping + +## 5. Health & Status Overview +• Component error rates and common issues +• Performance metrics and execution times +• Maintenance and update requirements +• Deprecated or outdated component usage + +## 6. Optimization Opportunities +• Underutilized or redundant components +• Configuration consolidation opportunities +• Component upgrade recommendations +• Efficiency improvement suggestions + +Please provide specific details including component names, configuration IDs, and +actionable insights for project optimization.`, + }, + { + name: 'error_analysis_report', + description: + 'Generate an analysis of recent errors and failures across the project. ' + + 'One-click error analysis with troubleshooting recommendations.', + content: `Analyze recent errors and failures across this Keboola project and +provide troubleshooting recommendations. + +**Error Analysis:** + +## 1. Error Frequency & Patterns +• Most common error types across all components +• Error frequency trends over time +• Components with highest failure rates +• Recurring vs one-time error patterns + +## 2. Critical Errors +• High-priority errors affecting data pipelines +• Errors causing data quality issues +• Security-related errors or warnings +• Errors impacting business-critical processes + +## 3. Component-Specific Issues +• Transformation errors and SQL issues +• Extractor connection and authentication problems +• Writer destination errors and data delivery failures +• Orchestration and scheduling conflicts + +## 4. Root Cause Analysis +• Infrastructure vs configuration-related errors +• Data-related errors (missing files, schema changes) +• Permission and access-related issues +• External service dependency failures + +## 5. Impact Assessment +• Business impact of each error category +• Data pipeline disruption analysis +• SLA and delivery timeline impacts +• Downstream system effect analysis + +## 6. Resolution Recommendations +• Immediate fixes for critical errors +• Preventive measures for recurring issues +• Configuration improvements to reduce errors +• Monitoring and alerting enhancements + +Please analyze actual error logs and job histories to provide specific error +instances with component names and detailed troubleshooting guidance.`, + }, + { + name: 'create_project_documentation', + description: + 'Generate comprehensive project documentation automatically. ' + + 'One-click documentation creation for the entire Keboola project.', + content: `Generate comprehensive, professional documentation for this Keboola +project that can be used for onboarding, maintenance, and knowledge sharing. + +**Documentation Structure:** + +## 1. Project Overview +• Executive summary of project purpose and objectives +• Key stakeholders and business owners +• Project scope and data processing capabilities +• Success metrics and KPIs + +## 2. Architecture Documentation +• High-level system architecture diagram description +• Data flow and pipeline overview +• Component interaction and dependencies +• Technical infrastructure and requirements + +## 3. Data Dictionary +• Complete inventory of all buckets and tables with names +• Column definitions and business meanings +• Data types, constraints, and validation rules +• Data lineage and source system mappings + +## 4. Component Documentation +• Detailed description of each component and its purpose +• Configuration parameters and their meanings +• Input/output specifications +• Business logic and transformation rules + +## 5. Operational Procedures +• Data pipeline monitoring and maintenance procedures +• Error handling and troubleshooting guides +• Backup and disaster recovery processes +• Change management and deployment procedures + +## 6. User Guides +• End-user access and data consumption guides +• Report and dashboard usage instructions +• Data quality and validation procedures +• FAQ and common troubleshooting scenarios + +## 7. Technical Reference +• API endpoints and integration specifications +• Security and access control documentation +• Performance tuning and optimization guides +• Development and testing procedures + +Please create detailed, professional documentation using actual project data +including specific names, configurations, and real examples.`, + }, +]; + +/** Registers the Keboola one-click prompts (port of add_keboola_prompts). */ +export const registerPrompts = (server: McpServer): void => { + for (const prompt of PROMPTS) { + server.registerPrompt( + prompt.name, + { title: prompt.name, description: prompt.description }, + () => ({ + messages: [{ role: 'user', content: { type: 'text', text: prompt.content } }], + }), + ); + } +}; diff --git a/src/resource-path.ts b/src/resource-path.ts new file mode 100644 index 000000000..a7cdc69db --- /dev/null +++ b/src/resource-path.ts @@ -0,0 +1,21 @@ +import { existsSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** + * Locates the on-disk `resources/` directory across both run modes: + * - source (vitest / tsx): this file is `src/resource-path.ts`, so `./resources` + * resolves to `src/resources`. + * - bundled (`dist/index.js`): `./resources` resolves to `dist/resources`, where + * the tsup build copies the tree (and which npm publishes, since `files: [dist]`). + * + * Everything collapses into a single bundled module, so every caller shares this + * file's `import.meta.url`; resolving once here avoids per-module path drift. + */ +const here = dirname(fileURLToPath(import.meta.url)); +const candidates = [join(here, 'resources'), join(here, '..', 'resources')]; + +export const resourcesDir = candidates.find((c) => existsSync(c)) ?? candidates[0]!; + +/** Builds an absolute path to a file under the resources directory. */ +export const resourcePath = (...segments: string[]): string => join(resourcesDir, ...segments); diff --git a/src/keboola_mcp_server/resources/data_app/qsapi_query_data_code.py b/src/resources/data_app/qsapi_query_data_code.py similarity index 100% rename from src/keboola_mcp_server/resources/data_app/qsapi_query_data_code.py rename to src/resources/data_app/qsapi_query_data_code.py diff --git a/src/keboola_mcp_server/resources/data_app/sapi_query_data_code.py b/src/resources/data_app/sapi_query_data_code.py similarity index 100% rename from src/keboola_mcp_server/resources/data_app/sapi_query_data_code.py rename to src/resources/data_app/sapi_query_data_code.py diff --git a/src/keboola_mcp_server/resources/flow_examples/conditional_flow_examples.jsonl b/src/resources/flow/conditional_flow_examples.jsonl similarity index 100% rename from src/keboola_mcp_server/resources/flow_examples/conditional_flow_examples.jsonl rename to src/resources/flow/conditional_flow_examples.jsonl diff --git a/src/keboola_mcp_server/resources/flow-schema.json b/src/resources/flow/flow-schema.json similarity index 100% rename from src/keboola_mcp_server/resources/flow-schema.json rename to src/resources/flow/flow-schema.json diff --git a/src/keboola_mcp_server/resources/flow_examples/legacy_flow_examples.jsonl b/src/resources/flow/legacy_flow_examples.jsonl similarity index 100% rename from src/keboola_mcp_server/resources/flow_examples/legacy_flow_examples.jsonl rename to src/resources/flow/legacy_flow_examples.jsonl diff --git a/src/keboola_mcp_server/resources/prompts/project_system_prompt.md b/src/resources/prompts/project_system_prompt.md similarity index 100% rename from src/keboola_mcp_server/resources/prompts/project_system_prompt.md rename to src/resources/prompts/project_system_prompt.md diff --git a/src/serialize.ts b/src/serialize.ts new file mode 100644 index 000000000..b27d6345f --- /dev/null +++ b/src/serialize.ts @@ -0,0 +1,68 @@ +import { encode, type JsonValue } from '@toon-format/toon'; + +/** + * Tool outputs are encoded as TOON (Token-Oriented Object Notation) — the same + * token-efficient, schema-aware format the Python server used via `toon-format`. + * `toonSerializeCompact` is the default; it drops null fields while preserving + * TOON's list-of-objects column alignment. + */ + +const isPlainObject = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +/** + * Drops null/undefined fields while keeping TOON's list-of-objects alignment. + * Port of the Python `_filter_toon_nulls`: + * - single-item object lists drop keys whose value is null; + * - multi-item object lists keep every key that has a value in *any* item + * (first-seen order), leaving null where an item lacks it, so all rows align. + */ +export const filterToonNulls = (data: unknown): unknown => { + if (Array.isArray(data)) { + if (data.length === 0) return data; + + if (data.every(isPlainObject)) { + if (data.length === 1) return [filterToonNulls(data[0])]; + + const orderedKeys: string[] = []; + const seen = new Set(); + for (const item of data as Record[]) { + for (const [key, value] of Object.entries(item)) { + if (value !== null && value !== undefined && !seen.has(key)) { + seen.add(key); + orderedKeys.push(key); + } + } + } + + return (data as Record[]).map((item) => { + const cleaned: Record = {}; + for (const key of orderedKeys) { + const value = item[key]; + cleaned[key] = value === null || value === undefined ? null : filterToonNulls(value); + } + return cleaned; + }); + } + + return data.map((item) => (item === null || item === undefined ? null : filterToonNulls(item))); + } + + if (isPlainObject(data)) { + const cleaned: Record = {}; + for (const [key, value] of Object.entries(data)) { + if (value === null || value === undefined) continue; + cleaned[key] = filterToonNulls(value); + } + return cleaned; + } + + return data; +}; + +export type ToolSerializer = (data: unknown) => string; + +export const toonSerialize: ToolSerializer = (data) => encode(data as JsonValue); + +export const toonSerializeCompact: ToolSerializer = (data) => + encode(filterToonNulls(data) as JsonValue); diff --git a/src/server.ts b/src/server.ts new file mode 100644 index 000000000..50c43b052 --- /dev/null +++ b/src/server.ts @@ -0,0 +1,240 @@ +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; + +import { getDocsSearch } from '@/clients/docsSearch'; +import { createKeboolaClients } from '@/clients/keboola'; +import type { Config } from '@/config'; +import { + type AuthorizationConfig, + hasAuthorizationFilters, + isToolNameAuthorized, + parseAuthorizationConfig, +} from '@/mcp/authorization'; +import { + authorizeToolCall, + filterToolsList, + type GatedTool, + type GatingContext, + getProjectFeatures, + getTokenRole, + isSemanticToolName, + type TokenInfo, +} from '@/mcp/filtering'; +import { registerTool } from '@/mcp/tool'; +import { registerPrompts } from '@/prompts'; +import { registerComponentTools } from '@/tools/components'; +import { registerDataAppTools } from '@/tools/data_apps'; +import { registerDocTools } from '@/tools/doc'; +import { registerFlowTools } from '@/tools/flow'; +import { registerJobTools } from '@/tools/jobs'; +import { registerOAuthTools } from '@/tools/oauth'; +import { registerProjectTools } from '@/tools/project'; +import { registerSearchTools } from '@/tools/search'; +import { registerSemanticTools } from '@/tools/semantic'; +import { registerSqlTools } from '@/tools/sql'; +import { registerStorageTools } from '@/tools/storage'; + +// Reading package.json at build time would need JSON import assertions; keep a +// constant and bump alongside package.json until the build wiring lands. +export const SERVER_NAME = 'keboola'; +export const SERVER_VERSION = '2.0.0-alpha.1'; + +/** + * Builds the MCP server and registers all tools/prompts/resources. + * + * A single scaffold tool proves the registration → schema → TOON-serialize → + * transport path end to end. Real tool modules land in later phases (see + * feature_spec/mcp-typescript-rewrite/PLAN.md §4). + */ +/** Options for {@link createServer}. */ +export type CreateServerOptions = { + /** + * Skip the project/role/branch + header authorization gating wrappers. Used by the + * TOOLS.md generator so `tools/list` returns every registered tool regardless of the + * (dummy) token's features/role. Never set this on a serving instance. + */ + skipGating?: boolean; +}; + +export const createServer = (config: Config, options: CreateServerOptions = {}): McpServer => { + const server = new McpServer({ name: SERVER_NAME, version: SERVER_VERSION }); + + // ponytail: scaffold tool, replaced when the real tool modules are ported. + registerTool(server, { + name: 'get_server_info', + title: 'Get server info', + description: 'Returns basic information about the running Keboola MCP server.', + annotations: { readOnlyHint: true }, + handler: () => ({ + name: SERVER_NAME, + version: SERVER_VERSION, + branchId: config.branchId ?? null, + hasStorageToken: Boolean(config.storageToken), + }), + }); + + registerProjectTools(server, config); + registerJobTools(server, config); + registerStorageTools(server, config); + registerOAuthTools(server, config); + registerComponentTools(server, config); + registerDocTools(server); + registerSearchTools(server, config); + registerSqlTools(server, config); + registerFlowTools(server, config); + registerSemanticTools(server, config); + registerDataAppTools(server, config); + + registerPrompts(server); + + if (!options.skipGating) { + wrapToolGating(server, config); + } + + return server; +}; + +type RawHandler = (request: unknown, extra: unknown) => Promise | unknown; + +/** Read-only hint map keyed by tool name, derived from registered tool annotations. */ +type ReadOnlyMap = Map; + +/** + * Verifies the Storage token to obtain project features + admin role. Returns an + * empty record when verification is not possible (e.g. no credentials configured) so + * that gating degrades to its no-feature / no-role defaults instead of failing the + * whole `tools/list` or `tools/call` request. + */ +const verifyToken = async (config: Config): Promise => { + try { + const clients = createKeboolaClients(config); + return (await clients.storage.tokens.verify()) as TokenInfo; + } catch { + return {}; + } +}; + +/** + * Wraps the low-level `tools/list` and `tools/call` request handlers (registered by + * the McpServer when the first tool was added) with both gating layers: + * + * 1. Project/role/branch gating — port of `ToolsFilteringMiddleware`. + * 2. Header authorization — port of `ToolAuthorizationMiddleware`. + * + * The SDK exposes no middleware hook, so we replace the handlers in the low-level + * `server.server` request-handler map, delegating to the originals after filtering. + */ +const wrapToolGating = (server: McpServer, config: Config): void => { + // The low-level Server keeps its request handlers in a private `_requestHandlers` + // Map keyed by JSON-RPC method. The SDK exposes no middleware hook, so we reach in + // to wrap the tool handlers that McpServer registered when the tools were added. + const handlers = (server.server as unknown as { _requestHandlers: Map }) + ._requestHandlers; + const originalList = handlers.get('tools/list'); + const originalCall = handlers.get('tools/call'); + if (!originalList || !originalCall) { + throw new Error('Tool request handlers are not initialized; register tools before gating.'); + } + + const authConfig = (): AuthorizationConfig => + parseAuthorizationConfig({ + allowedTools: config.allowedTools, + disallowedTools: config.disallowedTools, + readOnlyMode: config.readOnlyMode, + }); + + const toolReadOnly = (tool: { annotations?: { readOnlyHint?: boolean } }): boolean => + tool.annotations?.readOnlyHint === true; + + handlers.set('tools/list', async (request, extra) => { + const result = (await originalList(request, extra)) as { + tools: { name: string; annotations?: { readOnlyHint?: boolean } }[]; + }; + + const tokenInfo = await verifyToken(config); + const ctx: GatingContext = { + tokenRole: getTokenRole(tokenInfo), + features: getProjectFeatures(tokenInfo), + isOauth: Boolean(config.bearerToken), + // Discovery always treats the branch as main/production (Python forces + // branch_id=None for list requests). + isMainBranch: true, + // ponytail: gate on the index being *configured* (provider built), not a live + // reachability probe — a per-request DB round-trip on every tools/list is not worth + // it. If the index is configured but down, the tool call surfaces a clear error. + // For strict "reachable" gating, await getDocsSearch()?.isReady() and cache it. + docsIndexAvailable: getDocsSearch() !== null, + }; + + const gated: GatedTool[] = result.tools.map((t) => ({ + name: t.name, + readOnly: toolReadOnly(t), + })); + const allowedByProject = new Set(filterToolsList(gated, ctx).map((t) => t.name)); + + const auth = authConfig(); + const filtered = result.tools.filter((t) => { + if (!allowedByProject.has(t.name)) return false; + if (hasAuthorizationFilters(auth) && !isToolNameAuthorized(t.name, toolReadOnly(t), auth)) { + return false; + } + return true; + }); + + return { ...result, tools: filtered }; + }); + + // Per-call gating needs each tool's read-only hint by name; derive it from the + // registered tools via the (unwrapped) list handler once, lazily. + let readOnlyMap: ReadOnlyMap | undefined; + const getReadOnlyMap = async (extra: unknown): Promise => { + if (readOnlyMap) return readOnlyMap; + const listed = (await originalList({ method: 'tools/list', params: {} }, extra)) as { + tools: { name: string; annotations?: { readOnlyHint?: boolean } }[]; + }; + readOnlyMap = new Map(listed.tools.map((t) => [t.name, toolReadOnly(t)])); + return readOnlyMap; + }; + + handlers.set('tools/call', async (request, extra) => { + const params = (request as { params?: { name?: string } }).params ?? {}; + const toolName = params.name ?? ''; + + const roMap = await getReadOnlyMap(extra); + // Unknown tools fall through to the original handler, which reports them as + // "not found" (parity: the gating layers only decide on known tools). + const isReadOnly = roMap.get(toolName) ?? false; + const isKnown = roMap.has(toolName); + + if (isKnown) { + const tokenInfo = await verifyToken(config); + + // Header authorization first (port of ToolAuthorizationMiddleware.on_call_tool). + const auth = authConfig(); + if (hasAuthorizationFilters(auth) && !isToolNameAuthorized(toolName, isReadOnly, auth)) { + throw new McpError( + ErrorCode.InvalidRequest, + `Access denied: The tool "${toolName}" is not authorized for this client. ` + + `Contact your administrator to request access.`, + ); + } + + // Project/role/branch gating (port of ToolsFilteringMiddleware.on_call_tool). + const denial = authorizeToolCall({ + toolName, + isReadOnly, + isSemantic: isSemanticToolName(toolName), + tokenRole: getTokenRole(tokenInfo), + features: getProjectFeatures(tokenInfo), + isOauth: Boolean(config.bearerToken), + isMainBranch: config.branchId === undefined, + docsIndexAvailable: getDocsSearch() !== null, + }); + if (denial) { + throw new McpError(ErrorCode.InvalidRequest, denial); + } + } + + return originalCall(request, extra); + }); +}; diff --git a/src/tools/components/index.ts b/src/tools/components/index.ts new file mode 100644 index 000000000..4ea90d6b4 --- /dev/null +++ b/src/tools/components/index.ts @@ -0,0 +1,15 @@ +/** + * Public entry point for the components tool module. + * + * Preserves the import path `@/tools/components` for downstream consumers: + * - `registerComponentTools` (src/server.ts) + * - `fetchComponent` (src/tools/flow.ts) + * - `configPreviewInternals` (src/preview.ts) + * + * The model/schema layer (zod schemas, SQL/transformation utils, param-update + * helpers) is re-exported here so the former `@/tools/components.model` symbols + * remain reachable through `@/tools/components`. + */ +export { configPreviewInternals, registerComponentTools } from './tools'; +export { fetchComponent } from './utils'; +export * from './model'; diff --git a/src/tools/components/model.ts b/src/tools/components/model.ts new file mode 100644 index 000000000..831526248 --- /dev/null +++ b/src/tools/components/model.ts @@ -0,0 +1,715 @@ +/** + * Model & helper layer for the component WRITE tools. + * + * Ported from: + * - tools/components/sql_utils.py (SQL split/join — note: no sqlglot reformat here) + * - tools/components/tf_update.py (transformation block/code structural ops) + * - tools/components/utils.py (param-update utils, transformation config builders, + * bucket-name cleaning, structure summary, check_suitable) + * - tools/components/model.py (Zod schemas for the discriminated-union update ops) + */ + +import { z } from 'zod'; + +import { + CONDITIONAL_FLOW_COMPONENT_ID, + DATA_APP_COMPONENT_ID, + ORCHESTRATOR_COMPONENT_ID, +} from '@/constants'; + +export const SNOWFLAKE_TRANSFORMATION_ID = 'keboola.snowflake-transformation'; +export const BIGQUERY_TRANSFORMATION_ID = 'keboola.google-bigquery-transformation'; +export const PYTHON_TRANSFORMATION_ID = 'keboola.python-transformation-v2'; +export const R_TRANSFORMATION_ID = 'keboola.r-transformation-v2'; +export const VARIABLES_COMPONENT_ID = 'keboola.variables'; + +/** Components for which update_config manages folder metadata. */ +export const FOLDER_SUPPORTING_COMPONENT_IDS = new Set([ + PYTHON_TRANSFORMATION_ID, + R_TRANSFORMATION_ID, +]); + +type JsonDict = Record; + +// ============================================================================ +// VariableDefinition + zod schemas for update operations. +// ============================================================================ + +export const variableDefinitionSchema = z.object({ + name: z.string().describe('Variable name.'), + type: z + .enum(['string', 'vault']) + .default('string') + .describe('Variable type: "string" or "vault".'), + default_value: z.string().nullish().describe('Optional default value bound at creation time.'), +}); +export type VariableDefinition = z.infer; + +// --- Config (non-transformation) parameter updates --- +const configParamSet = z.object({ + op: z.literal('set'), + path: z + .string() + .describe('JSONPath to the parameter key to set (e.g., "api_key", "database.host")'), + value: z.any().describe('New value to set'), +}); +const configParamReplace = z.object({ + op: z.literal('str_replace'), + path: z.string().describe('JSONPath to the parameter key to modify'), + search_for: z.string().describe('Substring to search for (non-empty)'), + replace_with: z.string().describe('Replacement string (can be empty for deletion)'), +}); +const configParamRemove = z.object({ + op: z.literal('remove'), + path: z.string().describe('JSONPath to the parameter key to remove'), +}); +const configParamListAppend = z.object({ + op: z.literal('list_append'), + path: z.string().describe('JSONPath to the list parameter'), + value: z.any().describe('Value to append to the list'), +}); +export const configParamUpdateSchema = z.discriminatedUnion('op', [ + configParamSet, + configParamReplace, + configParamRemove, + configParamListAppend, +]); +export type ConfigParamUpdate = z.infer; + +// --- Simplified transformation blocks --- +export const tfCodeSchema = z.object({ + name: z.string().describe('A descriptive name for the code block'), + script: z.string().describe('The SQL script of the code block'), +}); +export const tfBlockSchema = z.object({ + name: z.string().describe('A descriptive name for the code block'), + codes: z.array(tfCodeSchema).describe('SQL code sub-blocks'), +}); + +const tfPosition = z.enum(['start', 'end']); + +const tfAddBlock = z.object({ + op: z.literal('add_block'), + block: tfBlockSchema.describe('The block to add'), + position: tfPosition.default('end'), +}); +const tfRemoveBlock = z.object({ + op: z.literal('remove_block'), + block_id: z.string().describe('The ID of the block to remove'), +}); +const tfRenameBlock = z.object({ + op: z.literal('rename_block'), + block_id: z.string(), + block_name: z.string().describe('The new name of the block'), +}); +const tfAddCode = z.object({ + op: z.literal('add_code'), + block_id: z.string(), + code: tfCodeSchema.describe('The code to add'), + position: tfPosition.default('end'), +}); +const tfRemoveCode = z.object({ + op: z.literal('remove_code'), + block_id: z.string(), + code_id: z.string(), +}); +const tfRenameCode = z.object({ + op: z.literal('rename_code'), + block_id: z.string(), + code_id: z.string(), + code_name: z.string().describe('The new name of the code'), +}); +const tfSetCode = z.object({ + op: z.literal('set_code'), + block_id: z.string(), + code_id: z.string(), + script: z.string().describe('The SQL script of the code to set'), +}); +const tfAddScript = z.object({ + op: z.literal('add_script'), + block_id: z.string(), + code_id: z.string(), + script: z.string().describe('The SQL script to add'), + position: tfPosition.default('end'), +}); +const tfStrReplace = z + .object({ + op: z.literal('str_replace'), + block_id: z.string().nullish(), + code_id: z.string().nullish(), + search_for: z.string().describe('Substring to search for (non-empty)'), + replace_with: z.string().describe('Replacement string (can be empty for deletion)'), + }) + .refine((v) => !(v.block_id == null && v.code_id != null), { + message: 'code_id must be None if block_id is None', + }); + +export const tfParamUpdateSchema = z.discriminatedUnion('op', [ + tfAddBlock, + tfRemoveBlock, + tfRenameBlock, + tfAddCode, + tfRemoveCode, + tfRenameCode, + tfSetCode, + tfAddScript, + tfStrReplace, +]); +export type TfParamUpdate = z.infer; + +export const STRUCTURAL_TF_OPS = new Set(['add_block', 'add_code', 'remove_block', 'remove_code']); + +// ============================================================================ +// SQL utilities (port of sql_utils.py split/join). sqlglot reformatting is NOT +// ported (no TS equivalent); scripts are split on statement boundaries as-is. +// ============================================================================ + +const SQL_SPLIT_REGEX = new RegExp( + '\\s*(' + + '(?:' + + "'[^'\\\\]*(?:\\\\.[^'\\\\]*)*'|" + + '"[^"\\\\]*(?:\\\\.[^"\\\\]*)*"|' + + '\\$\\$(?:(?!\\$\\$)[\\s\\S])*\\$\\$|' + + '/\\*[^*]*\\*+(?:[^*/][^*]*\\*+)*/|' + + '#[^\\n\\r]*|' + + '--[^\\n\\r]*|' + + '//[^\\n\\r]*|' + + '/(?![*/])|' + + '-(?!-)|' + + '\\$(?!\\$)|' + + '[^"\';#/$-]+' + + ')+' + + '(?:;|$)' + + ')', + 'gm', +); + +/** Splits a SQL script into individual statements (trimmed, non-empty). */ +export const splitSqlStatements = (script: string): string[] => { + if (!script || !script.trim()) return []; + const matches = script.match(SQL_SPLIT_REGEX) ?? []; + return matches.map((s) => s.trim()).filter((s) => s.length > 0); +}; + +/** Joins SQL statements into a single script separated by double newlines. */ +export const joinSqlStatements = (statements: string[]): string => { + if (!statements || statements.length === 0) return ''; + const parts: string[] = []; + for (const stmt of statements) { + const trimmed = stmt.trim(); + if (!trimmed) continue; + parts.push(trimmed, '\n\n'); + } + return parts.join(''); +}; + +// ============================================================================ +// Simplified <-> raw transformation parameter conversion. +// ============================================================================ + +export type SimplifiedBlocks = { + blocks: { name: string; codes: { name: string; script: string }[] }[]; +}; +export type RawTfParameters = { + blocks: { name: string; codes: { name: string; script: string[] }[] }[]; +}; + +export const toRawParameters = (params: SimplifiedBlocks): RawTfParameters => ({ + blocks: params.blocks.map((block) => ({ + name: block.name, + codes: block.codes.map((code) => ({ + name: code.name, + script: splitSqlStatements(code.script), + })), + })), +}); + +export const toSimplifiedParameters = (raw: RawTfParameters): SimplifiedBlocks => ({ + blocks: (raw.blocks ?? []).map((block) => ({ + name: block.name, + codes: (block.codes ?? []).map((code) => ({ + name: code.name, + script: joinSqlStatements( + Array.isArray(code.script) ? code.script : [String(code.script ?? '')], + ), + })), + })), +}); + +// ============================================================================ +// Transformation configuration builder (port of create_transformation_configuration). +// ============================================================================ + +export const cleanBucketName = (bucketName: string): string => { + const maxBucketLength = 96; + let name = bucketName.trim(); + // ASCII-fold (český -> cesky): NFKD then strip diacritics, drop non-ascii. + name = name.normalize('NFKD').replace(/[̀-ͯ]/g, ''); + // eslint-disable-next-line no-control-regex + name = name.replace(/[^\x00-\x7F]/g, ''); + name = name.replace(/\s+/g, '-'); + name = name.replace(/[^a-zA-Z0-9_-]/g, ''); + name = name.replace(/^_+/, ''); + return name.slice(0, maxBucketLength); +}; + +/** Builds the raw transformation configuration payload (parameters + storage). */ +export const createTransformationConfiguration = ( + codes: { name: string; script: string }[], + transformationName: string, + outputTables: string[], +): JsonDict => { + const rawParameters = toRawParameters({ blocks: [{ name: 'Blocks', codes }] }); + + const storage: JsonDict = { + input: { tables: [] as JsonDict[] }, + output: { tables: [] as JsonDict[] }, + }; + + if (outputTables.length > 0) { + const bucketName = cleanBucketName(transformationName); + const destination = `out.c-${bucketName}`; + (storage.output as JsonDict).tables = outputTables.map((outTable) => ({ + source: outTable, + destination: `${destination}.${outTable}`, + })); + } + + return { parameters: rawParameters, storage }; +}; + +// ============================================================================ +// JSONPath-free param update utilities (port of utils.py). +// ============================================================================ + +/** Sets a value in a nested dict using a dot-separated path; creates intermediate dicts. */ +export const setNestedValue = (data: JsonDict, path: string, value: unknown): void => { + const keys = path.split('.'); + let current: JsonDict = data; + for (let i = 0; i < keys.length - 1; i++) { + const key = keys[i]!; + if (!(key in current)) current[key] = {}; + const next = current[key]; + if (typeof next !== 'object' || next === null || Array.isArray(next)) { + const pathSoFar = keys.slice(0, i + 1).join('.'); + throw new Error( + `Cannot set nested value at path "${path}": encountered non-dict value at "${pathSoFar}".`, + ); + } + current = next as JsonDict; + } + current[keys[keys.length - 1]!] = value; +}; + +/** + * Resolves a simple dot/bracket path to the parent container + final key. + * Supports `a.b`, `a.b[2]`, `array[1]`, and quoted segments like `"#secret"`. + * Returns null when an intermediate node is missing. + */ +type PathRef = { parent: JsonDict | unknown[]; key: string | number; exists: boolean }; + +const parsePathSegments = (path: string): (string | number)[] => { + const segments: (string | number)[] = []; + for (const rawSeg of path.split('.')) { + const seg = rawSeg; + // Strip surrounding quotes from a quoted field name. + if ((seg.startsWith('"') && seg.endsWith('"')) || (seg.startsWith("'") && seg.endsWith("'"))) { + segments.push(seg.slice(1, -1)); + continue; + } + // Split out bracket indices: name[0][1] + const bracketRe = /\[(\d+)\]/g; + const name = seg.replace(/\[\d+\]/g, ''); + if (name) segments.push(name); + let m: RegExpExecArray | null; + while ((m = bracketRe.exec(seg)) !== null) { + segments.push(Number(m[1])); + } + } + return segments; +}; + +const resolveRef = (root: JsonDict, path: string, create: boolean): PathRef | null => { + const segments = parsePathSegments(path); + if (segments.length === 0) return null; + let current: JsonDict | unknown[] = root; + for (let i = 0; i < segments.length - 1; i++) { + const seg = segments[i]!; + const next = Array.isArray(current) + ? current[seg as number] + : (current as JsonDict)[seg as string]; + if (next === undefined || next === null) { + if (!create) return null; + const child: JsonDict = {}; + if (Array.isArray(current)) (current as unknown[])[seg as number] = child; + else (current as JsonDict)[seg as string] = child; + current = child; + } else { + current = next as JsonDict | unknown[]; + } + } + const finalKey = segments[segments.length - 1]!; + const exists = Array.isArray(current) + ? (finalKey as number) < current.length + : finalKey in (current as JsonDict); + return { parent: current, key: finalKey, exists }; +}; + +const getRefValue = (ref: PathRef): unknown => + Array.isArray(ref.parent) + ? ref.parent[ref.key as number] + : (ref.parent as JsonDict)[ref.key as string]; + +const setRefValue = (ref: PathRef, value: unknown): void => { + if (Array.isArray(ref.parent)) ref.parent[ref.key as number] = value; + else (ref.parent as JsonDict)[ref.key as string] = value; +}; + +const deleteRef = (ref: PathRef): void => { + if (Array.isArray(ref.parent)) ref.parent.splice(ref.key as number, 1); + else delete (ref.parent as JsonDict)[ref.key as string]; +}; + +const applyParamUpdate = (params: JsonDict, update: ConfigParamUpdate): JsonDict => { + // `$` targets the whole parameters object. + if (update.path === '$') { + if (update.op === 'set') return update.value as JsonDict; + } + + if (update.op === 'set') { + setNestedValue(params, update.path, update.value); + return params; + } + + if (update.op === 'str_replace') { + if (!update.search_for) throw new Error('Search string is empty'); + if (update.search_for === update.replace_with) { + throw new Error(`Search string and replace string are the same: "${update.search_for}"`); + } + const ref = resolveRef(params, update.path, false); + if (!ref || !ref.exists) throw new Error(`Path "${update.path}" does not exist`); + const value = getRefValue(ref); + let replaceCnt = 0; + if (typeof value === 'string') { + const occ = value.split(update.search_for).length - 1; + if (occ) { + replaceCnt += occ; + setRefValue(ref, value.split(update.search_for).join(update.replace_with)); + } + } else if (Array.isArray(value)) { + if (!value.every((item) => typeof item === 'string')) { + throw new Error(`Path "${update.path}" is not a string or list of strings`); + } + const newValue = value.map((item) => { + const s = item as string; + const occ = s.split(update.search_for).length - 1; + replaceCnt += occ; + return occ ? s.split(update.search_for).join(update.replace_with) : s; + }); + setRefValue(ref, newValue); + } else { + throw new Error(`Path "${update.path}" is not a string or list of strings`); + } + if (replaceCnt === 0) { + throw new Error(`Search string "${update.search_for}" not found in path "${update.path}"`); + } + return params; + } + + if (update.op === 'remove') { + const ref = resolveRef(params, update.path, false); + if (!ref || !ref.exists) throw new Error(`Path "${update.path}" does not exist`); + deleteRef(ref); + return params; + } + + if (update.op === 'list_append') { + const ref = resolveRef(params, update.path, false); + if (!ref || !ref.exists) throw new Error(`Path "${update.path}" does not exist`); + const value = getRefValue(ref); + if (!Array.isArray(value)) throw new Error(`Path "${update.path}" is not a list`); + value.push(update.value); + return params; + } + + return params; +}; + +/** Applies a list of parameter updates to a deep copy of `params`. */ +export const updateParams = (params: JsonDict, updates: ConfigParamUpdate[]): JsonDict => { + let result = structuredClone(params); + for (const update of updates) { + result = applyParamUpdate(result, update); + } + return result; +}; + +// ============================================================================ +// Transformation structural updates (port of tf_update.py). +// ============================================================================ + +type TfBlock = { id?: string; name: string; codes: TfCode[] }; +type TfCode = { id?: string; name: string; script: string }; +type TfParams = { blocks: TfBlock[] }; + +/** Numbers blocks (b0, b1…) and codes (b0.c0…). */ +export const addIds = (params: TfParams): TfParams => { + params.blocks.forEach((block, bidx) => { + block.id = `b${bidx}`; + block.codes.forEach((code, cidx) => { + code.id = `b${bidx}.c${cidx}`; + }); + }); + return params; +}; + +const findBlock = (params: TfParams, blockId: string): TfBlock | undefined => + params.blocks.find((b) => b.id === blockId); + +const findCode = (block: TfBlock | undefined, codeId: string): TfCode | undefined => + block?.codes.find((c) => c.id === codeId); + +const applyTfUpdate = (params: TfParams, op: TfParamUpdate): [TfParams, string] => { + switch (op.op) { + case 'add_block': { + if (!op.block.name.trim()) throw new Error('Invalid operation: block name cannot be empty'); + const newBlock: TfBlock = { + name: op.block.name, + codes: op.block.codes.map((c) => ({ ...c })), + }; + if (op.position === 'start') params.blocks.unshift(newBlock); + else params.blocks.push(newBlock); + return [params, `Added block with name "${op.block.name}"`]; + } + case 'remove_block': { + const idx = params.blocks.findIndex((b) => b.id === op.block_id); + if (idx === -1) throw new Error(`Block with id '${op.block_id}' does not exist`); + params.blocks.splice(idx, 1); + return [params, '']; + } + case 'rename_block': { + if (!op.block_name.trim()) throw new Error('Invalid operation: block name cannot be empty'); + const block = findBlock(params, op.block_id); + if (!block) throw new Error(`Block with id '${op.block_id}' does not exist`); + block.name = op.block_name; + return [params, '']; + } + case 'add_code': { + if (!op.code.name.trim()) throw new Error('Invalid operation: code name cannot be empty'); + const block = findBlock(params, op.block_id); + if (!block) throw new Error(`Block with id '${op.block_id}' does not exist`); + const newCode: TfCode = { name: op.code.name, script: op.code.script }; + if (op.position === 'start') block.codes.unshift(newCode); + else block.codes.push(newCode); + return [params, `Added code with name "${op.code.name}"`]; + } + case 'remove_code': { + const block = findBlock(params, op.block_id); + const code = findCode(block, op.code_id); + if (!block || !code) { + throw new Error(`Code with id '${op.code_id}' in block '${op.block_id}' does not exist`); + } + block.codes.splice(block.codes.indexOf(code), 1); + return [params, '']; + } + case 'rename_code': { + if (!op.code_name.trim()) throw new Error('Invalid operation: code name cannot be empty'); + const block = findBlock(params, op.block_id); + const code = findCode(block, op.code_id); + if (!code) { + throw new Error(`Code with id '${op.code_id}' in block '${op.block_id}' does not exist`); + } + code.name = op.code_name; + return [params, '']; + } + case 'set_code': { + if (!op.script.trim()) throw new Error('Invalid operation: script cannot be empty'); + const block = findBlock(params, op.block_id); + const code = findCode(block, op.code_id); + if (!code) { + throw new Error(`Code with id '${op.code_id}' in block '${op.block_id}' does not exist`); + } + code.script = op.script; + return [params, `Changed code with id '${op.code_id}' in block '${op.block_id}'`]; + } + case 'add_script': { + if (!op.script.trim()) throw new Error('Invalid operation: script cannot be empty'); + const block = findBlock(params, op.block_id); + const code = findCode(block, op.code_id); + if (!code) { + throw new Error(`Code with id '${op.code_id}' in block '${op.block_id}' does not exist`); + } + const current = code.script; + code.script = + op.position === 'start' + ? current + ? `${op.script} ${current}` + : op.script + : current + ? `${current} ${op.script}` + : op.script; + return [params, `Added script to code with id '${op.code_id}' in block '${op.block_id}'`]; + } + case 'str_replace': { + if (!op.search_for) throw new Error('Invalid operation: search string is empty'); + if (op.search_for === op.replace_with) { + throw new Error( + `Invalid operation: search string and replace string are the same: "${op.search_for}"`, + ); + } + let targets: TfCode[]; + let scope: string; + if (op.block_id == null) { + targets = params.blocks.flatMap((b) => b.codes); + scope = 'the transformation'; + } else if (op.code_id == null) { + const block = findBlock(params, op.block_id); + targets = block ? block.codes : []; + scope = `block "${op.block_id}"`; + } else { + const block = findBlock(params, op.block_id); + const code = findCode(block, op.code_id); + targets = code ? [code] : []; + scope = `code "${op.code_id}", block "${op.block_id}"`; + } + if (targets.length === 0) throw new Error(`No scripts found in ${scope}`); + let replaceCnt = 0; + for (const code of targets) { + if (code.script.includes(op.search_for)) { + replaceCnt += code.script.split(op.search_for).length - 1; + code.script = code.script.split(op.search_for).join(op.replace_with); + } + } + if (replaceCnt === 0) { + throw new Error(`Search string "${op.search_for}" not found in ${scope}`); + } + const word = replaceCnt === 1 ? 'occurrence' : 'occurrences'; + return [params, `Replaced ${replaceCnt} ${word} of "${op.search_for}" in ${scope}`]; + } + default: + return [params, '']; + } +}; + +/** Markdown summary of a transformation's block/code structure. */ +export const structureSummary = (params: TfParams): string => { + const lines = ['## Updated Transformation Structure', '']; + const blocks = params.blocks ?? []; + if (blocks.length === 0) { + return '## Updated Transformation Structure\n\nNo blocks found in transformation.\n'; + } + for (const block of blocks) { + lines.push(`### Block id: \`${block.id}\`, name: \`${block.name ?? ''}\``, ''); + const codes = block.codes ?? []; + if (codes.length === 0) { + lines.push('*No code blocks*', ''); + continue; + } + for (const code of codes) { + lines.push(`- **Code id: \`${code.id}\`, name: \`${code.name ?? ''}\`** SQL snippet:`, ''); + const script = code.script ?? ''; + if (script) { + let snippet = script.trim(); + if (snippet.length > 150) { + const truncated = snippet.length - 150; + snippet = `${snippet.slice(0, 150)}... (${truncated} chars truncated)`; + } + lines.push(' ```sql', ` ${snippet}`, ' ```'); + } else { + lines.push(' *Empty script*'); + } + lines.push(''); + } + } + return lines.join('\n'); +}; + +/** + * Applies transformation parameter updates to a simplified-blocks structure. + * Returns the updated simplified blocks and a change summary. + */ +export const updateTransformationParameters = ( + parameters: SimplifiedBlocks, + updates: TfParamUpdate[], +): [SimplifiedBlocks, string] => { + const isStructureChange = updates.some((u) => STRUCTURAL_TF_OPS.has(u.op)); + let paramsDict = addIds(structuredClone(parameters) as TfParams); + const messages: string[] = []; + for (const update of updates) { + const [updated, message] = applyTfUpdate(paramsDict, update); + paramsDict = updated; + if (message) messages.push(message); + } + if (isStructureChange) { + paramsDict = addIds(paramsDict); + messages.push(structureSummary(paramsDict)); + } + // Strip ids back out for the simplified shape (extra='ignore'). + const simplified: SimplifiedBlocks = { + blocks: paramsDict.blocks.map((b) => ({ + name: b.name, + codes: b.codes.map((c) => ({ name: c.name, script: c.script })), + })), + }; + return [simplified, messages.join('\n')]; +}; + +// ============================================================================ +// check_suitable (port of utils.py). +// ============================================================================ + +const UNSUITABLE_COMPONENTS_MESSAGES: Record = { + [DATA_APP_COMPONENT_ID]: 'Use the data applications tools.', + [CONDITIONAL_FLOW_COMPONENT_ID]: 'Use the flows tools.', + [ORCHESTRATOR_COMPONENT_ID]: 'Use the flows tools.', + [BIGQUERY_TRANSFORMATION_ID]: 'Use the SQL transformation tools.', + [SNOWFLAKE_TRANSFORMATION_ID]: 'Use the SQL transformation tools.', +}; + +export const checkSuitable = (toolName: string, componentId: string): void => { + const message = UNSUITABLE_COMPONENTS_MESSAGES[componentId]; + if (message) { + throw new Error( + `The "${toolName}" tool cannot be used with ${componentId} component. ${message}`, + ); + } +}; + +export const getSqlTransformationIdFromSqlDialect = (sqlDialect: string): string => { + const d = sqlDialect.toLowerCase(); + if (d === 'snowflake') return SNOWFLAKE_TRANSFORMATION_ID; + if (d === 'bigquery') return BIGQUERY_TRANSFORMATION_ID; + throw new Error(`Unsupported SQL dialect: ${sqlDialect}`); +}; + +// ============================================================================ +// Folder hint helpers (port of utils.py build_folder_hint / folder_field_description). +// ============================================================================ + +export const folderFieldDescription = (singular: string, plural: string): string => + `Folder name to organize this ${singular} in the Keboola UI. ` + + `Pass an empty string to remove an existing folder assignment. ` + + `Existing folder names are returned in the response change_summary when no folder is provided ` + + `and there are 20 or more ${plural} in the project. ` + + `If there are 20 or more ${plural}, you should assign one of the existing folders or ` + + `create a new one that clearly reflects the ${singular} purpose.`; + +export const buildFolderHint = ( + total: number, + existingFolders: string[], + configLabel: string, + updateTool: string, + lowerBound = false, +): string | null => { + if (total < 20) return null; + const countStr = lowerBound ? `at least ${total}` : String(total); + let hint = `Note: This project already has ${countStr} ${configLabel}. Consider organizing them with folders. `; + if (existingFolders.length > 0) { + hint += + `Existing folders: ${existingFolders.join(', ')}. ` + + `Call ${updateTool} with a folder= parameter to assign this to one.`; + } else { + hint += `No folders have been created yet. Call ${updateTool} with a folder= parameter to start organizing.`; + } + return hint; +}; diff --git a/src/tools/components/tools.ts b/src/tools/components/tools.ts new file mode 100644 index 000000000..dd982d1f0 --- /dev/null +++ b/src/tools/components/tools.ts @@ -0,0 +1,1324 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { createKeboolaClients, createLinksManager, type KeboolaClients } from '@/clients/keboola'; +import { RawHttpError } from '@/clients/raw'; +import type { Config } from '@/config'; +import { ALL_COMPONENT_TYPES, type ComponentType } from '@/constants'; +import type { Link } from '@/links'; +import { logger } from '@/logger'; +import { registerTool } from '@/mcp/tool'; +import { + type JsonDict, + validateProcessorsConfiguration, + validateRootParametersConfiguration, + validateRootStorageConfiguration, + validateRowParametersConfiguration, + validateRowStorageConfiguration, +} from '@/tools/validation'; +import { + buildFolderHint, + checkSuitable, + type ConfigParamUpdate, + configParamUpdateSchema, + createTransformationConfiguration, + FOLDER_SUPPORTING_COMPONENT_IDS, + folderFieldDescription, + getSqlTransformationIdFromSqlDialect, + setNestedValue, + tfCodeSchema, + type TfParamUpdate, + tfParamUpdateSchema, + toRawParameters, + toSimplifiedParameters, + updateParams, + updateTransformationParameters, + variableDefinitionSchema, +} from './model'; +import { + applyConfigurationVariables, + applyVarsToParentCfg, + clearFolderMetadata, + configurationCreate, + configurationDetail, + configurationList, + configurationRowCreate, + configurationRowDetail, + configurationRowUpdate, + configurationUpdate, + deleteVariablesConfig, + fetchComponent, + getConfigFolders, + jsonBlock, + nowIso, + pick, + type RawComponent, + type RawConfig, + resolveSqlDialect, + setCfgCreationMetadata, + setCfgUpdateMetadata, + setFolderMetadata, + toComponent, + toComponentForValidation, + toComponentSummary, + toConfigSummary, + toConfiguration, +} from './utils'; + +// Ported from tools/components/tools.py. + +type ConfigToolOutput = { + component_id: string; + configuration_id: string; + description: string; + version: number; + timestamp: string; + success: boolean; + links: Link[]; + change_summary?: string | null; +}; + +export const registerComponentTools = (server: McpServer, config: Config): void => { + registerTool(server, { + name: 'get_config_examples', + title: 'Get config examples', + description: 'Retrieves sample configuration examples for a specific component.', + annotations: { readOnlyHint: true }, + inputSchema: { + component_id: z + .string() + .describe('The ID of the component to get configuration examples for.'), + }, + handler: async ({ component_id }) => { + const { rawAi } = createKeboolaClients(config); + + let detail: { rootConfigurationExamples?: unknown[]; rowConfigurationExamples?: unknown[] }; + try { + // KEPT RAW: the AI catalog `docs/components/{id}` endpoint has no typed equivalent. + detail = await rawAi.get(`docs/components/${component_id}`); + } catch { + // Mirrors the Python tool: unknown/erroring component -> empty string. + return ''; + } + + const rootExamples = detail.rootConfigurationExamples ?? []; + const rowExamples = detail.rowConfigurationExamples ?? []; + + let markdown = `# Configuration Examples for \`${component_id}\`\n\n`; + if (rootExamples.length > 0) { + markdown += '## Root Configuration Examples\n\n'; + rootExamples.forEach((example, i) => { + markdown += jsonBlock('Root Configuration', i + 1, example); + }); + } + if (rowExamples.length > 0) { + markdown += '## Row Configuration Examples\n\n'; + rowExamples.forEach((example, i) => { + markdown += jsonBlock('Row Configuration', i + 1, example); + }); + } + return markdown; + }, + }); + + registerTool(server, { + name: 'get_components', + title: 'Get components', + description: 'Retrieves detailed information about one or more components by their IDs.', + annotations: { readOnlyHint: true }, + inputSchema: { + component_ids: z.array(z.string()).describe('IDs of the components to retrieve.'), + }, + handler: async ({ component_ids }) => { + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + + const components = await Promise.all( + component_ids.map(async (componentId) => { + const component = toComponent(await fetchComponent(clients, componentId)); + component.links = [ + linksManager.getConfigDashboardLink(componentId, component.component_name), + ]; + return component; + }), + ); + + return { components, links: [linksManager.getUsedComponentsLink()] }; + }, + }); + + registerTool(server, { + name: 'run_sync_action', + title: 'Run sync action', + description: + 'Executes a synchronous action for a component configuration or a component row configuration.', + inputSchema: { + action_name: z + .string() + .describe('The sync action to execute (e.g., "testConnection", "getTables").'), + component_id: z.string().describe('The ID of the component (e.g., "keboola.ex-db-mysql").'), + configuration_id: z + .string() + .describe('The ID of the configuration to use for the sync action.'), + configuration_row_id: z + .string() + .nullish() + .describe( + 'Optional row ID; row parameters/storage are shallow-merged on top of root config.', + ), + }, + handler: async ({ action_name, component_id, configuration_id, configuration_row_id }) => { + const clients = createKeboolaClients(config); + + const configDetail = await configurationDetail(clients, component_id, configuration_id); + const root = (configDetail.configuration as Record) ?? {}; + let parameters = (root.parameters as Record) ?? {}; + let storage = (root.storage as Record) ?? {}; + // runtime/authorization live only on the root config (docker-runner contract). + const runtime = (root.runtime as Record) ?? {}; + const authorization = (root.authorization as Record) ?? {}; + + if (configuration_row_id) { + const rowDetail = await configurationRowDetail( + clients, + component_id, + configuration_id, + configuration_row_id, + ); + const rowConfig = (rowDetail.configuration as Record) ?? {}; + parameters = { + ...parameters, + ...((rowConfig.parameters as Record) ?? {}), + }; + storage = { ...storage, ...((rowConfig.storage as Record) ?? {}) }; + } + + const configData: Record = { parameters, storage }; + if (Object.keys(runtime).length > 0) configData.runtime = runtime; + if (Object.keys(authorization).length > 0) configData.authorization = authorization; + + // MIGRATED → syncActions.sendSyncAction (POST /actions). + const payload: { + configData: Record; + componentId: string; + action: string; + branchId?: string; + } = { + configData, + componentId: component_id, + action: action_name, + }; + if (config.branchId) payload.branchId = config.branchId; + + return clients.syncActions.sendSyncAction(payload); + }, + }); + + registerTool(server, { + name: 'get_configs', + title: 'Get configs', + description: 'Retrieves component configurations in the project with optional filtering.', + annotations: { readOnlyHint: true }, + inputSchema: { + component_types: z + .array(z.enum(ALL_COMPONENT_TYPES)) + .default([]) + .describe( + 'Filter by component types; empty = all. Ignored when configs/component_ids given.', + ), + component_ids: z + .array(z.string()) + .default([]) + .describe('Filter by specific component IDs. Ignored when configs is given.'), + configs: z + .array(z.object({ component_id: z.string(), configuration_id: z.string() })) + .default([]) + .describe('Specific configs to retrieve full details for (grouped by component).'), + }, + handler: async ({ component_types, component_ids, configs }) => { + const clients = createKeboolaClients(config); + const links = await createLinksManager(config, clients); + + // Case 1: full details for specific configs. + if (configs.length > 0) { + const fetched = await Promise.all( + configs.map(async ({ component_id, configuration_id }) => { + const raw = (await configurationDetail( + clients, + component_id, + configuration_id, + )) as RawConfig; + const component = toComponentSummary(await fetchComponent(clients, component_id)); + const cfgLinks = links.getConfigurationLinks( + component_id, + configuration_id, + String(raw.name ?? ''), + ); + return toConfiguration(raw, component_id, component, cfgLinks); + }), + ); + return { configs: fetched }; + } + + // Case 2/3: list summaries grouped by component. + const componentsWithConfigs: unknown[] = []; + + const buildGroup = async (rawComponent: RawComponent, rawConfigs: RawConfig[]) => { + const componentId = (pick(rawComponent, 'id', 'componentId') ?? '') as string; + const component = toComponentSummary(rawComponent); + component.links = [links.getConfigDashboardLink(componentId, component.component_name)]; + const configSummaries = rawConfigs.map((raw) => + toConfigSummary(raw, componentId, [ + links.getComponentConfigLink(componentId, String(raw.id ?? ''), String(raw.name ?? '')), + ]), + ); + componentsWithConfigs.push({ component, configs: configSummaries }); + }; + + if (component_ids.length > 0) { + for (const componentId of component_ids) { + const rawConfigs = (await configurationList(clients, componentId)) as RawConfig[]; + // KEPT RAW: the component detail merge in fetchComponent has no typed equivalent; + // here we only need the component summary, fetched via the raw Storage detail. + const rawComponent = await clients.rawStorage.get( + `branch/${clients.branchId}/components/${componentId}`, + ); + await buildGroup(rawComponent, rawConfigs); + } + } else { + const types: readonly ComponentType[] = + component_types.length > 0 ? component_types : ALL_COMPONENT_TYPES; + for (const componentType of types) { + // KEPT RAW: the listing uses `include=configuration` to embed configs in one + // call; the typed getComponents does not expose the embedded-configuration shape. + const rawComponents = await clients.rawStorage.get( + `branch/${clients.branchId}/components`, + { + params: { componentType, include: 'configuration' }, + }, + ); + for (const rawComponent of rawComponents) { + await buildGroup(rawComponent, (rawComponent.configurations as RawConfig[]) ?? []); + } + } + } + + return { + components_with_configs: componentsWithConfigs, + links: [links.getUsedComponentsLink(), links.getTransformationsDashboardLink()], + }; + }, + }); + + // ========================================================================== + // CONFIGURATION MANAGEMENT WRITE TOOLS + // ========================================================================== + + const processorsBeforeField = z + .array(z.record(z.string(), z.any())) + .nullish() + .describe('The list of processors that will run before the configured component runs.'); + const processorsAfterField = z + .array(z.record(z.string(), z.any())) + .nullish() + .describe('The list of processors that will run after the configured component runs.'); + + registerTool(server, { + name: 'create_config', + title: 'Create config', + description: + 'Creates a root component configuration using the specified name, component ID, configuration JSON, and ' + + 'description.', + annotations: { destructiveHint: false }, + inputSchema: { + name: z + .string() + .describe( + 'A short, descriptive name summarizing the purpose of the component configuration.', + ), + description: z + .string() + .describe( + 'The detailed description of the component configuration explaining its purpose and functionality.', + ), + component_id: z + .string() + .describe('The ID of the component for which to create the configuration.'), + parameters: z + .record(z.string(), z.any()) + .describe('The component configuration parameters, adhering to the configuration_schema'), + storage: z + .record(z.string(), z.any()) + .nullish() + .describe( + 'The table and/or file input / output mapping of the component configuration. ' + + 'It is present only for components that have tables or file input mapping defined', + ), + processors_before: processorsBeforeField, + processors_after: processorsAfterField, + variables: z + .array(variableDefinitionSchema) + .nullish() + .describe( + 'Variable definitions to attach to this configuration. ' + + 'Each entry specifies a name, type ("string" or "vault"), and an optional default value. ' + + 'On creation, both `None` (omitted) and `[]` (empty list) mean "do not attach variables" — ' + + 'no `keboola.variables` config is created. To remove variables from an existing configuration, ' + + 'use `update_config` with `variables=[]`.', + ), + }, + handler: async (args) => { + checkSuitable('create_config', args.component_id); + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + + const component = toComponentForValidation(await fetchComponent(clients, args.component_id)); + + const storageCfg = validateRootStorageConfiguration( + args.storage as JsonDict | null | undefined, + component, + 'The "storage" field is not valid.', + ); + const parameters = validateRootParametersConfiguration( + args.parameters as JsonDict, + component, + 'The "parameters" field is not valid.', + ); + + const configurationPayload: JsonDict = { storage: storageCfg, parameters }; + + const fetchForValidation = async (id: string) => + toComponentForValidation(await fetchComponent(clients, id)); + + if (args.processors_before?.length) { + const validated = await validateProcessorsConfiguration( + fetchForValidation, + args.processors_before as JsonDict[], + 'The "processors_before" field is not valid.', + ); + setNestedValue(configurationPayload, 'processors.before', validated); + } + if (args.processors_after?.length) { + const validated = await validateProcessorsConfiguration( + fetchForValidation, + args.processors_after as JsonDict[], + 'The "processors_after" field is not valid.', + ); + setNestedValue(configurationPayload, 'processors.after', validated); + } + + const newRaw = await configurationCreate( + config, + clients, + args.component_id, + args.name, + args.description, + configurationPayload, + ); + const configurationId = String(newRaw.id); + + await setCfgCreationMetadata(clients, args.component_id, configurationId); + + let varsResult: JsonDict | null = null; + if (args.variables && args.variables.length > 0) { + varsResult = await applyConfigurationVariables( + config, + clients, + args.component_id, + configurationId, + args.variables, + ); + if (varsResult !== null) { + await setCfgUpdateMetadata( + clients, + args.component_id, + configurationId, + varsResult.version as number, + ); + } + } + + const output: ConfigToolOutput = { + component_id: args.component_id, + configuration_id: configurationId, + description: args.description, + version: ((varsResult ?? newRaw).version as number) ?? 0, + timestamp: nowIso(), + success: true, + links: linksManager.getConfigurationLinks(args.component_id, configurationId, args.name), + }; + return output; + }, + }); + + registerTool(server, { + name: 'add_config_row', + title: 'Add config row', + description: + 'Creates a component configuration row in the specified configuration_id, using the specified name, ' + + 'component ID, configuration JSON, and description.', + annotations: { destructiveHint: false }, + inputSchema: { + name: z + .string() + .describe( + 'A short, descriptive name summarizing the purpose of the component configuration.', + ), + description: z + .string() + .describe( + 'The detailed description of the component configuration explaining its purpose and functionality.', + ), + component_id: z + .string() + .describe('The ID of the component for which to create the configuration.'), + configuration_id: z + .string() + .describe('The ID of the configuration for which to create the configuration row.'), + parameters: z + .record(z.string(), z.any()) + .describe( + 'The component row configuration parameters, adhering to the configuration_row_schema', + ), + storage: z + .record(z.string(), z.any()) + .nullish() + .describe( + 'The table and/or file input / output mapping of the component configuration. ' + + 'It is present only for components that have tables or file input mapping defined', + ), + processors_before: processorsBeforeField, + processors_after: processorsAfterField, + }, + handler: async (args) => { + checkSuitable('add_config_row', args.component_id); + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + + const component = toComponentForValidation(await fetchComponent(clients, args.component_id)); + + const storageCfg = validateRowStorageConfiguration( + args.storage as JsonDict | null | undefined, + component, + 'The "storage" field is not valid.', + args.configuration_id, + ); + const parameters = validateRowParametersConfiguration( + args.parameters as JsonDict, + component, + 'The "parameters" field is not valid.', + args.configuration_id, + ); + + const configurationPayload: JsonDict = { storage: storageCfg, parameters }; + + const fetchForValidation = async (id: string) => + toComponentForValidation(await fetchComponent(clients, id)); + + if (args.processors_before?.length) { + const validated = await validateProcessorsConfiguration( + fetchForValidation, + args.processors_before as JsonDict[], + 'The "processors_before" field is not valid.', + ); + setNestedValue(configurationPayload, 'processors.before', validated); + } + if (args.processors_after?.length) { + const validated = await validateProcessorsConfiguration( + fetchForValidation, + args.processors_after as JsonDict[], + 'The "processors_after" field is not valid.', + ); + setNestedValue(configurationPayload, 'processors.after', validated); + } + + const newRaw = await configurationRowCreate( + config, + clients, + args.component_id, + args.configuration_id, + args.name, + args.description, + configurationPayload, + ); + + await setCfgUpdateMetadata( + clients, + args.component_id, + args.configuration_id, + newRaw.version as number, + ); + + const output: ConfigToolOutput = { + component_id: args.component_id, + configuration_id: args.configuration_id, + description: args.description, + version: (newRaw.version as number) ?? 0, + timestamp: nowIso(), + success: true, + links: linksManager.getConfigurationLinks( + args.component_id, + args.configuration_id, + args.name, + ), + }; + return output; + }, + }); + + registerTool(server, { + name: 'update_config', + title: 'Update config', + description: + 'Updates an existing root component configuration by modifying its parameters, storage mappings, name or ' + + 'description. Updates are PARTIAL — only provide the fields you want to change; parameter_updates apply ' + + 'granular diff operations to the existing parameters.', + annotations: { destructiveHint: true }, + inputSchema: { + change_description: z + .string() + .describe( + 'A clear, human-readable summary of what changed in this update. ' + + 'Be specific: e.g., "Updated API key", "Added customers table to input mapping".', + ), + component_id: z.string().describe('The ID of the component the configuration belongs to.'), + configuration_id: z.string().describe('The ID of the configuration to update.'), + name: z + .string() + .default('') + .describe( + 'New name for the configuration. Only provide if changing the name. ' + + 'Name should be short (typically under 50 characters) and descriptive.', + ), + description: z + .string() + .default('') + .describe( + 'New detailed description for the configuration. Only provide if changing the description. ' + + 'Should explain the purpose, data sources, and behavior of this configuration. ' + + 'Leave empty to preserve the original description.', + ), + parameter_updates: z + .array(configParamUpdateSchema) + .nullish() + .describe( + 'List of granular parameter update operations to apply. ' + + 'Each operation (set, str_replace, remove, list_append) modifies a specific ' + + 'value using JSONPath notation. Only provide if updating parameters - ' + + 'do not use for changing description, storage or processors. ' + + 'Paths are relative to the `parameters` object, not the configuration root ' + + '(e.g. use `tables`, not `parameters.tables`). ' + + 'Prefer simple JSONPaths (e.g., "array_param[1]", "object_param.key") ' + + 'and make the smallest possible updates - only change what needs changing. ' + + 'In case you need to replace the whole parameters section, you can use the `set` operation ' + + 'with `$` as path.', + ), + storage: z + .record(z.string(), z.any()) + .nullish() + .describe( + 'Complete storage configuration containing input/output table and file mappings. ' + + 'Only provide if updating storage mappings - this replaces the ENTIRE storage configuration.', + ), + processors_before: processorsBeforeField, + processors_after: processorsAfterField, + folder: z + .string() + .nullish() + .describe(folderFieldDescription('configuration', 'configurations')), + variables: z + .array(variableDefinitionSchema) + .nullish() + .describe( + 'Variable definitions for this configuration. ' + + 'Provide a non-empty list to create or replace all variable definitions. ' + + 'Provide an empty list ([]) to remove all variables. ' + + 'Omit (None) to leave existing variables unchanged.', + ), + }, + handler: async (args) => { + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + + const configurationPayload = await buildUpdatedConfigPayload({ + config, + clients, + componentId: args.component_id, + configurationId: args.configuration_id, + parameterUpdates: args.parameter_updates ?? null, + storage: (args.storage as JsonDict | undefined) ?? null, + processorsBefore: (args.processors_before as JsonDict[] | undefined) ?? null, + processorsAfter: (args.processors_after as JsonDict[] | undefined) ?? null, + isRow: false, + }); + + let varsConfigIdToDelete: string | null = null; + if (args.variables !== undefined && args.variables !== null) { + const res = await applyVarsToParentCfg( + config, + clients, + args.component_id, + args.configuration_id, + args.variables, + configurationPayload, + ); + varsConfigIdToDelete = res.varsConfigIdToDelete; + } + + const updatedRaw = await configurationUpdate( + config, + clients, + args.component_id, + args.configuration_id, + configurationPayload, + args.change_description, + args.name, + args.description, + ); + + if (varsConfigIdToDelete) { + await deleteVariablesConfig(clients, varsConfigIdToDelete); + } + + let folderHint: string | null = null; + if (FOLDER_SUPPORTING_COMPONENT_IDS.has(args.component_id)) { + folderHint = await applyFolderMetadata( + clients, + args.component_id, + args.configuration_id, + (args.folder as string | null | undefined) ?? null, + 'configurations', + 'update_config', + ); + } + + await setCfgUpdateMetadata( + clients, + args.component_id, + args.configuration_id, + updatedRaw.version as number, + ); + + const output: ConfigToolOutput = { + component_id: args.component_id, + configuration_id: args.configuration_id, + description: (updatedRaw.description as string) || '', + version: (updatedRaw.version as number) ?? 0, + timestamp: nowIso(), + success: true, + links: linksManager.getConfigurationLinks( + args.component_id, + args.configuration_id, + (updatedRaw.name as string) || '', + ), + change_summary: folderHint, + }; + return output; + }, + }); + + registerTool(server, { + name: 'update_config_row', + title: 'Update config row', + description: + 'Updates an existing component configuration row by modifying its parameters, storage mappings, name, or ' + + 'description. Updates are PARTIAL — only provide the fields you want to change; parameter_updates apply ' + + 'granular diff operations to the existing row parameters.', + annotations: { destructiveHint: true }, + inputSchema: { + change_description: z + .string() + .describe( + 'A clear, human-readable summary of what changed in this row update. Be specific.', + ), + component_id: z.string().describe('The ID of the component the configuration belongs to.'), + configuration_id: z + .string() + .describe('The ID of the parent configuration containing the row to update.'), + configuration_row_id: z + .string() + .describe('The ID of the specific configuration row to update.'), + name: z + .string() + .default('') + .describe( + 'New name for the configuration row. Only provide if changing the name. ' + + 'Name should be short (typically under 50 characters) and descriptive of this specific row.', + ), + description: z + .string() + .default('') + .describe( + 'New detailed description for the configuration row. Only provide if changing the description. ' + + 'Should explain the specific purpose and behavior of this individual row.', + ), + parameter_updates: z + .array(configParamUpdateSchema) + .nullish() + .describe( + 'List of granular parameter update operations to apply to this row. ' + + 'Each operation (set, str_replace, remove, list_append) modifies a specific ' + + 'parameter using JSONPath notation. Only provide if updating parameters - ' + + 'do not use for changing description or storage. ' + + "Paths are relative to the row's `parameters` object, not the row root " + + '(e.g. use `tables`, not `parameters.tables`). ' + + 'Prefer simple dot-delimited JSONPaths ' + + 'and make the smallest possible updates - only change what needs changing. ' + + 'In case you need to replace the whole parameters, you can use the `set` operation ' + + 'with `$` as path.', + ), + storage: z + .record(z.string(), z.any()) + .nullish() + .describe( + 'Complete storage configuration for this row containing input/output table and file mappings. ' + + 'Only provide if updating storage mappings - this replaces the ENTIRE storage configuration ' + + 'for this row.', + ), + processors_before: processorsBeforeField, + processors_after: processorsAfterField, + is_disabled: z + .boolean() + .nullish() + .describe( + "Enable or disable the configuration row. Set to True to disable execution (config row won't run), " + + 'False to enable execution (config row will run). Only provide if changing the status, ' + + 'leave as null to preserve current state.', + ), + }, + handler: async (args) => { + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + + const configurationPayload = await buildUpdatedConfigPayload({ + config, + clients, + componentId: args.component_id, + configurationId: args.configuration_id, + configurationRowId: args.configuration_row_id, + parameterUpdates: args.parameter_updates ?? null, + storage: (args.storage as JsonDict | undefined) ?? null, + processorsBefore: (args.processors_before as JsonDict[] | undefined) ?? null, + processorsAfter: (args.processors_after as JsonDict[] | undefined) ?? null, + isRow: true, + }); + + const updatedRaw = await configurationRowUpdate( + config, + clients, + args.component_id, + args.configuration_id, + args.configuration_row_id, + configurationPayload, + args.change_description, + args.name, + args.description, + args.is_disabled, + ); + + await setCfgUpdateMetadata( + clients, + args.component_id, + args.configuration_id, + updatedRaw.version as number, + ); + + const output: ConfigToolOutput = { + component_id: args.component_id, + configuration_id: args.configuration_id, + description: (updatedRaw.description as string) || '', + version: (updatedRaw.version as number) ?? 0, + timestamp: nowIso(), + success: true, + links: linksManager.getConfigurationLinks( + args.component_id, + args.configuration_id, + (updatedRaw.name as string) || '', + ), + }; + return output; + }, + }); + + // ========================================================================== + // SQL TRANSFORMATION WRITE TOOLS + // ========================================================================== + + registerTool(server, { + name: 'create_sql_transformation', + title: 'Create SQL transformation', + description: + 'Creates an SQL transformation using the specified name, SQL query following the current SQL dialect, a ' + + 'detailed description, and a list of created table names.', + annotations: { destructiveHint: false }, + inputSchema: { + name: z + .string() + .describe('A short, descriptive name summarizing the purpose of the SQL transformation.'), + description: z + .string() + .describe( + 'The detailed description of the SQL transformation capturing the user intent, explaining the ' + + 'SQL query, and the expected output.', + ), + sql_code_blocks: z + .array(tfCodeSchema) + .describe( + 'The SQL query code blocks, each containing a descriptive name and an executable SQL script ' + + 'written in the current SQL dialect. The query will be automatically reformatted to be more readable.', + ), + created_table_names: z + .array(z.string()) + .default([]) + .describe( + 'A list of created table names if they are generated within the SQL query statements ' + + '(e.g., using `CREATE TABLE ...`).', + ), + folder: z + .string() + .default('') + .describe(folderFieldDescription('transformation', 'transformations')), + variables: z + .array(variableDefinitionSchema) + .nullish() + .describe( + 'Variable definitions to attach to this transformation. ' + + 'Each entry specifies a name, type ("string" or "vault"), and an optional default value. ' + + 'On creation, both `None` (omitted) and `[]` (empty list) mean "do not attach variables" — ' + + 'no `keboola.variables` config is created. To remove variables from an existing transformation, ' + + 'use `update_sql_transformation` with `variables=[]`.', + ), + }, + handler: async (args) => { + const clients = createKeboolaClients(config); + const sqlDialect = await resolveSqlDialect(clients); + const componentId = getSqlTransformationIdFromSqlDialect(sqlDialect); + + const payload = createTransformationConfiguration( + args.sql_code_blocks, + args.name, + args.created_table_names, + ); + + const linksManager = await createLinksManager(config, clients); + + const newRaw = await configurationCreate( + config, + clients, + componentId, + args.name, + args.description, + payload, + ); + const configurationId = String(newRaw.id); + + await setCfgCreationMetadata(clients, componentId, configurationId); + + const folder = args.folder.trim(); + let changeSummary: string | null = null; + if (folder) { + try { + await setFolderMetadata(clients, componentId, configurationId, folder); + } catch { + logger.warn(`Unable to set folder metadata for "${componentId}"/"${configurationId}".`); + } + } else { + try { + const [total, existingFolders, lowerBound] = await getConfigFolders(clients, componentId); + changeSummary = buildFolderHint( + total, + existingFolders, + 'SQL transformations', + 'update_sql_transformation', + lowerBound, + ); + } catch { + logger.warn(`Unable to fetch transformation folders for "${componentId}".`); + } + } + + let varsResult: JsonDict | null = null; + if (args.variables && args.variables.length > 0) { + varsResult = await applyConfigurationVariables( + config, + clients, + componentId, + configurationId, + args.variables, + ); + if (varsResult !== null) { + await setCfgUpdateMetadata( + clients, + componentId, + configurationId, + varsResult.version as number, + ); + } + } + + const output: ConfigToolOutput = { + component_id: componentId, + configuration_id: configurationId, + description: args.description, + version: ((varsResult ?? newRaw).version as number) ?? 0, + timestamp: nowIso(), + success: true, + links: linksManager.getTransformationLinks(componentId, configurationId, args.name), + change_summary: changeSummary, + }; + return output; + }, + }); + + registerTool(server, { + name: 'update_sql_transformation', + title: 'Update SQL transformation', + description: + 'Updates an existing SQL transformation configuration by modifying its SQL code, storage mappings, name or ' + + 'description. parameter_updates apply PARTIAL, granular diff operations to the transformation blocks/codes; ' + + 'storage is a complete replacement.', + annotations: { destructiveHint: true }, + inputSchema: { + change_description: z + .string() + .describe( + 'A clear, human-readable summary of what changed in this transformation update. ' + + 'Be specific: e.g., "Added JOIN with customers table", "Updated WHERE clause to filter active records".', + ), + configuration_id: z + .string() + .describe('The ID of the transformation configuration to update.'), + name: z + .string() + .default('') + .describe( + 'New name for the transformation. Only provide if changing the name. ' + + 'Name should be short (typically under 50 characters) and descriptive.', + ), + description: z + .string() + .default('') + .describe( + 'New detailed description for the transformation. Only provide if changing the description. ' + + 'Should explain what the transformation does, data sources, and business logic. ' + + 'Leave empty to preserve the original description.', + ), + parameter_updates: z + .array(tfParamUpdateSchema) + .nullish() + .describe( + 'List of operations to apply to the transformation structure (blocks, codes, SQL scripts). ' + + 'Each operation modifies specific elements using block_id and code_id identifiers. ' + + 'Only provide if updating SQL code or block structure - do not use for description or storage changes. ' + + 'Use get_configs first to retrieve the current transformation structure and identify the block_id and ' + + 'code_id values needed for your operations. IDs are automatically assigned. Available operations: ' + + 'add_block, remove_block, rename_block, add_code, remove_code, rename_code, set_code, add_script, ' + + 'str_replace.', + ), + storage: z + .record(z.string(), z.any()) + .nullish() + .describe( + 'Complete storage configuration for transformation input/output table mappings. ' + + 'Only provide if updating storage mappings - this replaces the ENTIRE storage configuration.', + ), + folder: z + .string() + .nullish() + .describe(folderFieldDescription('transformation', 'transformations')), + variables: z + .array(variableDefinitionSchema) + .nullish() + .describe( + 'Variable definitions for this transformation. ' + + 'Provide a non-empty list to create or replace all variable definitions. ' + + 'Provide an empty list ([]) to remove all variables. ' + + 'Omit (None) to leave existing variables unchanged.', + ), + }, + handler: async (args) => { + const clients = createKeboolaClients(config); + const sqlDialect = await resolveSqlDialect(clients); + const sqlTransformationId = getSqlTransformationIdFromSqlDialect(sqlDialect); + const linksManager = await createLinksManager(config, clients); + + let configDetails: JsonDict; + try { + configDetails = await configurationDetail( + clients, + sqlTransformationId, + args.configuration_id, + ); + } catch (error) { + if (error instanceof RawHttpError && error.status === 404) { + throw new Error( + `Configuration '${args.configuration_id}' was not found under SQL transformation component ` + + `'${sqlTransformationId}'. If this is a Python or R transformation, use 'update_config' ` + + `with component_id 'keboola.python-transformation-v2' or 'keboola.r-transformation-v2' ` + + `instead of 'update_sql_transformation'.`, + ); + } + throw error; + } + + const transformation = toComponentForValidation( + await fetchComponent(clients, sqlTransformationId), + ); + + const updatedConfiguration = structuredClone( + (configDetails.configuration as JsonDict) ?? {}, + ) as JsonDict; + + let msg = ''; + if (args.parameter_updates && args.parameter_updates.length > 0) { + const currentRaw = (updatedConfiguration.parameters as + | { + blocks?: { name: string; codes: { name: string; script: string[] }[] }[]; + } + | undefined) ?? { blocks: [] }; + const simplified = toSimplifiedParameters({ blocks: currentRaw.blocks ?? [] }); + const [updatedParams, message] = updateTransformationParameters( + simplified, + args.parameter_updates as TfParamUpdate[], + ); + msg = message; + const updatedRawParams = toRawParameters(updatedParams); + const parametersCfg = validateRootParametersConfiguration( + updatedRawParams as unknown as JsonDict, + transformation, + 'Applying the "parameter_updates" resulted in an invalid configuration.', + args.configuration_id, + ); + updatedConfiguration.parameters = parametersCfg; + } + + if (args.storage !== undefined && args.storage !== null) { + updatedConfiguration.storage = validateRootStorageConfiguration( + args.storage as JsonDict, + transformation, + 'The "storage" field is not valid.', + args.configuration_id, + ); + } + + let varsConfigIdToDelete: string | null = null; + if (args.variables !== undefined && args.variables !== null) { + const res = await applyVarsToParentCfg( + config, + clients, + sqlTransformationId, + args.configuration_id, + args.variables, + updatedConfiguration, + ); + varsConfigIdToDelete = res.varsConfigIdToDelete; + } + + const updatedRaw = await configurationUpdate( + config, + clients, + sqlTransformationId, + args.configuration_id, + updatedConfiguration, + args.change_description, + args.name, + args.description, + ); + + if (varsConfigIdToDelete) { + await deleteVariablesConfig(clients, varsConfigIdToDelete); + } + + let folderHint: string | null = null; + if (args.folder === undefined || args.folder === null) { + try { + const [total, existingFolders, lowerBound] = await getConfigFolders( + clients, + sqlTransformationId, + ); + folderHint = buildFolderHint( + total, + existingFolders, + 'SQL transformations', + 'update_sql_transformation', + lowerBound, + ); + } catch { + logger.warn(`Unable to fetch transformation folders for "${sqlTransformationId}".`); + } + } else { + const folderStripped = args.folder.trim(); + if (folderStripped) { + try { + await setFolderMetadata( + clients, + sqlTransformationId, + args.configuration_id, + folderStripped, + ); + } catch { + logger.warn(`Unable to set folder metadata for "${sqlTransformationId}".`); + } + } else { + await clearFolderMetadata(clients, sqlTransformationId, args.configuration_id); + } + } + + await setCfgUpdateMetadata( + clients, + sqlTransformationId, + args.configuration_id, + updatedRaw.version as number, + ); + + const changeSummary = [msg, folderHint].filter(Boolean).join(' ') || null; + + const output: ConfigToolOutput = { + component_id: sqlTransformationId, + configuration_id: args.configuration_id, + description: (updatedRaw.description as string) || '', + version: (updatedRaw.version as number) ?? 0, + timestamp: nowIso(), + success: true, + links: linksManager.getTransformationLinks( + sqlTransformationId, + args.configuration_id, + (updatedRaw.name as string) || '', + ), + change_summary: changeSummary, + }; + return output; + }, + }); +}; + +// ============================================================================ +// Shared update-payload builder for update_config / update_config_row. +// ============================================================================ + +const buildUpdatedConfigPayload = async (opts: { + config: Config; + clients: KeboolaClients; + componentId: string; + configurationId: string; + configurationRowId?: string; + parameterUpdates: ConfigParamUpdate[] | null; + storage: JsonDict | null; + processorsBefore: JsonDict[] | null; + processorsAfter: JsonDict[] | null; + isRow: boolean; +}): Promise => { + const { clients, componentId, configurationId, isRow } = opts; + checkSuitable(isRow ? 'update_config_row' : 'update_config', componentId); + + const current = isRow + ? await configurationRowDetail(clients, componentId, configurationId, opts.configurationRowId!) + : await configurationDetail(clients, componentId, configurationId); + const component = toComponentForValidation(await fetchComponent(clients, componentId)); + + const payload = structuredClone((current.configuration as JsonDict) ?? {}) as JsonDict; + + if (opts.storage !== null) { + payload.storage = isRow + ? validateRowStorageConfiguration( + opts.storage, + component, + 'The "storage" field is not valid.', + configurationId, + opts.configurationRowId, + ) + : validateRootStorageConfiguration( + opts.storage, + component, + 'The "storage" field is not valid.', + configurationId, + ); + } + + const fetchForValidation = async (id: string) => + toComponentForValidation(await fetchComponent(clients, id)); + + if (opts.processorsBefore !== null) { + const validated = await validateProcessorsConfiguration( + fetchForValidation, + opts.processorsBefore, + 'The "processors_before" field is not valid.', + ); + setNestedValue(payload, 'processors.before', validated); + } + if (opts.processorsAfter !== null) { + const validated = await validateProcessorsConfiguration( + fetchForValidation, + opts.processorsAfter, + 'The "processors_after" field is not valid.', + ); + setNestedValue(payload, 'processors.after', validated); + } + + if (opts.parameterUpdates && opts.parameterUpdates.length > 0) { + const currentParams = (payload.parameters as JsonDict) ?? {}; + const updated = updateParams(currentParams, opts.parameterUpdates); + const initial = isRow + ? 'Applying the "parameter_updates" resulted in an invalid row configuration.' + : 'Applying the "parameter_updates" resulted in an invalid configuration.'; + payload.parameters = isRow + ? validateRowParametersConfiguration( + updated, + component, + initial, + configurationId, + opts.configurationRowId, + ) + : validateRootParametersConfiguration(updated, component, initial, configurationId); + } + + return payload; +}; + +/** + * Additive re-export of the config-mutation internals the `/preview/configuration` + * endpoint reuses to build a config diff WITHOUT writing (port of the Python + * `_prepare_mutator` reuse of `update_config_internal` / `update_config_row_internal`). + * + * These are the same functions the `update_config` / `update_config_row` handlers call: + * `buildUpdatedConfigPayload` only issues GET requests (config + component fetch + + * pure validation), so it is safe to run against a read-only client. Exposed here so + * `preview.ts` can compute the original/updated configuration pair without duplicating + * the validation logic; the tool handlers and their behavior are unchanged. + */ +export const configPreviewInternals = { + buildUpdatedConfigPayload, + configurationDetail, + configurationRowDetail, +}; + +/** Sets/clears folder metadata or returns a hint (port of apply_folder_metadata). */ +const applyFolderMetadata = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, + folder: string | null, + kind: string, + toolName: string, +): Promise => { + if (folder === null) { + try { + const [total, existingFolders, lowerBound] = await getConfigFolders(clients, componentId); + return buildFolderHint(total, existingFolders, kind, toolName, lowerBound); + } catch { + logger.warn(`Unable to fetch ${kind} folders for component "${componentId}".`); + return null; + } + } + const normalized = folder.trim(); + if (normalized) { + try { + await setFolderMetadata(clients, componentId, configurationId, normalized); + } catch { + logger.warn(`Unable to set folder metadata for "${componentId}"/"${configurationId}".`); + } + } else { + await clearFolderMetadata(clients, componentId, configurationId); + } + return null; +}; diff --git a/src/tools/components/utils.ts b/src/tools/components/utils.ts new file mode 100644 index 000000000..25adab021 --- /dev/null +++ b/src/tools/components/utils.ts @@ -0,0 +1,733 @@ +import { redactSecrets } from '@/clients/encryption'; +import { type KeboolaClients } from '@/clients/keboola'; +import { createRawClient, type RawClient, RawHttpError } from '@/clients/raw'; +import { deriveServiceUrls } from '@/clients/urls'; +import type { Config } from '@/config'; +import { MetadataField } from '@/constants'; +import type { Link } from '@/links'; +import { logger } from '@/logger'; +import { type ComponentForValidation, type JsonDict } from '@/tools/validation'; +import { type VariableDefinition, VARIABLES_COMPONENT_ID } from './model'; + +// Ported from tools/components/{utils,api_models}.py. + +export type RawConfig = Record; +export type RawComponent = Record; +export type MetadataItem = { key?: string; value?: string }; + +const metadataProperty = (metadata: MetadataItem[] | undefined, key: string): string | undefined => + (metadata ?? []).find((item) => item.key === key)?.value; + +export const pick = (raw: RawComponent, ...keys: string[]): T | undefined => { + for (const key of keys) { + if (raw[key] !== undefined && raw[key] !== null) return raw[key] as T; + } + return undefined; +}; + +/** Configuration root/row summary (list mode) — port of ConfigSummary.from_api_response. */ +export const toConfigSummary = (raw: RawConfig, componentId: string, links: Link[]) => { + const metadata = (raw.metadata as MetadataItem[]) ?? []; + const rows = (raw.rows as RawConfig[]) ?? null; + return { + configuration_root: { + component_id: componentId, + configuration_id: String(raw.id ?? ''), + name: raw.name ?? '', + description: raw.description ?? null, + is_disabled: raw.isDisabled ?? false, + is_deleted: raw.isDeleted ?? false, + folder: metadataProperty(metadata, MetadataField.CONFIGURATION_FOLDER_NAME) ?? '', + }, + configuration_rows: rows + ? rows.map((row) => ({ + component_id: componentId, + configuration_id: String(raw.id ?? ''), + row_configuration_id: String(row.id ?? ''), + name: row.name ?? '', + description: row.description ?? null, + is_disabled: row.isDisabled ?? false, + is_deleted: row.isDeleted ?? false, + })) + : null, + links, + }; +}; + +/** + * Full configuration root/rows (detail mode) — port of Configuration.from_api_response. + * NOTE: transformation parameter *simplification* (Snowflake/BigQuery) is deferred until + * create_sql_transformation lands (needs the inverse); transformations return raw params. + */ +export const toConfiguration = ( + raw: RawConfig, + componentId: string, + component: unknown, + links: Link[], +) => { + const metadata = (raw.metadata as MetadataItem[]) ?? []; + const cfg = (raw.configuration as Record) ?? {}; + const rows = (raw.rows as RawConfig[]) ?? null; + return { + configuration_root: { + component_id: componentId, + configuration_id: String(raw.id ?? ''), + name: raw.name ?? '', + description: raw.description ?? null, + version: raw.version ?? 0, + is_disabled: raw.isDisabled ?? false, + is_deleted: raw.isDeleted ?? false, + folder: metadataProperty(metadata, MetadataField.CONFIGURATION_FOLDER_NAME) ?? '', + parameters: redactSecrets(cfg.parameters ?? {}), + storage: cfg.storage ?? null, + processors: redactSecrets(cfg.processors ?? null), + variables_id: cfg.variables_id ?? null, + variables_values_id: cfg.variables_values_id ?? null, + variables: cfg.variables ?? null, + configuration_metadata: metadata, + }, + configuration_rows: rows + ? rows.map((row) => { + const rowCfg = (row.configuration as Record) ?? {}; + return { + component_id: componentId, + configuration_id: String(raw.id ?? ''), + configuration_row_id: String(row.id ?? ''), + name: row.name ?? '', + description: row.description ?? null, + version: row.version ?? 0, + is_disabled: row.isDisabled ?? false, + is_deleted: row.isDeleted ?? false, + parameters: redactSecrets(rowCfg.parameters ?? {}), + storage: rowCfg.storage ?? null, + processors: redactSecrets(rowCfg.processors ?? null), + values: rowCfg.values ?? null, + configuration_metadata: (rowCfg.metadata as unknown) ?? [], + }; + }) + : null, + component, + links, + }; +}; + +export const toComponentSummary = (raw: RawComponent) => { + const flags = (pick(raw, 'flags', 'componentFlags') ?? []) as string[]; + return { + component_id: pick(raw, 'id', 'componentId', 'component_id') ?? '', + component_name: pick(raw, 'name', 'componentName', 'component_name') ?? '', + component_type: pick(raw, 'type', 'componentType', 'component_type') ?? '', + capabilities: capabilitiesFromFlags(flags), + links: [] as Link[], + }; +}; + +export const jsonBlock = (label: string, index: number, example: unknown): string => + `${index}. ${label}:\n\`\`\`json\n${JSON.stringify(example, null, 2)}\n\`\`\`\n\n`; + +/** Capabilities derived from developer-portal flags (port of ComponentCapabilities.from_flags). */ +const capabilitiesFromFlags = (flags: string[]) => ({ + is_row_based: flags.includes('genericDockerUI-rows'), + has_table_input: + flags.includes('genericDockerUI-tableInput') || + flags.includes('genericDockerUI-simpleTableInput'), + has_table_output: flags.includes('genericDockerUI-tableOutput'), + has_file_input: flags.includes('genericDockerUI-fileInput'), + has_file_output: flags.includes('genericDockerUI-fileOutput'), + requires_oauth: flags.includes('genericDockerUI-authorization'), +}); + +/** Maps a raw component (AI catalog or Storage API) to the Component output shape. */ +export const toComponent = (raw: RawComponent) => { + const flags = (pick(raw, 'flags', 'componentFlags') ?? []) as string[]; + const data = (pick>(raw, 'data') ?? {}) as Record; + return { + component_id: pick(raw, 'id', 'componentId', 'component_id') ?? '', + component_name: pick(raw, 'name', 'componentName', 'component_name') ?? '', + component_type: pick(raw, 'type', 'componentType', 'component_type') ?? '', + component_categories: pick(raw, 'categories', 'componentCategories') ?? [], + capabilities: capabilitiesFromFlags(flags), + documentation_url: pick(raw, 'documentationUrl', 'documentation_url') ?? null, + documentation: pick(raw, 'documentation') ?? null, + configuration_schema: pick(raw, 'configurationSchema', 'configuration_schema') ?? null, + configuration_row_schema: + pick(raw, 'configurationRowSchema', 'configuration_row_schema') ?? null, + sync_actions: (data.synchronous_actions as string[] | undefined) ?? null, + links: [] as unknown[], + }; +}; + +/** + * Fetches a component, preferring the AI catalog (docs + schemas) and merging the + * Storage API `data` (sync actions); falls back to Storage API on 404. Port of + * components/utils.py `fetch_component`. + * + * KEPT RAW: the AI catalog `docs/components/{id}` endpoint has no typed api-client + * equivalent, and the result merges the AI doc shape with the Storage component + * `data` field — a bespoke merge the typed `getComponent` does not reproduce. + */ +export const fetchComponent = async ( + clients: KeboolaClients, + componentId: string, +): Promise => { + try { + const fromAi = await clients.rawAi.get(`docs/components/${componentId}`); + const fromStorage = await clients.rawStorage.get( + `branch/${clients.branchId}/components/${componentId}`, + ); + fromAi.data = fromStorage.data ?? {}; + return fromAi; + } catch (error) { + if (error instanceof RawHttpError && error.status === 404) { + return clients.rawStorage.get( + `branch/${clients.branchId}/components/${componentId}`, + ); + } + throw error; + } +}; + +// ============================================================================ +// WRITE-TOOL HELPERS (ported from components/tools.py + utils.py). +// ============================================================================ + +const CREATED_BY_MCP = 'KBC.MCP.createdBy'; +const UPDATED_BY_MCP_PREFIX = 'KBC.MCP.updatedBy.version.'; + +/** Maps a fetched raw component into the minimal shape the validators need. */ +export const toComponentForValidation = (raw: RawComponent): ComponentForValidation => { + const flags = (pick(raw, 'flags', 'componentFlags') ?? []) as string[]; + return { + component_id: pick(raw, 'id', 'componentId', 'component_id') ?? '', + component_type: pick(raw, 'type', 'componentType', 'component_type') ?? '', + capabilities: { is_row_based: flags.includes('genericDockerUI-rows') }, + configuration_schema: + (pick(raw, 'configurationSchema', 'configuration_schema') as JsonDict | null) ?? + null, + configuration_row_schema: + (pick( + raw, + 'configurationRowSchema', + 'configuration_row_schema', + ) as JsonDict | null) ?? null, + }; +}; + +/** Builds a raw Encryption-service client rooted at the project's encryption URL. */ +const encryptionClient = (config: Config): RawClient => { + const urls = deriveServiceUrls(config.storageApiUrl ?? ''); + return createRawClient({ baseUrl: urls.encryption, token: config.storageToken }); +}; + +const isEncryptedValue = (value: unknown): boolean => + typeof value === 'string' && value.startsWith('KBC::'); + +const REDACTED_SECRET_VALUE = '[REDACTED]'; + +/** Yields [key, value] for every '#'-prefixed key, recursively. */ +const iterSecretItems = (value: unknown, out: [string, unknown][] = []): [string, unknown][] => { + if (Array.isArray(value)) { + for (const item of value) iterSecretItems(item, out); + } else if (value !== null && typeof value === 'object') { + for (const [key, item] of Object.entries(value)) { + if (key.startsWith('#')) out.push([key, item]); + else iterSecretItems(item, out); + } + } + return out; +}; + +/** + * Encrypts plaintext '#'-prefixed secrets before a config is written to Storage. + * Fail-closed: refuses to store redacted placeholders; otherwise calls the encryption + * service. Port of StorageClient._encrypt_secrets. + * + * KEPT RAW: the Encryption service (`encrypt`) is a separate service not covered by + * the typed Storage api-client. + */ +const encryptSecrets = async ( + config: Config, + clients: KeboolaClients, + componentId: string, + configuration: JsonDict, +): Promise => { + const items = iterSecretItems(configuration); + const plaintextKeys = items.filter(([, v]) => !isEncryptedValue(v)).map(([k]) => k); + if (plaintextKeys.length === 0) return configuration; + + const redactedKeys = items.filter(([, v]) => v === REDACTED_SECRET_VALUE).map(([k]) => k); + if (redactedKeys.length > 0) { + throw new Error( + `The configuration contains redacted secret values for keys: ${[...new Set(redactedKeys)].sort().join(', ')}. ` + + 'These are placeholders returned on configuration reads, not the actual secret values. ' + + 'Either leave the existing secret values untouched or ask the user to provide new ones.', + ); + } + + const token = await clients.storage.tokens.verify(); + const projectId = String((token.owner as { id: string | number }).id); + return encryptionClient(config).post('encrypt', { + params: { componentId, projectId }, + body: configuration, + }); +}; + +// ============================================================================ +// Configuration CRUD — typed where the api-client matches the SAPI shape exactly, +// raw where the typed method diverges (see per-method notes). +// ============================================================================ + +/** Branch-scoped Storage config base path (used by the kept-raw operations). */ +const cfgBase = (clients: KeboolaClients, componentId: string): string => + `branch/${clients.branchId}/components/${componentId}/configs`; + +/** MIGRATED → storage.componentsAndConfigurations.createConfiguration (POST configs). */ +export const configurationCreate = async ( + config: Config, + clients: KeboolaClients, + componentId: string, + name: string, + description: string, + configuration: JsonDict, +): Promise => + clients.storage.componentsAndConfigurations.createConfiguration({ + branchId: clients.branchId, + componentId, + name, + description, + configuration: await encryptSecrets(config, clients, componentId, configuration), + }) as Promise; + +/** + * KEPT RAW: the typed `updateConfiguration` JSON-encodes the `configuration` field + * to a string before sending; the SAPI call this server makes sends it as a nested + * JSON object, so the typed method's request shape diverges. + */ +export const configurationUpdate = async ( + config: Config, + clients: KeboolaClients, + componentId: string, + configurationId: string, + configuration: JsonDict, + changeDescription: string, + updatedName?: string, + updatedDescription?: string, +): Promise => { + const body: JsonDict = { + configuration: await encryptSecrets(config, clients, componentId, configuration), + changeDescription, + }; + if (updatedName) body.name = updatedName; + if (updatedDescription) body.description = updatedDescription; + return clients.rawStorage.put(`${cfgBase(clients, componentId)}/${configurationId}`, { + body, + }); +}; + +/** MIGRATED → storage.componentsAndConfigurations.createConfigurationRow (POST rows). */ +export const configurationRowCreate = async ( + config: Config, + clients: KeboolaClients, + componentId: string, + configId: string, + name: string, + description: string, + configuration: JsonDict, +): Promise => + clients.storage.componentsAndConfigurations.createConfigurationRow({ + branchId: clients.branchId, + componentId, + configId, + name, + description, + configuration: await encryptSecrets(config, clients, componentId, configuration), + }) as Promise; + +/** + * KEPT RAW: the typed client has no `updateConfigurationRow` method, and the only + * row writer (`createConfigurationRow`) cannot carry `isDisabled`/`changeDescription` + * onto an existing row update. + */ +export const configurationRowUpdate = async ( + config: Config, + clients: KeboolaClients, + componentId: string, + configId: string, + rowId: string, + configuration: JsonDict, + changeDescription: string, + updatedName?: string, + updatedDescription?: string, + isDisabled?: boolean | null, +): Promise => { + const body: JsonDict = { + configuration: await encryptSecrets(config, clients, componentId, configuration), + changeDescription, + }; + if (updatedName) body.name = updatedName; + if (updatedDescription) body.description = updatedDescription; + if (isDisabled !== undefined && isDisabled !== null) body.isDisabled = isDisabled; + return clients.rawStorage.put( + `${cfgBase(clients, componentId)}/${configId}/rows/${rowId}`, + { body }, + ); +}; + +/** + * KEPT RAW: callers (variables resolution, update_sql_transformation) branch on + * `RawHttpError.status === 404` for control flow; the typed `getConfiguration` + * throws an `ApiError` of a different shape, so its error contract diverges. + */ +export const configurationDetail = ( + clients: KeboolaClients, + componentId: string, + configurationId: string, +): Promise => + clients.rawStorage.get(`${cfgBase(clients, componentId)}/${configurationId}`); + +/** + * KEPT RAW: the typed client has no single-row GET; only the parent config GET + * (which embeds rows) is exposed. + */ +export const configurationRowDetail = ( + clients: KeboolaClients, + componentId: string, + configId: string, + rowId: string, +): Promise => + clients.rawStorage.get(`${cfgBase(clients, componentId)}/${configId}/rows/${rowId}`); + +/** + * KEPT RAW: paired with the kept-raw `configurationDetail` for a consistent error + * contract (callers fall back from a 404 detail to a `RawHttpError`-typed list scan). + */ +export const configurationList = ( + clients: KeboolaClients, + componentId: string, +): Promise => clients.rawStorage.get(cfgBase(clients, componentId)); + +// ============================================================================ +// Configuration metadata helpers. +// +// KEPT RAW: configuration-metadata endpoints (`.../configs/{id}/metadata`) are not +// exposed by the typed api-client. +// ============================================================================ + +const metadataUpdate = ( + clients: KeboolaClients, + componentId: string, + configurationId: string, + metadata: Record, +): Promise => + clients.rawStorage.post( + `${cfgBase(clients, componentId)}/${configurationId}/metadata`, + { + body: { metadata: Object.entries(metadata).map(([key, value]) => ({ key, value })) }, + }, + ); + +const metadataGet = ( + clients: KeboolaClients, + componentId: string, + configurationId: string, +): Promise => + clients.rawStorage.get( + `${cfgBase(clients, componentId)}/${configurationId}/metadata`, + ); + +export const setCfgCreationMetadata = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, +): Promise => { + try { + await metadataUpdate(clients, componentId, configurationId, { [CREATED_BY_MCP]: 'true' }); + } catch (error) { + logger.error( + { err: error }, + `Failed to set "${CREATED_BY_MCP}" metadata for ${configurationId}.`, + ); + } +}; + +export const setCfgUpdateMetadata = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, + version: number | string, +): Promise => { + const key = `${UPDATED_BY_MCP_PREFIX}${version}`; + try { + await metadataUpdate(clients, componentId, configurationId, { [key]: 'true' }); + } catch (error) { + logger.error({ err: error }, `Failed to set "${key}" metadata for ${configurationId}.`); + } +}; + +const setFolderMetadata = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, + folder: string, +): Promise => { + const normalized = folder.trim(); + if (!normalized) return; + await metadataUpdate(clients, componentId, configurationId, { + [MetadataField.CONFIGURATION_FOLDER_NAME]: normalized, + }); +}; + +const clearFolderMetadata = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, +): Promise => { + try { + const metadata = await metadataGet(clients, componentId, configurationId); + for (const entry of metadata) { + if ((entry as MetadataItem).key === MetadataField.CONFIGURATION_FOLDER_NAME) { + const id = (entry as { id?: string }).id; + if (id === undefined) continue; + await clients.rawStorage.delete( + `${cfgBase(clients, componentId)}/${configurationId}/metadata/${id}`, + ); + } + } + } catch { + logger.warn(`Unable to clear folder metadata for "${componentId}"/"${configurationId}".`); + } +}; + +export { setFolderMetadata, clearFolderMetadata }; + +/** Counts configs + distinct folders for the folder hint (port of get_config_folders). */ +export const getConfigFolders = async ( + clients: KeboolaClients, + componentId: string, +): Promise<[number, string[], boolean]> => { + // KEPT RAW: the search endpoint's `metadataKeys[N]` bracketed-index query param is + // not expressible through the typed `searchComponentConfigurations` query shape. + const folderConfigs = await clients.rawStorage.get( + `branch/${clients.branchId}/search/component-configurations`, + { params: { componentId, 'metadataKeys[0]': MetadataField.CONFIGURATION_FOLDER_NAME } }, + ); + const seen = new Set(); + const folders: string[] = []; + for (const cfg of folderConfigs) { + for (const meta of (cfg.metadata as MetadataItem[]) ?? []) { + if (meta.key === MetadataField.CONFIGURATION_FOLDER_NAME) { + const name = (meta.value ?? '').trim(); + if (name && !seen.has(name)) { + seen.add(name); + folders.push(name); + } + } + } + } + if (folderConfigs.length >= 20) return [folderConfigs.length, folders, true]; + const allConfigs = await configurationList(clients, componentId); + const total = allConfigs.length; + if (total < 20) return [total, [], false]; + return [total, folders, false]; +}; + +/** + * Resolves the workspace SQL dialect ('snowflake' | 'bigquery'). The TS port has no + * WorkspaceManager yet, so this mirrors the essential resolution: prefer an existing + * workspace's backend, else the token owner's defaultBackend. + * + * KEPT RAW: the branch `workspaces` listing is not exposed by the typed api-client. + */ +export const resolveSqlDialect = async (clients: KeboolaClients): Promise => { + try { + const workspaces = await clients.rawStorage.get( + `branch/${clients.branchId}/workspaces`, + ); + for (const ws of workspaces) { + const backend = (ws.connection as JsonDict | undefined)?.backend as string | undefined; + if (backend === 'snowflake' || backend === 'bigquery') return backend; + } + } catch { + // fall through to token-based default + } + const token = await clients.storage.tokens.verify(); + const defaultBackend = (token.owner as { defaultBackend?: string } | undefined)?.defaultBackend; + if (defaultBackend === 'snowflake' || defaultBackend === 'bigquery') return defaultBackend; + throw new Error(`Unexpected default backend: ${defaultBackend}`); +}; + +// ============================================================================ +// Variables management (port of _apply_vars_to_parent_cfg / apply_configuration_variables). +// ============================================================================ + +/** + * Creates/updates/clears the keboola.variables config linked to a parent. Mirrors + * _apply_vars_to_parent_cfg: mutates parentCfg with link fields and returns the id of + * any variables config the caller must delete AFTER writing the parent. + */ +export const applyVarsToParentCfg = async ( + config: Config, + clients: KeboolaClients, + componentId: string, + configId: string, + variables: VariableDefinition[], + parentCfg: JsonDict, +): Promise<{ changed: boolean; varsConfigIdToDelete: string | null }> => { + const varsName = `Variables definition for ${componentId}/${configId}`; + + const findVarsConfig = async (): Promise => { + const existingId = parentCfg.variables_id as string | undefined; + if (existingId) { + try { + return await configurationDetail(clients, VARIABLES_COMPONENT_ID, existingId); + } catch (error) { + if (!(error instanceof RawHttpError) || error.status !== 404) throw error; + } + } + const all = await configurationList(clients, VARIABLES_COMPONENT_ID); + const found = all.find((c) => c.name === varsName); + if (!found) return null; + return configurationDetail(clients, VARIABLES_COMPONENT_ID, String(found.id)); + }; + + const existing = await findVarsConfig(); + + if (variables.length === 0) { + const varsConfigIdToDelete = existing ? String(existing.id) : null; + let changed = false; + for (const key of ['variables_id', 'variables_values_id']) { + if (key in parentCfg) { + delete parentCfg[key]; + changed = true; + } + } + return { changed, varsConfigIdToDelete }; + } + + const varDefs = variables.map((v) => ({ name: v.name, type: v.type })); + const varsConfiguration = { variables: varDefs }; + let varsConfigId: string; + if (existing === null) { + const created = await configurationCreate( + config, + clients, + VARIABLES_COMPONENT_ID, + varsName, + '', + varsConfiguration, + ); + varsConfigId = String(created.id); + } else { + varsConfigId = String(existing.id); + await configurationUpdate( + config, + clients, + VARIABLES_COMPONENT_ID, + varsConfigId, + varsConfiguration, + 'Update variable definitions', + ); + } + + const defaults = variables + .filter((v) => v.default_value !== null && v.default_value !== undefined) + .map((v) => ({ name: v.name, value: v.default_value })); + const existingRows = ((existing ?? {}).rows as JsonDict[] | undefined) ?? []; + const defaultRow = existingRows.find((r) => r.name === 'Default Values'); + let defaultValuesRowId: string | null = null; + if (defaults.length > 0) { + const rowCfg = { values: defaults }; + if (!defaultRow) { + const createdRow = await configurationRowCreate( + config, + clients, + VARIABLES_COMPONENT_ID, + varsConfigId, + 'Default Values', + '', + rowCfg, + ); + defaultValuesRowId = String(createdRow.id); + } else { + defaultValuesRowId = String(defaultRow.id); + await configurationRowUpdate( + config, + clients, + VARIABLES_COMPONENT_ID, + varsConfigId, + defaultValuesRowId, + rowCfg, + 'Update default variable values', + ); + } + } else if (defaultRow) { + await configurationRowUpdate( + config, + clients, + VARIABLES_COMPONENT_ID, + varsConfigId, + String(defaultRow.id), + { values: [] }, + 'Clear default variable values', + ); + } + + parentCfg.variables_id = varsConfigId; + if (defaultValuesRowId !== null) parentCfg.variables_values_id = defaultValuesRowId; + else delete parentCfg.variables_values_id; + return { changed: true, varsConfigIdToDelete: null }; +}; + +/** Full create-or-clear variables flow used by create_config / create_sql_transformation. */ +export const applyConfigurationVariables = async ( + config: Config, + clients: KeboolaClients, + componentId: string, + configId: string, + variables: VariableDefinition[], +): Promise => { + const parent = await configurationDetail(clients, componentId, configId); + const parentCfg = structuredClone((parent.configuration as JsonDict) ?? {}); + const { changed, varsConfigIdToDelete } = await applyVarsToParentCfg( + config, + clients, + componentId, + configId, + variables, + parentCfg, + ); + if (!changed && !varsConfigIdToDelete) return null; + const changeDescription = variables.length ? 'Link variables' : 'Unlink variables'; + let result: JsonDict | null = null; + if (changed) { + result = await configurationUpdate( + config, + clients, + componentId, + configId, + parentCfg, + changeDescription, + ); + } + if (varsConfigIdToDelete) { + await deleteVariablesConfig(clients, varsConfigIdToDelete); + } + return result; +}; + +/** + * Deletes a keboola.variables config with skip_trash semantics (two deletes). + * + * KEPT RAW: skip-trash requires issuing the DELETE twice; the typed + * `deleteConfiguration` performs a single delete. + */ +export const deleteVariablesConfig = async ( + clients: KeboolaClients, + varsConfigId: string, +): Promise => { + const path = `${cfgBase(clients, VARIABLES_COMPONENT_ID)}/${varsConfigId}`; + await clients.rawStorage.delete(path); + await clients.rawStorage.delete(path); // skip_trash = two deletes +}; + +export const nowIso = (): string => new Date().toISOString(); diff --git a/src/tools/data_apps/client.ts b/src/tools/data_apps/client.ts new file mode 100644 index 000000000..d221dddeb --- /dev/null +++ b/src/tools/data_apps/client.ts @@ -0,0 +1,418 @@ +import type { KeboolaClients } from '@/clients/keboola'; +import { createRawClient, RawHttpError } from '@/clients/raw'; +import { deriveServiceUrls } from '@/clients/urls'; +import type { Config } from '@/config'; +import { DATA_APP_COMPONENT_ID, MetadataField } from '@/constants'; +import type { ProjectLinksManager } from '@/links'; +import { logger } from '@/logger'; +import { + type AppGitRepoResponse, + type AppRunInfo, + type AppRunResponse, + type CreatedGitCredentialResponse, + type DataApp, + type DataAppResponse, + type DataAppSummary, + parseAppGitRepoResponse, + parseAppRunResponse, + parseCredentialResponse, + parseDataAppResponse, +} from './model'; +import { + APP_RUN_LOG_LINES, + APP_RUN_MESSAGE_LIMIT, + asRecord, + dataAppFromApiResponses, + isDraftConfig, + type MetadataItem, + summaryFromDataApp, + usesBasicAuthentication, +} from './utils'; + +// --------------------------------------------------------------------------- +// Data Science access. +// +// Apps CRUD (list/get/create/patch/delete) and runs are migrated to the typed +// `clients.dataScience` client. Three endpoints stay on a raw client rooted at the +// data-science URL because the typed `createDataScienceClient` exposes no method for +// them (verified against node_modules/@keboola/api-client/dist/dataScience/index.d.ts): +// - GET apps/{id}/git-repo (managed-repo URL read) +// - POST apps/{id}/git-repo/credentials (mint one-time http_token) +// - GET apps/{id}/logs/tail (typed getAppLogsTail re-parses the +// text/plain body into structured +// LogEntry[] + drops empty lines, which +// would change tool output; we need the +// raw lines verbatim) +// --------------------------------------------------------------------------- + +export type DataScience = { + getDataApp: (id: string) => Promise; + listDataApps: (limit: number, offset: number) => Promise; + createDataApp: (params: { + name: string; + description: string; + config: Record; + branchId: string | null; + appType: string; + useManagedGitRepo: boolean; + }) => Promise; + deployDataApp: ( + id: string, + configVersion: string | null, + mode: string | null, + ) => Promise; + suspendDataApp: (id: string) => Promise; + deleteDataApp: (id: string) => Promise; + // kept-raw endpoints (no typed method) + createAppGitCredential: (id: string) => Promise; + getAppGitRepo: (id: string) => Promise; + listAppRuns: (id: string, limit: number) => Promise; + tailAppLogs: (id: string, lines: number) => Promise; +}; + +export const createDataScience = (clients: KeboolaClients, config: Config): DataScience => { + const ds = clients.dataScience; + // Raw client for the three uncovered data-science endpoints (git-repo read, + // credential create, raw logs tail). Same baseUrl/token as the typed client. + const urls = deriveServiceUrls(config.storageApiUrl ?? ''); + const token = config.bearerToken ? `Bearer ${config.bearerToken}` : config.storageToken; + const raw = createRawClient({ baseUrl: urls.dataScience, token }); + + return { + getDataApp: async (id) => + parseDataAppResponse((await ds.getApp(id)) as unknown as Record), + listDataApps: async (limit, offset) => { + const resp = await ds.getApps({ limit, offset }); + return resp.map((r) => parseDataAppResponse(r as unknown as Record)); + }, + createDataApp: async ({ + name, + description, + config: cfg, + branchId, + appType, + useManagedGitRepo, + }) => { + const body: Record = { + branchId, + name, + type: appType, + description, + config: cfg, + }; + if (useManagedGitRepo) body.useManagedGitRepo = true; + const resp = await ds.createApp(body as Parameters[0]); + return parseDataAppResponse(resp as unknown as Record); + }, + deployDataApp: async (id, configVersion, mode) => { + const body: Record = { + desiredState: 'running', + restartIfRunning: true, + updateDependencies: false, + }; + if (configVersion !== null) body.configVersion = configVersion; + if (mode !== null) body.mode = mode; + const resp = await ds.patchApp(id, body as Parameters[1]); + return parseDataAppResponse(resp as unknown as Record); + }, + suspendDataApp: async (id) => { + const resp = await ds.patchApp(id, { desiredState: 'stopped' }); + return parseDataAppResponse(resp as unknown as Record); + }, + deleteDataApp: async (id) => { + await ds.deleteApp(id); + }, + // KEEP RAW: no typed method for POST apps/{id}/git-repo/credentials. + createAppGitCredential: async (id) => + parseCredentialResponse( + await raw.post>(`apps/${id}/git-repo/credentials`, { + body: { type: 'http_token', permissions: 'readWrite' }, + }), + ), + // KEEP RAW: no typed method for GET apps/{id}/git-repo. + getAppGitRepo: async (id) => + parseAppGitRepoResponse(await raw.get>(`apps/${id}/git-repo`)), + listAppRuns: async (id, limit) => { + const resp = await ds.getAppRuns(id, { limit, offset: 0 }); + return resp.map((r) => parseAppRunResponse(r as unknown as Record)); + }, + // KEEP RAW: the typed getAppLogsTail parses the text/plain body into structured + // LogEntry[] (and drops empty lines); we need the raw text split into lines verbatim. + tailAppLogs: async (id, lines) => + raw.getText(`apps/${id}/logs/tail`, { params: { lines: Math.max(lines, 1) } }), + }; +}; + +// --------------------------------------------------------------------------- +// Storage helpers (KEEP RAW: storage-config writes/metadata via rawStorage, matching +// the exact Python SAPI calls; the typed storage client diverges from these shapes). +// --------------------------------------------------------------------------- +export const storageHelpers = (clients: KeboolaClients) => { + const branch = clients.branchId; + const cfgBase = (configurationId: string): string => + `branch/${branch}/components/${DATA_APP_COMPONENT_ID}/configs/${configurationId}`; + + return { + configurationDetail: (configurationId: string) => + clients.rawStorage.get>(cfgBase(configurationId)), + configurationList: () => + clients.rawStorage.get[]>( + `branch/${branch}/components/${DATA_APP_COMPONENT_ID}/configs`, + ), + configurationUpdate: (params: { + configurationId: string; + configuration: Record; + changeDescription: string; + updatedName?: string; + updatedDescription?: string | null; + }) => { + const body: Record = { + configuration: params.configuration, + changeDescription: params.changeDescription, + }; + if (params.updatedName) body.name = params.updatedName; + if (params.updatedDescription) body.description = params.updatedDescription; + return clients.rawStorage.put>(cfgBase(params.configurationId), { + body, + }); + }, + configurationMetadataGet: (configurationId: string) => + clients.rawStorage.get(`${cfgBase(configurationId)}/metadata`), + configurationMetadataUpdate: (configurationId: string, metadata: Record) => + clients.rawStorage.post(`${cfgBase(configurationId)}/metadata`, { + body: { + metadata: Object.entries(metadata).map(([key, value]) => ({ key, value })), + }, + }), + configurationMetadataDelete: (configurationId: string, metadataId: string) => + clients.rawStorage.delete(`${cfgBase(configurationId)}/metadata/${metadataId}`), + configurationVersionLatest: async (configurationId: string): Promise => { + const versions = await clients.rawStorage.get<{ version?: number }[]>( + `${cfgBase(configurationId)}/versions`, + ); + let latest = 0; + for (const v of versions) { + if (typeof v.version === 'number' && v.version > latest) latest = v.version; + } + return latest; + }, + branchesList: () => clients.rawStorage.get[]>('dev-branches'), + workspaceList: () => + clients.rawStorage.get[]>(`branch/${branch}/workspaces`), + }; +}; + +export type StorageHelpers = ReturnType; + +// --------------------------------------------------------------------------- +// Project feature check (port of KeboolaClient.has_feature): tokens/verify.owner.features +// --------------------------------------------------------------------------- +export const hasFeature = async (clients: KeboolaClients, feature: string): Promise => { + const token = (await clients.storage.tokens.verify()) as { owner?: { features?: string[] } }; + return (token.owner?.features ?? []).includes(feature); +}; + +// --- minimal workspace resolution (Streamlit path only) --------------------- +// Resolves workspace_id / sql_dialect / branch_id from the configured workspaceSchema, +// matching the standard MCP path in workspace.py (_find_ws_by_schema + backend dialect). +export const resolveWorkspace = async ( + config: Config, + helpers: StorageHelpers, +): Promise<{ workspaceId: string; sqlDialect: string; branchId: string }> => { + if (!config.workspaceSchema) { + throw new Error( + 'No Keboola workspace schema configured; required to create or update a Streamlit data app.', + ); + } + const workspaces = await helpers.workspaceList(); + const match = workspaces.find((ws) => { + const connection = (ws.connection as { schema?: string; backend?: string } | undefined) ?? {}; + return connection.schema === config.workspaceSchema; + }); + if (!match) { + throw new Error( + `No Keboola workspace found or the workspace has no read-only storage access: ` + + `workspace_schema=${config.workspaceSchema}`, + ); + } + const connection = (match.connection as { backend?: string }) ?? {}; + const backend = String(connection.backend ?? ''); + const workspaceId = String(match.id ?? ''); + + let branchId = config.branchId ?? ''; + if (!branchId) { + const branches = await helpers.branchesList(); + const defaultBranch = branches.find((b) => b.isDefault === true); + if (!defaultBranch?.id) throw new Error('Cannot determine the default branch ID'); + branchId = String(defaultBranch.id); + } + return { workspaceId, sqlDialect: backend, branchId }; +}; + +// --- data app fetch / build (ports of _fetch_data_app etc.) ----------------- + +const buildDataAppWithRepo = async ( + ds: DataScience, + dataAppScience: DataAppResponse, + rawConfig: Record, +): Promise => { + const dataApp = dataAppFromApiResponses( + dataAppScience, + rawConfig, + MetadataField.CONFIGURATION_FOLDER_NAME, + ); + if (dataAppScience.type === 'python-js') { + try { + const repo = await ds.getAppGitRepo(dataAppScience.id); + dataApp.repo_url = repo.https_url; + } catch (error) { + logger.warn(`Could not fetch git repo URL for python-js app ${dataAppScience.id}: ${error}`); + } + } + return dataApp; +}; + +export const fetchDataApp = async ( + ds: DataScience, + helpers: StorageHelpers, + configurationId: string, +): Promise => { + const rawConfig = await helpers.configurationDetail(configurationId); + const config = asRecord(rawConfig.configuration); + const dataAppId = String(asRecord(config.parameters).id ?? ''); + const dataAppScience = await ds.getDataApp(dataAppId); + if (dataAppScience.component_id !== DATA_APP_COMPONENT_ID) { + throw new Error( + `Data app tools only support ${DATA_APP_COMPONENT_ID} component, but the data app ` + + `"${dataAppId}" has component_id "${dataAppScience.component_id}".`, + ); + } + return buildDataAppWithRepo(ds, dataAppScience, rawConfig); +}; + +const appRunInfoFromResponse = (run: AppRunResponse): AppRunInfo => { + const lines = (run.startup_logs ?? '').trim().split('\n'); + const startupLogs = lines.slice(Math.max(0, lines.length - APP_RUN_LOG_LINES)).filter(Boolean); + let failureMessage = run.failure_reason?.message ?? null; + if (failureMessage && failureMessage.length > APP_RUN_MESSAGE_LIMIT) { + failureMessage = '…' + failureMessage.slice(-(APP_RUN_MESSAGE_LIMIT - 1)); + } + return { + state: run.state, + created_at: run.created_at, + stopped_at: run.stopped_at, + failure_reason: run.failure_reason?.reason ?? null, + failure_message: failureMessage, + startup_logs: startupLogs, + }; +}; + +export const fetchLogs = async (ds: DataScience, dataAppId: string): Promise => { + try { + const text = await ds.tailAppLogs(dataAppId, 20); + return text.split('\n'); + } catch (error) { + if (error instanceof RawHttpError) return []; + throw error; + } +}; + +export const fetchLatestRun = async ( + ds: DataScience, + dataAppId: string, +): Promise => { + try { + const runs = await ds.listAppRuns(dataAppId, 1); + if (runs.length === 0) return null; + return appRunInfoFromResponse(runs[0]!); + } catch (error) { + logger.error({ err: error }, `Failed to fetch app runs for data app: ${dataAppId}`); + return null; + } +}; + +export const withDeploymentInfo = ( + dataApp: DataApp, + logs: string[], + lastRun: AppRunInfo | null, +): DataApp => { + dataApp.deployment_info = { + version: dataApp.config_version, + state: dataApp.state, + url: dataApp.deployment_url ?? 'deployment link not available yet', + last_request_timestamp: null, + last_start_timestamp: null, + logs, + last_run: lastRun, + }; + return dataApp; +}; + +const fetchProdDrafts = async ( + ds: DataScience, + helpers: StorageHelpers, + prodConfigurationId: string, +): Promise<{ drafts: DataAppSummary[]; unavailable: number }> => { + const configs = await helpers.configurationList(); + const draftCfgIds: string[] = []; + for (const cfg of configs) { + const body = asRecord(cfg.configuration); + if (!isDraftConfig(body)) continue; + const dataAppBlock = asRecord(asRecord(body.parameters).dataApp); + if (dataAppBlock.parentConfigurationId === prodConfigurationId) { + if (typeof cfg.id === 'string') draftCfgIds.push(cfg.id); + } + } + if (draftCfgIds.length === 0) return { drafts: [], unavailable: 0 }; + + const results = await Promise.all( + draftCfgIds.map(async (cfgId): Promise => { + try { + const draft = await fetchDataApp(ds, helpers, cfgId); + const summary = summaryFromDataApp(draft); + summary.repo_url = draft.repo_url; + return summary; + } catch (error) { + logger.error( + { err: error }, + `Failed to fetch draft data app by configuration ID: ${cfgId}`, + ); + return null; + } + }), + ); + const drafts = results.filter((s): s is DataAppSummary => s !== null); + return { drafts, unavailable: draftCfgIds.length - drafts.length }; +}; + +export const fetchDataAppDetailsTask = async ( + ds: DataScience, + helpers: StorageHelpers, + linksManager: ProjectLinksManager, + configurationId: string, +): Promise => { + try { + let dataApp = await fetchDataApp(ds, helpers, configurationId); + dataApp.links = linksManager.getDataAppLinks( + dataApp.configuration_id, + dataApp.name, + dataApp.deployment_url ?? undefined, + usesBasicAuthentication(asRecord(dataApp.configuration.authorization)), + ); + const logs = await fetchLogs(ds, dataApp.data_app_id); + const lastRun = await fetchLatestRun(ds, dataApp.data_app_id); + dataApp = withDeploymentInfo(dataApp, logs, lastRun); + if (dataApp.type === 'python-js' && !isDraftConfig(dataApp.configuration)) { + const { drafts, unavailable } = await fetchProdDrafts(ds, helpers, dataApp.configuration_id); + dataApp.drafts = drafts; + dataApp.drafts_unavailable = unavailable; + } + return dataApp; + } catch (error) { + logger.error( + { err: error }, + `Failed to fetch data app by configuration ID: ${configurationId}`, + ); + return configurationId; + } +}; diff --git a/src/tools/data_apps/index.ts b/src/tools/data_apps/index.ts new file mode 100644 index 000000000..8ed704efa --- /dev/null +++ b/src/tools/data_apps/index.ts @@ -0,0 +1,3 @@ +// Public entrypoint for the data_apps tool module. server.ts imports +// `registerDataAppTools` from `@/tools/data_apps`, which resolves to this index. +export { registerDataAppTools } from './tools'; diff --git a/src/tools/data_apps/metadata.ts b/src/tools/data_apps/metadata.ts new file mode 100644 index 000000000..f45c027f0 --- /dev/null +++ b/src/tools/data_apps/metadata.ts @@ -0,0 +1,124 @@ +import { MetadataField } from '@/constants'; +import { logger } from '@/logger'; +import type { StorageHelpers } from './client'; +import { CREATED_BY_MCP, type MetadataItem, UPDATED_BY_MCP_PREFIX } from './utils'; + +// --- metadata helpers (ports of components/utils.py) ------------------------ + +export const setCfgCreationMetadata = async ( + helpers: StorageHelpers, + configurationId: string, +): Promise => { + try { + await helpers.configurationMetadataUpdate(configurationId, { [CREATED_BY_MCP]: 'true' }); + } catch (error) { + logger.error( + { err: error }, + `Failed to set "${CREATED_BY_MCP}" metadata for ${configurationId}`, + ); + } +}; + +export const setCfgUpdateMetadata = async ( + helpers: StorageHelpers, + configurationId: string, + configurationVersion: number, +): Promise => { + const key = `${UPDATED_BY_MCP_PREFIX}${configurationVersion}`; + try { + await helpers.configurationMetadataUpdate(configurationId, { [key]: 'true' }); + } catch (error) { + logger.error({ err: error }, `Failed to set "${key}" metadata for ${configurationId}`); + } +}; + +const buildFolderHint = ( + total: number, + existingFolders: string[], + configLabel: string, + updateTool: string, + lowerBound: boolean, +): string | null => { + if (total < 20) return null; + const countStr = lowerBound ? `at least ${total}` : String(total); + let hint = `Note: This project already has ${countStr} ${configLabel}. Consider organizing them with folders. `; + if (existingFolders.length > 0) { + hint += + `Existing folders: ${existingFolders.join(', ')}. ` + + `Call ${updateTool} with a folder= parameter to assign this to one.`; + } else { + hint += `No folders have been created yet. Call ${updateTool} with a folder= parameter to start organizing.`; + } + return hint; +}; + +const getConfigFolders = async ( + helpers: StorageHelpers, +): Promise<{ total: number; folders: string[]; lowerBound: boolean }> => { + const allConfigs = await helpers.configurationList(); + const seen = new Set(); + const folders: string[] = []; + let folderBearing = 0; + for (const cfg of allConfigs) { + const metadata = (cfg.metadata as MetadataItem[]) ?? []; + let hasFolder = false; + for (const meta of metadata) { + if (meta.key === MetadataField.CONFIGURATION_FOLDER_NAME) { + hasFolder = true; + const folderName = (meta.value ?? '').trim(); + if (folderName && !seen.has(folderName)) { + seen.add(folderName); + folders.push(folderName); + } + } + } + if (hasFolder) folderBearing += 1; + } + // configuration_list does not embed metadata server-side the way the search endpoint does, + // so we derive the total from the same list (faithful to the resulting hint behavior). + const total = allConfigs.length; + if (folderBearing >= 20) return { total: folderBearing, folders, lowerBound: true }; + if (total < 20) return { total, folders: [], lowerBound: false }; + return { total, folders, lowerBound: false }; +}; + +export const applyFolderMetadata = async ( + helpers: StorageHelpers, + configurationId: string, + folder: string | null | undefined, + plural: string, + toolName: string, + isNew = false, +): Promise => { + if (folder == null) { + try { + const { total, folders, lowerBound } = await getConfigFolders(helpers); + return buildFolderHint(total, folders, plural, toolName, lowerBound); + } catch { + logger.warn(`Unable to fetch ${plural} folders for configuration "${configurationId}".`); + return null; + } + } + const normalized = folder.trim(); + if (normalized) { + try { + await helpers.configurationMetadataUpdate(configurationId, { + [MetadataField.CONFIGURATION_FOLDER_NAME]: normalized, + }); + } catch { + logger.warn(`Unable to set folder metadata for configuration "${configurationId}".`); + } + } else if (!isNew) { + try { + const metadata = await helpers.configurationMetadataGet(configurationId); + for (const entry of metadata) { + if (entry.key === MetadataField.CONFIGURATION_FOLDER_NAME && entry.id) { + await helpers.configurationMetadataDelete(configurationId, entry.id); + } + } + } catch { + logger.warn(`Unable to clear folder metadata for configuration "${configurationId}".`); + } + } + return null; +}; diff --git a/src/tools/data_apps/model.ts b/src/tools/data_apps/model.ts new file mode 100644 index 000000000..39f2e53c1 --- /dev/null +++ b/src/tools/data_apps/model.ts @@ -0,0 +1,193 @@ +import { z } from 'zod'; + +import type { Link } from '@/links'; + +// Models ported from tools/data_apps.py + clients/data_science.py. +// snake_case output field names are preserved verbatim for 1:1 tool-output parity. + +/** Authentication type accepted by the modify_* tools. */ +export const AUTHENTICATION_TYPES = ['no-auth', 'basic-auth', 'default'] as const; +export type AuthenticationType = (typeof AUTHENTICATION_TYPES)[number]; + +// --- data-science API response models (port of clients/data_science.py) --- + +/** + * Raw data-science `apps/{id}` response. Validation aliases from the Python models + * (camelCase + snake_case) are normalized via `parseDataAppResponse`. + */ +export type DataAppResponse = { + id: string; + project_id: string; + component_id: string; + branch_id: string | null; + config_id: string; + config_version: string; + type: string; + state: string; + desired_state?: string; + last_request_timestamp?: string | null; + last_start_timestamp?: string | null; + url?: string | null; + auto_suspend_after_seconds?: number | null; + size?: string | null; +}; + +const str = (value: unknown): string => (value == null ? '' : String(value)); +const strOrNull = (value: unknown): string | null => (value == null ? null : String(value)); + +/** Picks the first present alias (camelCase preferred), mirroring AliasChoices. */ +const pick = (raw: Record, ...keys: string[]): unknown => { + for (const key of keys) { + if (raw[key] !== undefined && raw[key] !== null) return raw[key]; + } + return undefined; +}; + +export const parseDataAppResponse = (raw: Record): DataAppResponse => ({ + id: str(pick(raw, 'id', 'data_app_id')), + project_id: str(pick(raw, 'projectId', 'project_id')), + component_id: str(pick(raw, 'componentId', 'component_id')), + branch_id: strOrNull(pick(raw, 'branchId', 'branch_id')), + config_id: str(pick(raw, 'configId', 'config_id')), + config_version: str(pick(raw, 'configVersion', 'config_version')), + type: str(raw.type), + state: str(raw.state), + desired_state: raw.desiredState != null ? String(raw.desiredState) : undefined, + last_request_timestamp: strOrNull(pick(raw, 'lastRequestTimestamp', 'last_request_timestamp')), + last_start_timestamp: strOrNull(pick(raw, 'lastStartTimestamp', 'last_start_timestamp')), + url: strOrNull(raw.url), + auto_suspend_after_seconds: + (pick(raw, 'autoSuspendAfterSeconds', 'auto_suspend_after_seconds') as number | undefined) ?? + null, + size: strOrNull(raw.size), +}); + +export type CreatedGitCredentialResponse = { + id: string; + type: string; + name: string; + permissions: string; + owner_admin_id: string | null; + created_at: string | null; + secret: string | null; +}; + +export const parseCredentialResponse = ( + raw: Record, +): CreatedGitCredentialResponse => ({ + id: str(raw.id), + type: str(raw.type), + name: raw.name != null ? String(raw.name) : '', + permissions: str(raw.permissions), + owner_admin_id: strOrNull(pick(raw, 'ownerAdminId', 'owner_admin_id')), + created_at: strOrNull(pick(raw, 'createdAt', 'created_at')), + secret: raw.secret != null ? String(raw.secret) : null, +}); + +export type AppGitRepoResponse = { + ssh_url: string | null; + https_url: string | null; + is_managed_git_repo: boolean; +}; + +export const parseAppGitRepoResponse = (raw: Record): AppGitRepoResponse => ({ + ssh_url: strOrNull(pick(raw, 'sshUrl', 'ssh_url')), + https_url: strOrNull(pick(raw, 'httpsUrl', 'https_url')), + is_managed_git_repo: Boolean(pick(raw, 'isManagedGitRepo', 'is_managed_git_repo') ?? false), +}); + +export type AppRunResponse = { + id: string; + app_id: string | null; + state: string; + created_at: string | null; + started_at: string | null; + stopped_at: string | null; + startup_logs: string | null; + failure_reason: { reason: string | null; message: string | null } | null; + mode: string | null; +}; + +export const parseAppRunResponse = (raw: Record): AppRunResponse => { + const failure = pick(raw, 'failureReason', 'failure_reason') as + | Record + | undefined; + return { + id: str(raw.id), + app_id: strOrNull(pick(raw, 'appId', 'app_id')), + state: str(raw.state), + created_at: strOrNull(pick(raw, 'createdAt', 'created_at')), + started_at: strOrNull(pick(raw, 'startedAt', 'started_at')), + stopped_at: strOrNull(pick(raw, 'stoppedAt', 'stopped_at')), + startup_logs: strOrNull(pick(raw, 'startupLogs', 'startup_logs')), + failure_reason: failure + ? { + reason: failure.reason != null ? String(failure.reason) : null, + message: failure.message != null ? String(failure.message) : null, + } + : null, + mode: raw.mode != null ? String(raw.mode) : null, + }; +}; + +// --- tool output models (port of the Pydantic BaseModels in data_apps.py) --- + +export type AppRunInfo = { + state: string; + created_at: string | null; + stopped_at: string | null; + failure_reason: string | null; + failure_message: string | null; + startup_logs: string[]; +}; + +export type DataAppSummary = { + component_id: string; + configuration_id: string; + data_app_id: string; + project_id: string; + branch_id: string; + config_version: string; + state: string; + type: string; + deployment_url: string | null; + auto_suspend_after_seconds: number | null; + repo_url: string | null; +}; + +export type DeploymentInfo = { + version: string; + state: string; + url: string | null; + last_request_timestamp: string | null; + last_start_timestamp: string | null; + logs: string[]; + last_run: AppRunInfo | null; +}; + +export type DataApp = { + name: string; + description: string | null; + component_id: string; + configuration_id: string; + data_app_id: string; + project_id: string; + branch_id: string; + config_version: string; + state: string; + type: string; + deployment_url: string | null; + auto_suspend_after_seconds: number | null; + repo_url: string | null; + configuration: Record; + folder: string; + deployment_info: DeploymentInfo | null; + drafts: DataAppSummary[]; + drafts_unavailable: number; + links: Link[]; +}; + +// Zod schemas for the modify_* / deploy / delete tool inputs that need enums/literals. +export const modeSchema = z.enum(['dev', 'production']); +export const actionSchema = z.enum(['deploy', 'stop']); +export const authenticationTypeSchema = z.enum(AUTHENTICATION_TYPES); diff --git a/src/tools/data_apps/tools.ts b/src/tools/data_apps/tools.ts new file mode 100644 index 000000000..f6f152093 --- /dev/null +++ b/src/tools/data_apps/tools.ts @@ -0,0 +1,846 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { createKeboolaClients, createLinksManager } from '@/clients/keboola'; +import type { Config } from '@/config'; +import { DATA_APP_COMPONENT_ID } from '@/constants'; +import type { Link } from '@/links'; +import { logger } from '@/logger'; +import { registerTool } from '@/mcp/tool'; +import { toonSerializeCompact } from '@/serialize'; +import { + createDataScience, + fetchDataApp, + fetchDataAppDetailsTask, + fetchLatestRun, + fetchLogs, + hasFeature, + resolveWorkspace, + storageHelpers, + withDeploymentInfo, +} from './client'; +import { applyFolderMetadata, setCfgCreationMetadata, setCfgUpdateMetadata } from './metadata'; +import { actionSchema, authenticationTypeSchema, type DataApp, modeSchema } from './model'; +import { + asRecord, + buildAuthenticatedCloneUrl, + buildDataAppConfig, + DATA_APPS_STORAGE_WORKSPACE_FEATURE, + DEFAULT_DRAFT_BRANCH, + encryptConfig, + folderFieldDescription, + getAuthorization, + getSecrets, + isDraftConfig, + MANAGED_GIT_REPO_USERNAME, + responseForState, + SECRET_WORKSPACE_ID, + summaryFromApiResponse, + summaryFromDataApp, + updateExistingCodeDataAppConfig, + updateExistingDataAppConfig, + usesBasicAuthentication, + validateDataAppStorage, +} from './utils'; + +// Ported from tools/data_apps.py. Data App tools are blocked outside the main branch +// centrally via tool filtering; this module registers them normally. + +// =========================================================================== +// Tool registration +// =========================================================================== + +export const registerDataAppTools = (server: McpServer, config: Config): void => { + const makeContext = () => { + const clients = createKeboolaClients(config); + const ds = createDataScience(clients, config); + const helpers = storageHelpers(clients); + return { clients, ds, helpers }; + }; + + registerTool(server, { + name: 'modify_streamlit_data_app', + title: 'Modify Streamlit data app', + description: `Creates or updates a Streamlit data app. + +Considerations: +- The \`source_code\` parameter must be a complete and runnable Streamlit app. It must include a placeholder \`{QUERY_DATA_FUNCTION}\` where a \`query_data\` function will be injected. This function queries the workspace to get data, it accepts a string of SQL query following current sql dialect and returns a pandas DataFrame with the results from the workspace. +- Write SQL queries so they are compatible with the current workspace backend, you can ensure this by using the \`query_data\` tool to inspect the data in the workspace before using it in the data app. +- If you're updating an existing data app, provide the \`configuration_id\` parameter and the \`change_description\` parameter. To keep existing data app values during an update, leave them as empty strings, lists, or None appropriately based on the parameter type. +- After creating or updating a data app with this tool, ALWAYS call \`deploy_data_app(action="deploy", configuration_id=...)\` to start a new app or restart an existing app so changes take effect. Without this step, a newly created app will not start, and an existing app will keep running the previous deployment without the latest changes. +- New apps use the HTTP basic authentication by default for security unless explicitly specified otherwise; when updating, set \`authentication_type\` to \`default\` to keep the existing authentication type configuration (including OIDC setups) unless explicitly specified otherwise. + +SQL & DATA TYPE RULES: +- Use delimited identifiers for the current SQL dialect for all column names and aliases in SQL. Match the exact identifier case used in SQL when referencing columns in Python code. +- \`query_data\` RETURNS ALL COLUMNS AS STRINGS regardless of SQL CAST. Always convert types in Python after loading: \`df["col"] = pd.to_numeric(df["col"], errors="coerce").fillna(0)\` and \`df["date"] = pd.to_datetime(df["date"], errors="coerce")\`.`, + annotations: { destructiveHint: true }, + inputSchema: { + name: z.string().describe('Name of the data app (max ~50 chars to fit DNS label limit).'), + description: z.string().describe('Description of the data app.'), + source_code: z.string().describe('Complete Python/Streamlit source code for the data app.'), + packages: z + .array(z.string()) + .describe( + 'Python packages used in the source code that will be installed by `pip install` ' + + 'into the environment before the code runs. For example: ["pandas", "requests~=2.32"].', + ), + authentication_type: authenticationTypeSchema.describe( + 'Authentication type, "no-auth" removes authentication completely, "basic-auth" sets the data ' + + 'app to be secured using the HTTP basic authentication, and "default" keeps the existing ' + + 'authentication type when updating.', + ), + configuration_id: z + .string() + .default('') + .describe( + 'The ID of existing data app configuration when updating, otherwise empty string.', + ), + change_description: z + .string() + .default('') + .describe( + 'The description of the change when updating (e.g. "Update Code"), otherwise empty string.', + ), + folder: z.string().nullish().describe(folderFieldDescription('data app', 'data apps')), + }, + serializer: toonSerializeCompact, + handler: async (args) => { + const { clients, ds, helpers } = makeContext(); + const linksManager = await createLinksManager(config, clients); + const projectId = String( + ((await clients.storage.tokens.verify()) as { owner: { id: string | number } }).owner.id, + ); + const ws = await resolveWorkspace(config, helpers); + const secrets = getSecrets(ws.workspaceId, ws.branchId); + + if (args.configuration_id) { + const dataAppPre = await fetchDataApp(ds, helpers, args.configuration_id); + let updatedConfig = updateExistingDataAppConfig( + dataAppPre.configuration, + args.name, + args.source_code, + args.packages, + args.authentication_type, + secrets, + ws.sqlDialect, + ); + updatedConfig = await encryptConfig(config, updatedConfig, { + projectId, + componentId: DATA_APP_COMPONENT_ID, + }); + const updateResp = await helpers.configurationUpdate({ + configurationId: args.configuration_id, + configuration: updatedConfig, + changeDescription: args.change_description || 'Change Data App', + updatedName: args.name || dataAppPre.name, + updatedDescription: args.description || dataAppPre.description || undefined, + }); + // --- write committed past this point; response building is best-effort --- + const newVersion = String(updateResp.version ?? ''); + try { + if (/^\d+$/.test(newVersion)) { + await setCfgUpdateMetadata(helpers, args.configuration_id, Number(newVersion)); + } + const folderHint = await applyFolderMetadata( + helpers, + args.configuration_id, + args.folder, + 'data apps', + 'modify_streamlit_data_app', + ); + const dataApp = await fetchDataApp(ds, helpers, args.configuration_id); + const links = linksManager.getDataAppLinks( + dataApp.configuration_id, + args.name, + dataApp.deployment_url ?? undefined, + usesBasicAuthentication(asRecord(dataApp.configuration.authorization)), + ); + return { + response: responseForState(dataApp.state), + change_summary: folderHint, + data_app: summaryFromDataApp(dataApp), + links, + }; + } catch (error) { + logger.error( + { err: error }, + `Data app configuration ${args.configuration_id} was updated (version ${newVersion || '?'}) ` + + `but building the response failed; returning a partial success.`, + ); + const summary = summaryFromDataApp(dataAppPre); + summary.config_version = newVersion || summary.config_version; + let links: Link[] = []; + try { + links = linksManager.getDataAppLinks( + args.configuration_id, + args.name || dataAppPre.name, + dataAppPre.deployment_url ?? undefined, + usesBasicAuthentication(asRecord(dataAppPre.configuration.authorization)), + ); + } catch { + links = []; + } + return { + response: responseForState(dataAppPre.state), + change_summary: + `The configuration WAS updated (version ${newVersion || 'unknown'}), but loading the full app ` + + `details failed, so this response is partial. Do NOT retry the update -- the change is already ` + + `applied. Call deploy_data_app to apply it to the running app.`, + data_app: summary, + links, + }; + } + } + + // Create new data app. + let createCfg = buildDataAppConfig( + args.name, + args.source_code, + args.packages, + args.authentication_type, + secrets, + ws.sqlDialect, + ); + createCfg = await encryptConfig(config, createCfg, { + projectId, + componentId: DATA_APP_COMPONENT_ID, + }); + const dataAppResp = await ds.createDataApp({ + name: args.name, + description: args.description, + config: createCfg, + branchId: config.branchId ?? null, + appType: 'streamlit', + useManagedGitRepo: false, + }); + try { + await setCfgCreationMetadata(helpers, dataAppResp.config_id); + const folderHint = await applyFolderMetadata( + helpers, + dataAppResp.config_id, + args.folder, + 'data apps', + 'modify_streamlit_data_app', + true, + ); + const links = linksManager.getDataAppLinks( + dataAppResp.config_id, + args.name, + dataAppResp.url ?? undefined, + usesBasicAuthentication(asRecord(createCfg.authorization)), + ); + return { + response: 'created', + change_summary: folderHint, + data_app: summaryFromApiResponse(dataAppResp), + links, + }; + } catch (error) { + logger.error( + { err: error }, + `Data app ${dataAppResp.id} was created (configuration ${dataAppResp.config_id}) but building ` + + `the response failed; returning a partial success.`, + ); + let links: Link[] = []; + try { + links = linksManager.getDataAppLinks( + dataAppResp.config_id, + args.name, + dataAppResp.url ?? undefined, + usesBasicAuthentication(asRecord(createCfg.authorization)), + ); + } catch { + links = []; + } + return { + response: 'created', + change_summary: + `The data app WAS created (configuration ${dataAppResp.config_id}), but building the full response ` + + `failed, so this response is partial. Do NOT retry creation -- it would create a duplicate. ` + + `Call deploy_data_app to start the app.`, + data_app: summaryFromApiResponse(dataAppResp), + links, + }; + } + }, + }); + + registerTool(server, { + name: 'modify_python_js_data_app', + title: 'Modify python-js data app', + description: `Creates or updates a python-js data app. + +Two-app project model. Every python-js project has a persistent **prod app** that owns the only managed git repository for the project, and zero or more **drafts** parented to that prod app. A draft is a Storage configuration with \`parameters.dataApp.isDraft=true\` and \`parameters.dataApp.parentConfigurationId=\`; it's an *external-git* app that clones the parent prod's repo at a pinned branch on every deploy. Drafts are surfaced in the Keboola UI under their parent prod app. Use \`deploy_data_app(mode='dev')\` to deploy a draft as a dev version of the data app (hot reload + auto-auth for iframe preview); use \`delete_python_js_data_app_draft\` to tear a draft down after its branch has been promoted. + +**MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, branch-delete — is yours. MCP gives you authenticated clone URLs and manages configs/deploys; it never invokes git. + +**The draft flow is mandatory — never edit prod source directly.** Every source-code change goes through a draft branch that the user previews and explicitly approves first. NEVER push directly to \`main\`: \`main\` only ever advances by merging an approved draft branch, and only after the user has approved that draft's preview. + +## Argument rules +- \`parent_configuration_id\` is **create-only**. Rejected on update. +- \`branch\` is **create-only** and only valid when \`parent_configuration_id\` is set. Defaults to \`'init'\`. Must not be \`'main'\`. Rejected on prod create and on update. +- \`slug\` is required on create and immutable after. +- The **update path** (passing \`configuration_id\`) is for changing \`name\`, \`description\`, \`authentication_type\`, \`auto_suspend_after_seconds\`, \`storage\` on either a prod app or a draft. Source code changes go through the git flow above, not this tool. + +## Authentication +New apps default to HTTP basic authentication for safety. Pass \`authentication_type='no-auth'\` to expose publicly. On update, \`authentication_type='default'\` preserves the existing \`authorization\` block (including OIDC setups configured outside the MCP); \`'basic-auth'\` / \`'no-auth'\` overwrite it. + +## Slug constraint +Must be DNS-label-safe (lowercase letters, digits, hyphens, ≤63 chars). For drafts, append a short suffix (e.g. \`-draft-abc123\`) to keep slugs unique across the prod and its drafts.`, + annotations: { destructiveHint: true }, + inputSchema: { + name: z.string().describe('Name of the data app (max ~50 chars to fit DNS label limit).'), + description: z.string().describe('Description of the data app.'), + configuration_id: z + .string() + .default('') + .describe( + 'The ID of existing data app configuration when updating, otherwise empty string.', + ), + change_description: z + .string() + .default('') + .describe( + 'The description of the change when updating (e.g. "Bump image"), otherwise empty string.', + ), + slug: z + .string() + .nullish() + .describe( + 'URL-safe slug for the data app (used as a subdomain). Required when creating; immutable after.', + ), + parent_configuration_id: z + .string() + .nullish() + .describe( + 'Storage configuration ID of the prod python-js data app this draft will iterate against. ' + + 'When set on create, the new app is created as a **draft**: no managed repo is provisioned ' + + "for it; instead its `parameters.dataApp.git` block is populated to point at the prod app's " + + 'managed repo, with a freshly-minted prod-app HTTPS token and the chosen draft branch. ' + + 'Leave None on create to make a **prod app** (which gets its own managed repo). Rejected on update.', + ), + branch: z + .string() + .nullish() + .describe( + 'Draft branch to pin the new draft to. Only valid on the draft create path ' + + '(when `parent_configuration_id` is set). Defaults to `init` when unset. Must not be `main` ' + + '(reserved for the prod app). Rejected on prod create and on update.', + ), + authentication_type: authenticationTypeSchema + .default('default') + .describe( + 'Authentication type. "no-auth" removes authentication completely, "basic-auth" secures the ' + + 'data app via HTTP basic authentication, and "default" means: on create, apply basic auth ' + + '(safe default for new apps); on update, keep the existing authentication configuration ' + + '(including OIDC setups configured outside the MCP).', + ), + auto_suspend_after_seconds: z + .number() + .int() + .default(900) + .describe('Number of seconds after which the running data app is automatically suspended.'), + storage: z + .record(z.string(), z.any()) + .nullish() + .describe( + 'Complete storage configuration for the data app (input/output table mappings). ' + + 'Replaces the ENTIRE storage block when updating an existing app. Leave unset (None) to ' + + 'preserve the existing storage configuration; pass an empty dict to explicitly clear it.', + ), + folder: z.string().nullish().describe(folderFieldDescription('data app', 'data apps')), + }, + serializer: toonSerializeCompact, + handler: async (args) => { + if (args.configuration_id) { + if (args.slug) throw new Error('slug cannot be changed after the data app is created.'); + if (args.parent_configuration_id) { + throw new Error( + 'parent_configuration_id is only valid when creating a draft (no configuration_id).', + ); + } + if (args.branch) { + throw new Error('branch is only valid when creating a draft (no configuration_id).'); + } + } else { + if (!args.slug) { + throw new Error('slug is required when creating a python-js data app.'); + } + if (args.branch != null && !args.parent_configuration_id) { + throw new Error( + 'branch is only valid on the draft create path (pair it with parent_configuration_id).', + ); + } + } + + const { clients, ds, helpers } = makeContext(); + const linksManager = await createLinksManager(config, clients); + + const validatedStorage = validateDataAppStorage(args.storage); + + const hasStorageWorkspace = await hasFeature(clients, DATA_APPS_STORAGE_WORKSPACE_FEATURE); + let legacySecrets: Record | null = null; + if (!hasStorageWorkspace) { + const ws = await resolveWorkspace(config, helpers); + legacySecrets = { [SECRET_WORKSPACE_ID]: ws.workspaceId }; + } + + if (args.configuration_id) { + let dataApp = await fetchDataApp(ds, helpers, args.configuration_id); + const updatedConfig = updateExistingCodeDataAppConfig( + dataApp.configuration, + args.auto_suspend_after_seconds, + args.authentication_type, + legacySecrets, + validatedStorage, + ); + await helpers.configurationUpdate({ + configurationId: args.configuration_id, + configuration: updatedConfig, + changeDescription: args.change_description || 'Update python-js data app', + updatedName: args.name || dataApp.name, + updatedDescription: args.description || dataApp.description || undefined, + }); + dataApp = await fetchDataApp(ds, helpers, args.configuration_id); + await setCfgUpdateMetadata(helpers, args.configuration_id, Number(dataApp.config_version)); + const folderHint = await applyFolderMetadata( + helpers, + args.configuration_id, + args.folder, + 'data apps', + 'modify_python_js_data_app', + ); + const repoUrl = dataApp.repo_url; + const links = linksManager.getDataAppLinks( + dataApp.configuration_id, + args.name || dataApp.name, + dataApp.deployment_url ?? undefined, + usesBasicAuthentication(asRecord(dataApp.configuration.authorization)), + ); + const summary = summaryFromDataApp(dataApp); + summary.repo_url = repoUrl; + return { + response: responseForState(dataApp.state), + change_summary: folderHint, + data_app: summary, + repo_url: repoUrl, + links, + }; + } + + // Create new python-js data app (prod or draft). + const slug = args.slug!; + const usesBasicAuth = + args.authentication_type === 'basic-auth' || args.authentication_type === 'default'; + const authorizationModel = getAuthorization(usesBasicAuth); + + let gitCloneUrl: string | null = null; + let draftBranch: string | null = null; + let gitBlock: Record | null = null; + + if (args.parent_configuration_id) { + const parent = await fetchDataApp(ds, helpers, args.parent_configuration_id); + if (parent.type !== 'python-js') { + throw new Error( + `parent_configuration_id "${args.parent_configuration_id}" is type "${parent.type}", but only ` + + `python-js prod apps can parent a draft.`, + ); + } + if (isDraftConfig(parent.configuration)) { + throw new Error( + `parent_configuration_id "${args.parent_configuration_id}" is itself a python-js **draft**, ` + + "not a prod app. Drafts iterate against the prod app's repo and cannot parent another " + + "draft — pass the prod app's configuration_id (a draft's parentConfigurationId points to it).", + ); + } + if (!parent.repo_url) { + throw new Error( + `Parent python-js data app "${args.parent_configuration_id}" has no managed git repo URL. ` + + 'This indicates a platform-side bug — retry or contact support.', + ); + } + draftBranch = (args.branch || DEFAULT_DRAFT_BRANCH).trim(); + if (!draftBranch || /\s/.test(draftBranch)) { + throw new Error(`branch "${args.branch}" is not a valid git branch name.`); + } + if (draftBranch === 'main') { + throw new Error( + 'branch "main" is reserved for the prod app — pick a different draft branch.', + ); + } + const cred = await ds.createAppGitCredential(parent.data_app_id); + if (!cred.secret) { + throw new Error( + `Parent data app ${parent.data_app_id} credentials endpoint returned no \`secret\` for an ` + + `http_token credential. This indicates a platform-side bug — retry or contact support.`, + ); + } + gitBlock = { + repository: parent.repo_url, + username: MANAGED_GIT_REPO_USERNAME, + '#password': cred.secret, + branch: draftBranch, + }; + gitCloneUrl = buildAuthenticatedCloneUrl(parent.repo_url, cred.secret); + } + + const dataAppBlock: Record = { slug }; + if (legacySecrets) dataAppBlock.secrets = legacySecrets; + if (gitBlock) dataAppBlock.git = gitBlock; + if (args.parent_configuration_id != null) { + dataAppBlock.isDraft = true; + dataAppBlock.parentConfigurationId = args.parent_configuration_id; + } + let configPayload: Record = { + parameters: { + autoSuspendAfterSeconds: args.auto_suspend_after_seconds, + dataApp: dataAppBlock, + }, + authorization: authorizationModel, + }; + if (hasStorageWorkspace) { + configPayload.runtime = { workspace: { enabled: true } }; + } + if (validatedStorage && Object.keys(validatedStorage).length > 0) { + configPayload.storage = validatedStorage; + } + + if (gitBlock !== null) { + const projectId = String( + ((await clients.storage.tokens.verify()) as { owner: { id: string | number } }).owner.id, + ); + configPayload = await encryptConfig(config, configPayload, { + projectId, + componentId: DATA_APP_COMPONENT_ID, + }); + } + + const dataAppResp = await ds.createDataApp({ + name: args.name, + description: args.description, + config: configPayload, + branchId: config.branchId ?? null, + appType: 'python-js', + useManagedGitRepo: args.parent_configuration_id == null, + }); + + let repoUrl: string; + if (args.parent_configuration_id) { + repoUrl = gitBlock!.repository as string; + } else { + const repoResp = await ds.getAppGitRepo(dataAppResp.id); + if (repoResp.https_url == null) { + throw new Error( + `Data app ${dataAppResp.id} reports no HTTPS clone URL despite having a managed git repo. ` + + 'This indicates a platform-side bug — retry or contact support.', + ); + } + repoUrl = repoResp.https_url; + } + await setCfgCreationMetadata(helpers, dataAppResp.config_id); + const folderHint = await applyFolderMetadata( + helpers, + dataAppResp.config_id, + args.folder, + 'data apps', + 'modify_python_js_data_app', + true, + ); + const links = linksManager.getDataAppLinks( + dataAppResp.config_id, + args.name, + dataAppResp.url ?? undefined, + usesBasicAuth, + ); + const summary = summaryFromApiResponse(dataAppResp); + summary.repo_url = repoUrl; + return { + response: 'created', + change_summary: folderHint, + data_app: summary, + repo_url: repoUrl, + git_clone_url: gitCloneUrl, + branch: draftBranch, + links, + }; + }, + }); + + registerTool(server, { + name: 'create_python_js_data_app_git_credential', + title: 'Create python-js data app git credential', + description: `Mints a one-time HTTPS token on a python-js **prod** data app so the caller can clone, pull, and push to the app's managed git repo over HTTPS. + +**Always call against the prod app's configuration_id** — drafts have no managed repo of their own, so calling this on a draft fails. The prod app is the canonical repo owner; drafts iterate against branches of that same repo. + +**MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, branch-delete — is yours. This tool only mints credentials. + +Returns a ready-to-use \`git_clone_url\` of the form \`https://kai:@/.git\` plus the raw \`secret\`. The token is returned **only** at creation — the platform cannot return it again on any subsequent read. Stash the URL (or the secret) somewhere the LLM can reuse for the rest of the session. + +## Constraints +- Only python-js prod data apps have a managed git repo. Streamlit apps reject the call with a clear error. +- Permissions are always \`readWrite\`.`, + annotations: { destructiveHint: false }, + inputSchema: { + configuration_id: z.string().describe('Storage configuration ID of the python-js data app.'), + }, + serializer: toonSerializeCompact, + handler: async ({ configuration_id }) => { + const { clients, ds, helpers } = makeContext(); + const linksManager = await createLinksManager(config, clients); + + const dataApp = await fetchDataApp(ds, helpers, configuration_id); + if (dataApp.type !== 'python-js') { + throw new Error( + `create_python_js_data_app_git_credential only supports python-js data apps, but configuration ` + + `"${configuration_id}" is type "${dataApp.type}".`, + ); + } + if (isDraftConfig(dataApp.configuration)) { + const dataAppBlock = asRecord(asRecord(dataApp.configuration.parameters).dataApp); + const parentCfgId = dataAppBlock.parentConfigurationId; + const hint = + typeof parentCfgId === 'string' ? ` (parentConfigurationId="${parentCfgId}")` : ''; + throw new Error( + `Configuration "${configuration_id}" is a python-js **draft**, which has no managed git repo ` + + `of its own. Mint credentials against the parent prod app instead${hint}.`, + ); + } + + const repoResp = await ds.getAppGitRepo(dataApp.data_app_id); + if (repoResp.https_url == null) { + throw new Error( + `Data app ${dataApp.data_app_id} reports no HTTPS clone URL despite being a python-js managed-repo ` + + `app. This indicates a platform-side bug — retry or contact support.`, + ); + } + + const credentialResp = await ds.createAppGitCredential(dataApp.data_app_id); + if (!credentialResp.secret) { + throw new Error( + `Data app ${dataApp.data_app_id} credentials endpoint returned no \`secret\` for an http_token ` + + `credential. This indicates a platform-side bug — retry or contact support.`, + ); + } + + const gitCloneUrl = buildAuthenticatedCloneUrl(repoResp.https_url, credentialResp.secret); + const links = linksManager.getDataAppLinks( + dataApp.configuration_id, + dataApp.name, + dataApp.deployment_url ?? undefined, + false, + ); + return { + response: 'created', + configuration_id: dataApp.configuration_id, + data_app_id: dataApp.data_app_id, + credential_id: credentialResp.id, + git_clone_url: gitCloneUrl, + secret: credentialResp.secret, + permissions: credentialResp.permissions, + links, + }; + }, + }); + + registerTool(server, { + name: 'get_data_apps', + title: 'Get data apps', + description: `Lists summaries of data apps in the project given the limit and offset or gets details of a data apps by providing their configuration IDs. + +WHEN NOT TO USE: +- Do NOT list all data apps just to find one by name. Use \`search\` with item_types=["data-app"] instead. +- Only list all data apps when you need a complete inventory. + +Considerations: +- If configuration_ids are provided, the tool will return details of the data apps by their configuration IDs. +- If no configuration_ids are provided, the tool will list all data apps in the project given the limit and offset. +- Data App detail contains configuration, metadata, source code, links, and deployment info along with the latest data app logs to investigate in-app errors. The logs may be updated after opening the data app URL. +- \`deployment_info.last_run\` carries the outcome of the most recent deployment attempt. For an app that fails to start, check its \`failure_reason\`/\`failure_message\` FIRST — they cover setup-phase failures (e.g. invalid secrets, git clone errors, failing setup scripts) that happen before the container starts and therefore never appear in the regular logs. +- \`repo_url\` (managed git repo URL for python-js apps) is ONLY populated on the detail path (when \`configuration_ids\` is provided). The inventory list always returns \`repo_url=None\`, even for python-js apps with a managed repo — to retrieve the URL, call this tool again with the target \`configuration_ids\`. +- When called with \`configuration_ids=[]\` for a python-js **prod** app, the response includes a \`drafts: [...]\` array of every draft (configs with \`isDraft=true\` and \`parentConfigurationId == \`) currently in the project. Drafts in trash are not included. The array is empty for drafts themselves and for Streamlit apps.`, + annotations: { readOnlyHint: true }, + inputSchema: { + configuration_ids: z + .array(z.string()) + .default([]) + .describe('The IDs of the data app configurations.'), + limit: z.number().int().default(100).describe('The limit of the data apps to fetch.'), + offset: z.number().int().default(0).describe('The offset of the data apps to fetch.'), + }, + serializer: toonSerializeCompact, + handler: async ({ configuration_ids, limit, offset }) => { + const { clients, ds, helpers } = makeContext(); + const linksManager = await createLinksManager(config, clients); + + if (configuration_ids.length > 0) { + const details = await Promise.all( + configuration_ids.map((id) => fetchDataAppDetailsTask(ds, helpers, linksManager, id)), + ); + const found = details.filter((d): d is DataApp => typeof d !== 'string'); + const notFound = details.filter((d): d is string => typeof d === 'string'); + if (notFound.length > 0) { + logger.error(`Could not find Data Apps Configurations for IDs: ${notFound.join(', ')}`); + } + return { data_apps: found }; + } + + let dataApps = await ds.listDataApps(limit, offset); + dataApps = dataApps.filter((app) => app.component_id === DATA_APP_COMPONENT_ID); + return { + data_apps: dataApps.map(summaryFromApiResponse), + links: [linksManager.getDataAppDashboardLink()], + }; + }, + }); + + registerTool(server, { + name: 'deploy_data_app', + title: 'Deploy data app', + description: `Deploys/redeploys a data app or stops a running data app in the Keboola environment asynchronously, given the action and the configuration ID. + +**MCP never runs git on your behalf.** All git work — clone, branch, commit, push, merge, branch-delete — is yours. This tool only triggers deploys against existing git state. + +## Mode (python-js apps) +- \`mode='dev'\` deploys the target as a **dev version of the data app** — the runtime uses a development \`setup.sh\` (hot reload) and the data-app proxy enables an auto-auth path so an iframe preview can render without a manual login. Only meaningful on **draft** configs (python-js apps with \`isDraft=true\`). +- For prod redeploys (including after merging a draft's branch into \`main\`), use no \`mode\` — the prod app picks up the current \`main\`. +- The branch a draft deploys from is pinned in \`parameters.dataApp.git.branch\` at create time; there is no deploy-time override. +- python-js apps do NOT fetch a Storage \`configVersion\` for deployment (their source lives in git, not in the Storage configuration); this is handled automatically. + +## Streamlit apps +Streamlit apps have no managed git repo, so \`mode\` has no effect on the deployed app. \`mode=None\` is the expected call shape. + +## General considerations +- Redeploying a data app takes some time, and the app may temporarily report status "stopped" during the restart. +- After deployment, the deployment info includes the app URL and the latest logs to help diagnose in-app errors.`, + annotations: { destructiveHint: false }, + inputSchema: { + action: actionSchema.describe('The action to perform.'), + configuration_id: z.string().describe('The ID of the data app configuration.'), + mode: modeSchema + .nullish() + .describe( + 'Deployment mode. Set to "dev" to deploy a python-js draft as a **dev version of the data ' + + 'app** — the runtime uses a development `setup.sh` (hot reload), and the data-app proxy ' + + 'enables an auto-auth path so an iframe preview can render without a manual login. ' + + 'Only meaningful on **draft** configs (python-js apps with `isDraft=true`). Leave None ' + + '(default) for prod redeploys and for Streamlit apps.', + ), + }, + serializer: toonSerializeCompact, + handler: async ({ action, configuration_id, mode }) => { + const { clients, ds, helpers } = makeContext(); + const linksManager = await createLinksManager(config, clients); + + if (action === 'deploy') { + let dataApp = await fetchDataApp(ds, helpers, configuration_id); + if (dataApp.state === 'stopping') { + throw new Error('Data app is currently "stopping", could not be started at the moment.'); + } + let configVersionArg: string | null = null; + if (dataApp.type !== 'python-js') { + const version = await helpers.configurationVersionLatest(dataApp.configuration_id); + configVersionArg = String(version); + } + await ds.deployDataApp(dataApp.data_app_id, configVersionArg, mode ?? null); + dataApp = await fetchDataApp(ds, helpers, configuration_id); + dataApp = withDeploymentInfo( + dataApp, + await fetchLogs(ds, dataApp.data_app_id), + await fetchLatestRun(ds, dataApp.data_app_id), + ); + const links = linksManager.getDataAppLinks( + dataApp.configuration_id, + dataApp.name, + dataApp.deployment_url ?? undefined, + usesBasicAuthentication(asRecord(dataApp.configuration.authorization)), + ); + return { state: dataApp.state, deployment_info: dataApp.deployment_info, links }; + } + + // action === 'stop' + let dataApp = await fetchDataApp(ds, helpers, configuration_id); + if (dataApp.state === 'starting' || dataApp.state === 'restarting') { + throw new Error('Data app is currently "starting", could not be stopped at the moment.'); + } + await ds.suspendDataApp(dataApp.data_app_id); + dataApp = await fetchDataApp(ds, helpers, configuration_id); + const links = linksManager.getDataAppLinks( + dataApp.configuration_id, + dataApp.name, + undefined, + usesBasicAuthentication(asRecord(dataApp.configuration.authorization)), + ); + return { state: dataApp.state, deployment_info: null, links }; + }, + }); + + registerTool(server, { + name: 'delete_python_js_data_app_draft', + title: 'Delete python-js data app draft', + description: `Deletes a python-js DRAFT data app — both the data-app instance (DSAPI) and its Storage configuration. + +**MCP never runs git on your behalf.** Deleting the feature branch on the remote is your job; this tool only tears down the draft config and its data-app instance. + +WHEN TO CALL: at the end of a promote-to-prod sequence, after you have merged the draft's branch into \`main\`, pushed, deleted the feature branch from the remote, and redeployed the prod app. The Keboola UI lists drafts under their parent prod app; once you call this tool, the draft disappears from that list. + +WHAT THIS TOOL REFUSES: + - prod apps (no \`isDraft\` flag) — protects against accidental prod deletion; + - Streamlit apps — they have no draft concept. + +WHAT THIS TOOL DOES NOT DO: + - Run git. Deleting the feature branch on the remote is your job. + - Revoke the prod-side git credential minted when the draft was created. + +After a successful call, pivot back to the parent prod app (its configuration_id is returned in the response) or to \`get_data_apps\` for further work.`, + annotations: { destructiveHint: true }, + inputSchema: { + configuration_id: z + .string() + .describe('Storage configuration ID of the python-js draft data app to delete.'), + }, + serializer: toonSerializeCompact, + handler: async ({ configuration_id }) => { + const { clients, ds, helpers } = makeContext(); + const linksManager = await createLinksManager(config, clients); + + const dataApp = await fetchDataApp(ds, helpers, configuration_id); + if (dataApp.type !== 'python-js') { + throw new Error( + `delete_python_js_data_app_draft only supports python-js data apps, but configuration ` + + `"${configuration_id}" is type "${dataApp.type}".`, + ); + } + if (!isDraftConfig(dataApp.configuration)) { + throw new Error( + `Configuration "${configuration_id}" is a python-js **prod** app, not a draft ` + + '(parameters.dataApp.isDraft is not true). This tool only deletes drafts — ' + + 'prod apps must be deleted from the Keboola UI.', + ); + } + + const dataAppBlock = asRecord(asRecord(dataApp.configuration.parameters).dataApp); + const parentCfgId = dataAppBlock.parentConfigurationId; + const parentConfigurationId = typeof parentCfgId === 'string' ? parentCfgId : null; + + await ds.deleteDataApp(dataApp.data_app_id); + + const links = linksManager.getDataAppLinks( + parentConfigurationId ?? configuration_id, + parentConfigurationId ? 'parent prod app' : dataApp.name, + undefined, + false, + ); + return { + response: 'deleted', + configuration_id, + data_app_id: dataApp.data_app_id, + parent_configuration_id: parentConfigurationId, + links, + }; + }, + }); + + // debug, not info: createServer() runs per HTTP request, so this fires on every request. + logger.debug('Data app tools initialized.'); +}; diff --git a/src/tools/data_apps/utils.ts b/src/tools/data_apps/utils.ts new file mode 100644 index 000000000..e5c6c7b8b --- /dev/null +++ b/src/tools/data_apps/utils.ts @@ -0,0 +1,404 @@ +import { readFileSync } from 'node:fs'; + +import { createRawClient } from '@/clients/raw'; +import { deriveServiceUrls } from '@/clients/urls'; +import type { Config } from '@/config'; +import { DATA_APP_COMPONENT_ID } from '@/constants'; +import { resourcePath } from '@/resource-path'; +import type { AuthenticationType, DataApp, DataAppResponse, DataAppSummary } from './model'; + +// Pure helpers + config builders ported from tools/data_apps.py. No I/O except the +// resource-code loader and the encryption client (a one-off raw POST that the typed +// api-client does not cover). + +// MCP-only metadata keys (port of config.py MetadataField; not yet in constants.ts). +export const CREATED_BY_MCP = 'KBC.MCP.createdBy'; +export const UPDATED_BY_MCP_PREFIX = 'KBC.MCP.updatedBy.version.'; + +// --- resource code templates ------------------------------------------------- +// Copied from src/keboola_mcp_server/resources/data_app/* into src/resources/data_app/*. +// Loaded once at module init via fs, resolving relative to this module's URL so it works +// when running from source (vitest / tsx). Mirrors the Python `importlib.resources` read. +const readResource = (name: string): string => + readFileSync(resourcePath('data_app', name), { encoding: 'utf-8' }); + +export const QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE = readResource('qsapi_query_data_code.py'); +export const STORAGE_QUERY_DATA_FUNCTION_CODE = readResource('sapi_query_data_code.py'); + +const DEFAULT_STREAMLIT_THEME = + '[theme]\nfont = "sans serif"\ntextColor = "#222529"\nbackgroundColor = "#FFFFFF"\n' + + 'secondaryBackgroundColor = "#E6F2FF"\nprimaryColor = "#1F8FFF"'; +const DEFAULT_PACKAGES = ['pandas', 'httpx']; + +export const MANAGED_GIT_REPO_USERNAME = 'kai'; +export const DEFAULT_DRAFT_BRANCH = 'init'; + +export const APP_RUN_LOG_LINES = 30; +export const APP_RUN_MESSAGE_LIMIT = 3000; + +const INJECTED_BLOCK_RE = + /(?[\s\S]*?)#\s###\sINJECTED_CODE\s####[\s\S]*?#\s###\sEND_OF_INJECTED_CODE\s####(?[\s\S]*)/; + +export const SECRET_WORKSPACE_ID = 'WORKSPACE_ID'; +export const SECRET_BRANCH_ID = 'BRANCH_ID'; + +export const DATA_APPS_STORAGE_WORKSPACE_FEATURE = 'data-apps-storage-workspace'; + +const MAX_DNS_LABEL_LENGTH = 63; + +// --------------------------------------------------------------------------- +// Encryption client (port of clients/encryption.py): POST encrypt with id params. +// KEEP RAW: the typed api-client does not expose the encryption service. +// --------------------------------------------------------------------------- +export const encryptConfig = async ( + config: Config, + body: Record, + params: { projectId: string; componentId: string }, +): Promise> => { + const urls = deriveServiceUrls(config.storageApiUrl ?? ''); + const token = config.bearerToken ? `Bearer ${config.bearerToken}` : config.storageToken; + const enc = createRawClient({ baseUrl: urls.encryption, token }); + return enc.post>('encrypt', { + body, + params: { componentId: params.componentId, projectId: params.projectId }, + }); +}; + +// --- pure helpers (ports of the module-level functions in data_apps.py) ----- + +export const getAuthorization = (authWithPassword: boolean): Record => { + if (authWithPassword) { + return { + app_proxy: { + auth_providers: [{ id: 'simpleAuth', type: 'password' }], + auth_rules: [{ type: 'pathPrefix', value: '/', auth_required: true, auth: ['simpleAuth'] }], + }, + }; + } + return { + app_proxy: { + auth_providers: [], + auth_rules: [{ type: 'pathPrefix', value: '/', auth_required: false }], + }, + }; +}; + +export const usesBasicAuthentication = (authorization: Record): boolean => { + try { + const rules = ((authorization.app_proxy as { auth_rules?: Record[] }) + .auth_rules ?? []) as Record[]; + return rules.some( + (rule) => + rule.auth_required === true && + Array.isArray(rule.auth) && + (rule.auth as unknown[]).includes('simpleAuth'), + ); + } catch { + return false; + } +}; + +export class DataAppSlugTooLongError extends Error {} + +export const getDataAppSlug = (name: string): string => { + const slug = name + .trim() + .toLowerCase() + .replaceAll(' ', '-') + .replace(/[^a-z0-9-]/g, ''); + if (slug.length > MAX_DNS_LABEL_LENGTH) { + throw new DataAppSlugTooLongError( + `Data app name "${name}" generates a URL slug that is ${slug.length} characters long, ` + + `which exceeds the maximum DNS label length of ${MAX_DNS_LABEL_LENGTH} characters. ` + + `Please use a shorter name (the slug "${slug.slice(0, 20)}..." is too long). ` + + `The name should generate a slug of at most ${MAX_DNS_LABEL_LENGTH} characters after ` + + `converting to lowercase, replacing spaces with hyphens, and removing special characters.`, + ); + } + return slug; +}; + +const getQueryFunctionCode = (sqlDialect: string): string => { + const dialect = sqlDialect.toLowerCase(); + if (dialect === 'snowflake') return QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE; + if (dialect === 'bigquery') return STORAGE_QUERY_DATA_FUNCTION_CODE; + throw new Error(`Unsupported SQL dialect: ${sqlDialect}`); +}; + +const stripInjectedQueryCode = (sourceCode: string): string => { + let out = sourceCode; + for (const snippet of [ + QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE, + STORAGE_QUERY_DATA_FUNCTION_CODE, + ]) { + out = out.split(snippet).join(''); + } + return out; +}; + +const injectQueryToSourceCode = (sourceCode: string, sqlDialect: string): string => { + if (!sourceCode) return ''; + const queryFunctionCode = getQueryFunctionCode(sqlDialect); + if (sourceCode.includes(queryFunctionCode)) return sourceCode; + + let stripped = stripInjectedQueryCode(sourceCode); + if (stripped.includes('{QUERY_DATA_FUNCTION}')) { + return stripped.replaceAll('{QUERY_DATA_FUNCTION}', queryFunctionCode); + } + const match = INJECTED_BLOCK_RE.exec(stripped); + if (match?.groups) { + const before = (match.groups.before ?? '').replace(/\s+$/, ''); + const after = (match.groups.after ?? '').replace(/^\s+/, ''); + return `${before}\n\n${queryFunctionCode}\n\n${after}`; + } + stripped = stripped.replace(/^\s+/, ''); + return `${queryFunctionCode}\n\n${stripped}`; +}; + +export const getSecrets = (workspaceId: string, branchId: string): Record => ({ + [SECRET_WORKSPACE_ID]: workspaceId, + [SECRET_BRANCH_ID]: branchId, +}); + +const sortedUnique = (items: string[]): string[] => + Array.from(new Set(items)).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + +export const buildDataAppConfig = ( + name: string, + sourceCode: string, + packages: string[], + authenticationType: AuthenticationType, + secrets: Record, + sqlDialect: string, +): Record => { + const allPackages = sortedUnique([...packages, ...DEFAULT_PACKAGES]); + const slug = getDataAppSlug(name) || 'Data-App'; + const parameters: Record = { + size: 'tiny', + autoSuspendAfterSeconds: 900, + dataApp: { + slug, + streamlit: { 'config.toml': DEFAULT_STREAMLIT_THEME }, + secrets, + }, + script: [injectQueryToSourceCode(sourceCode, sqlDialect)], + packages: allPackages, + }; + const authorization = getAuthorization( + authenticationType === 'basic-auth' || authenticationType === 'default', + ); + return { parameters, authorization }; +}; + +const deepClone = (value: T): T => JSON.parse(JSON.stringify(value)) as T; + +export const asRecord = (value: unknown): Record => + value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {}; + +export const updateExistingDataAppConfig = ( + existingConfig: Record, + name: string, + sourceCode: string, + packages: string[], + authenticationType: AuthenticationType, + secrets: Record, + sqlDialect: string, +): Record => { + const newConfig = deepClone(existingConfig); + const params = asRecord(newConfig.parameters); + newConfig.parameters = params; + const dataApp = asRecord(params.dataApp); + params.dataApp = dataApp; + + dataApp.slug = getDataAppSlug(name) || dataApp.slug; + if (sourceCode) params.script = [injectQueryToSourceCode(sourceCode, sqlDialect)]; + params.packages = + packages.length > 0 + ? sortedUnique([...packages, ...DEFAULT_PACKAGES]) + : sortedUnique([...((params.packages as string[]) ?? []), ...DEFAULT_PACKAGES]); + + const updatedSecrets = { ...asRecord(dataApp.secrets) }; + for (const [key, value] of Object.entries(secrets)) { + if (!(key in updatedSecrets)) updatedSecrets[key] = value; + } + dataApp.secrets = updatedSecrets; + + if (authenticationType !== 'default') { + newConfig.authorization = getAuthorization(authenticationType === 'basic-auth'); + } + normalizeConfigStorage(newConfig); + return newConfig; +}; + +const pruneEmptyStorageObjects = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(pruneEmptyStorageObjects); + if (value && typeof value === 'object') { + const pruned: Record = {}; + for (const [key, sub] of Object.entries(value)) { + const prunedSub = pruneEmptyStorageObjects(sub); + if (prunedSub && typeof prunedSub === 'object' && !Array.isArray(prunedSub)) { + if (Object.keys(prunedSub).length === 0) continue; + } + pruned[key] = prunedSub; + } + return pruned; + } + return value; +}; + +const normalizeConfigStorage = (config: Record): void => { + if (!('storage' in config)) return; + const storage = config.storage; + const pruned = + storage && typeof storage === 'object' && !Array.isArray(storage) + ? (pruneEmptyStorageObjects(storage) as Record) + : null; + if (pruned && Object.keys(pruned).length > 0) { + config.storage = pruned; + } else { + delete config.storage; + } +}; + +export const validateDataAppStorage = ( + storage: Record | null | undefined, +): Record | null => { + if (storage == null) return null; + // Accept both raw `storage` dict and pre-wrapped {'storage': storage}. + const storageCfg = Object.keys(storage).length > 0 ? asRecord(storage.storage ?? storage) : {}; + // NOTE: the JSON-schema validation (validate_storage_configuration_against_schema) + // is owned by a not-yet-ported validation module; structural unwrap + prune are + // preserved here for parity. See REPORT gap note. + return pruneEmptyStorageObjects(storageCfg) as Record; +}; + +export const updateExistingCodeDataAppConfig = ( + existingConfig: Record, + autoSuspendAfterSeconds: number, + authenticationType: AuthenticationType, + secrets: Record | null, + storage: Record | null, +): Record => { + const newConfig = deepClone(existingConfig); + const params = asRecord(newConfig.parameters); + newConfig.parameters = params; + params.autoSuspendAfterSeconds = autoSuspendAfterSeconds; + if (authenticationType !== 'default') { + newConfig.authorization = getAuthorization(authenticationType === 'basic-auth'); + } + if (secrets && Object.keys(secrets).length > 0) { + const dataApp = asRecord(params.dataApp); + params.dataApp = dataApp; + const updatedSecrets = { ...asRecord(dataApp.secrets) }; + for (const [key, value] of Object.entries(secrets)) { + if (!(key in updatedSecrets)) updatedSecrets[key] = value; + } + dataApp.secrets = updatedSecrets; + } + if (storage !== null) newConfig.storage = storage; + normalizeConfigStorage(newConfig); + return newConfig; +}; + +export const isDraftConfig = (configuration: Record): boolean => { + const parameters = configuration.parameters; + if (!parameters || typeof parameters !== 'object' || Array.isArray(parameters)) return false; + const dataApp = (parameters as Record).dataApp; + if (!dataApp || typeof dataApp !== 'object' || Array.isArray(dataApp)) return false; + return (dataApp as Record).isDraft === true; +}; + +export const buildAuthenticatedCloneUrl = (httpsUrl: string, secret: string): string => { + let parts: URL; + try { + parts = new URL(httpsUrl); + } catch { + throw new Error(`Could not parse HTTPS clone URL: '${httpsUrl}'`); + } + if (!parts.protocol || !parts.host) { + throw new Error(`Could not parse HTTPS clone URL: '${httpsUrl}'`); + } + const host = parts.host; // includes port if present + const scheme = parts.protocol.replace(':', ''); + const netloc = `${MANAGED_GIT_REPO_USERNAME}:${encodeURIComponent(secret)}@${host}`; + return `${scheme}://${netloc}${parts.pathname}${parts.search}${parts.hash}`; +}; + +export const folderFieldDescription = (singular: string, plural: string): string => + `Folder name to organize this ${singular} in the Keboola UI. ` + + `Pass an empty string to remove an existing folder assignment. ` + + `Existing folder names are returned in the response change_summary when no folder is provided ` + + `and there are 20 or more ${plural} in the project. ` + + `If there are 20 or more ${plural}, you should assign one of the existing folders or ` + + `create a new one that clearly reflects the ${singular} purpose.`; + +export const responseForState = (state: string): string => + state === 'running' || state === 'starting' + ? 'updated (redeploy required to apply changes in the running app)' + : 'updated'; + +// --- summary projections (pure) --------------------------------------------- + +export const summaryFromDataApp = (dataApp: DataApp): DataAppSummary => ({ + component_id: dataApp.component_id, + configuration_id: dataApp.configuration_id, + data_app_id: dataApp.data_app_id, + project_id: dataApp.project_id, + branch_id: dataApp.branch_id, + config_version: dataApp.config_version, + state: dataApp.state, + type: dataApp.type, + deployment_url: dataApp.deployment_url, + auto_suspend_after_seconds: dataApp.auto_suspend_after_seconds, + repo_url: dataApp.repo_url, +}); + +export const summaryFromApiResponse = (api: DataAppResponse): DataAppSummary => ({ + component_id: api.component_id, + configuration_id: api.config_id, + data_app_id: api.id, + project_id: api.project_id, + branch_id: api.branch_id ?? '', + config_version: api.config_version, + state: api.state, + type: api.type, + deployment_url: api.url ?? null, + auto_suspend_after_seconds: api.auto_suspend_after_seconds ?? null, + repo_url: null, +}); + +export const dataAppFromApiResponses = ( + apiResponse: DataAppResponse, + rawConfig: Record, + metadataFolderField: string, +): DataApp => { + const metadata = (rawConfig.metadata as MetadataItem[]) ?? []; + return { + component_id: DATA_APP_COMPONENT_ID, + configuration_id: String(rawConfig.id ?? ''), + data_app_id: apiResponse.id, + project_id: apiResponse.project_id, + branch_id: apiResponse.branch_id ?? '', + config_version: String(rawConfig.version ?? ''), + state: apiResponse.state, + type: apiResponse.type, + deployment_url: apiResponse.url ?? null, + auto_suspend_after_seconds: apiResponse.auto_suspend_after_seconds ?? null, + name: String(rawConfig.name ?? ''), + description: (rawConfig.description as string | null) ?? null, + folder: getMetadataProperty(metadata, metadataFolderField) ?? '', + configuration: asRecord(rawConfig.configuration), + repo_url: null, + deployment_info: null, + drafts: [], + drafts_unavailable: 0, + links: [], + }; +}; + +export type MetadataItem = { id?: string; key?: string; value?: string }; + +export const getMetadataProperty = (metadata: MetadataItem[], key: string): string | undefined => + metadata.find((m) => m.key === key)?.value; diff --git a/src/tools/doc.ts b/src/tools/doc.ts new file mode 100644 index 000000000..688b4aab6 --- /dev/null +++ b/src/tools/doc.ts @@ -0,0 +1,27 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { getDocsSearch } from '@/clients/docsSearch'; +import { registerTool } from '@/mcp/tool'; + +// Ported from tools/doc.py. Backed by the pgvector docs-search index (RFC: +// feature_spec/docs-search-pgvector/) instead of the legacy AI docs service. The docs +// index is process-level infrastructure (no per-request Config needed). + +export const registerDocTools = (server: McpServer): void => { + registerTool(server, { + name: 'docs_query', + title: 'Query documentation', + description: 'Answers a question using the Keboola documentation as a source.', + annotations: { readOnlyHint: true }, + inputSchema: { + query: z.string().describe('Natural language query to search for in the documentation.'), + }, + handler: async ({ query }) => { + const docs = getDocsSearch(); + if (!docs) throw new Error('The documentation index is not available.'); + const answer = await docs.answerQuestion(query); + return { text: answer.text, source_urls: answer.sourceUrls }; + }, + }); +}; diff --git a/src/tools/flow/descriptions.ts b/src/tools/flow/descriptions.ts new file mode 100644 index 000000000..66fb2961f --- /dev/null +++ b/src/tools/flow/descriptions.ts @@ -0,0 +1,119 @@ +// Tool descriptions for the flow tools, preserved verbatim from the Python docstrings. +// Extracted from tools.ts to keep the handler module focused. + +export const CREATE_FLOW_DESCRIPTION = `Creates a new legacy (non-conditional) flow using \`keboola.orchestrator\`. + +PRE-REQUISITES: +- Always use \`get_flow_schema\` with flow_type="keboola.orchestrator" and review \`get_flow_examples\` if unknown +- Collect component configuration IDs for every task you include + +RULES: +- \`phases\` and \`tasks\` must follow the orchestrator schema; each entry must include \`id\` and \`name\` +- Phases run sequentially; tasks inside a phase run in parallel +- Use \`dependsOn\` on phases to sequence them; reference other phase ids +- Always share the returned links with the user + +WHEN TO USE: +- Simple/linear orchestrations without branching or conditions +- ETL/ELT pipelines where phases just need ordering and parallel task groups`; + +export const CREATE_CONDITIONAL_FLOW_DESCRIPTION = `Creates a new conditional flow configuration using \`keboola.flow\`. + +PRE-REQUISITES: +- Always use \`get_flow_schema\` with flow_type="keboola.flow" and review \`get_flow_examples\` if unknown +- Gather component configuration IDs for all tasks you include + +RULES: +- \`phases\` and \`tasks\` must follow the keboola.flow schema; each entry needs \`id\` and \`name\` +- Exactly one entry phase (no incoming transitions); all phases must be reachable +- Connect phases via \`next\` transitions; no cycles or dangling phases; empty \`next\` means flow end +- Task/phase failures already stop the flow; add retries/conditions only if the user requests them +- Always share the returned links with the user + +WHEN TO USE: +- Flows needing branching, conditions, retries, or notifications +- Default choice when user simply says "create a flow," unless they explicitly want legacy orchestrator behavior`; + +export const UPDATE_FLOW_DESCRIPTION = `Updates an existing flow configuration (either legacy \`keboola.orchestrator\` or conditional \`keboola.flow\`). + +PRE-REQUISITES: +- Always use \`get_flow_schema\` (and \`get_flow_examples\`) for that flow type you want to update to follow the +required structure and see the examples if unknown +- Only pass \`phases\`/\`tasks\` when you want to replace them; omit to keep the existing ones unchanged + +RULES (ALL FLOWS): +- \`flow_type\` must match the stored component id of the flow; do not switch flow types during update +- \`phases\` and \`tasks\` must follow the schema for the selected flow type; include at least \`id\` and \`name\` +- Tasks must reference existing component configurations; keep dependencies consistent +- Always provide a clear \`change_description\` and surface any links returned in the response to the user + +CONDITIONAL FLOWS (\`keboola.flow\`): +- Maintain a single entry phase and ensure every phase is reachable; connect phases via \`next\` transitions +- No cycles or dangling phases; failed tasks already stop the flow, so only add retries/conditions if requested + +LEGACY FLOWS (\`keboola.orchestrator\`): +- Phases run sequentially; tasks inside a phase run in parallel; \`dependsOn\` references other phase ids +- Use \`continueOnFailure\` or best-effort patterns only when the user explicitly asks for them + +WHEN TO USE: +- Renaming a flow, updating descriptions, adding/removing phases or tasks, adjusting dependencies, +or enabling/disabling flow execution`; + +export const MODIFY_FLOW_DESCRIPTION = `Updates an existing flow configuration (either legacy \`keboola.orchestrator\` or conditional \`keboola.flow\`) or +manages schedules for this flow. + +PRE-REQUISITES: +- Always use \`get_flow_schema\` (and \`get_flow_examples\`) for that flow type you want to update to follow the +required structure and see the examples if unknown +- Only pass \`phases\`/\`tasks\` when you want to replace them; omit to keep the existing ones unchanged + +RULES (ALL FLOWS): +- \`flow_type\` must match the stored component id of the flow; do not switch flow types during update +- \`phases\` and \`tasks\` must follow the schema for the selected flow type; include at least \`id\` and \`name\` +- Tasks must reference existing component configurations; keep dependencies consistent +- Always provide a clear \`change_description\` and surface any links returned in the response to the user +- A flow can have multiple schedules for automation runs. Add/update/remove schedules only if requested. +- When updating a flow or a schedule, specify only the fields you want to update, others will be kept unchanged. + +CONDITIONAL FLOWS (\`keboola.flow\`): +- Maintain a single entry phase and ensure every phase is reachable; connect phases via \`next\` transitions +- No cycles or dangling phases; failed tasks already stop the flow, so only add retries/conditions if requested + +LEGACY FLOWS (\`keboola.orchestrator\`): +- Phases run sequentially; tasks inside a phase run in parallel; \`dependsOn\` references other phase ids +- Use \`continueOnFailure\` or best-effort patterns only when the user explicitly asks for them + +WHEN TO USE: +- Renaming a flow, updating descriptions, adding/removing phases or tasks, updating schedules, +adjusting dependencies, or enabling/disabling flow execution`; + +export const GET_FLOWS_DESCRIPTION = `Lists flows or retrieves full details for specific flows. + +WHEN NOT TO USE: +- Do NOT call with \`flow_ids=[]\` just to find a flow by name. Use \`search\` with + item_types=["flow"] instead. +- Only use \`flow_ids=[]\` when you need a complete list of all flows in the project. + +OPTIONS: +- \`flow_ids=[]\` → summaries of all flows in the project +- \`flow_ids=["id1", ...]\` → full details (including phases/tasks) for those flows`; + +export const GET_FLOW_SCHEMA_DESCRIPTION = `Returns the JSON schema for the given flow type (markdown). + +PRE-REQUISITES: +- Unknown schema for the target flow type: \`keboola.flow\` (conditional) or \`keboola.orchestrator\` (legacy) + +RULES: +- Projects without conditional flows enabled cannot request \`keboola.flow\` schema +- Use the returned schema to shape \`phases\` and \`tasks\` for \`create_flow\` / \`create_conditional_flow\` / +\`update_flow\``; + +export const GET_FLOW_EXAMPLES_DESCRIPTION = `Retrieves examples of valid flow configurations. + +PRE-REQUISITES: +- Unknown examples for the target flow type: \`keboola.flow\` (conditional) or \`keboola.orchestrator\` (legacy) to help +build the specific flow configuration by mirroring the structure/fields. + +RULES: +- Conditional-flow examples require conditional flows to be enabled; otherwise use legacy orchestrator examples +- Present the examples or cite unavailability to the user`; diff --git a/src/tools/flow/index.ts b/src/tools/flow/index.ts new file mode 100644 index 000000000..a7526eb81 --- /dev/null +++ b/src/tools/flow/index.ts @@ -0,0 +1,3 @@ +// Flow tools package (ported from the Python `tools/flow/` package). Public surface: +// `registerFlowTools`, consumed by `src/server.ts`. +export { registerFlowTools } from './tools'; diff --git a/src/tools/flow/model.ts b/src/tools/flow/model.ts new file mode 100644 index 000000000..418587472 --- /dev/null +++ b/src/tools/flow/model.ts @@ -0,0 +1,328 @@ +import { Ajv } from 'ajv'; +import { z } from 'zod'; + +import { fetchComponent } from '../components'; + +import type { KeboolaClients } from '@/clients/keboola'; +import { + CONDITIONAL_FLOW_COMPONENT_ID, + type FlowType, + ORCHESTRATOR_COMPONENT_ID, +} from '@/constants'; +import { logger } from '@/logger'; + +// Ported from tools/flow/{model,utils}.py and clients/validation.py: the zod input +// schemas + structural/JSON-schema flow validation live here. + +// ============================================================================= +// SHARED ALIASES +// ============================================================================= + +export type RawConfig = Record; +export type MetadataItem = { id?: string; key?: string; value?: string }; +export type Phase = Record; +export type Task = Record; + +// MCP tracking metadata keys (not yet in the shared constants module; kept local to avoid +// editing a cross-module file). Mirror config.py CREATED_BY_MCP / UPDATED_BY_MCP_PREFIX. +export const CREATED_BY_MCP = 'KBC.MCP.createdBy'; +export const UPDATED_BY_MCP_PREFIX = 'KBC.MCP.updatedBy.version.'; + +// ============================================================================= +// SCHEDULE REQUEST MODEL (snake_case params, mirrors ScheduleRequest) +// ============================================================================= + +export type ScheduleRequest = { + action: 'add' | 'update' | 'remove'; + schedule_id?: string | null; + timezone?: string | null; + cron_tab?: string | null; + state?: 'enabled' | 'disabled' | null; +}; + +export const scheduleRequestSchema = z.object({ + action: z.enum(['add', 'update', 'remove']).describe('Action to perform on the schedule.'), + schedule_id: z + .string() + .nullish() + .describe('ID of the schedule configuration to update. None if creating a new schedule.'), + timezone: z + .string() + .nullish() + .describe('Timezone for the schedule. Default UTC if None provided.'), + cron_tab: z + .string() + .nullish() + .describe( + 'Cron expression for the schedule following the format: `* * * * *`.' + + 'Where 1. minutes, 2. hours, 3. days of month, 4. months, 5. days of week. Example: `15,45 1,13 * * 0`', + ), + state: z.enum(['enabled', 'disabled']).nullish().describe('Enable or disable the schedule.'), +}); + +export const normalizeScheduleRequests = ( + raw: z.infer[], +): ScheduleRequest[] => + raw.map((r) => ({ + action: r.action, + schedule_id: r.schedule_id ?? null, + timezone: r.timezone ?? null, + cron_tab: r.cron_tab ?? null, + state: r.state ?? null, + })); + +export const flowTypeSchema = z.enum([CONDITIONAL_FLOW_COMPONENT_ID, ORCHESTRATOR_COMPONENT_ID]); + +// ============================================================================= +// STRUCTURAL VALIDATION (port of utils._validate_*_flow_structure) +// ============================================================================= + +const normalizeDependsOn = (phase: Phase): (string | number)[] => { + const value = phase.dependsOn ?? phase.depends_on ?? phase['depends-on'] ?? []; + if (!Array.isArray(value)) { + throw new Error(`Invalid phase configuration: dependsOn must be a list.`); + } + return value as (string | number)[]; +}; + +const checkCircularDependencies = ( + edges: Map, + allNodeIds: Set, +): void => { + const visited = new Set(); + + const hasCycle = ( + nodeId: string | number, + recStack: Set, + path: (string | number)[], + ): (string | number)[] | null => { + visited.add(nodeId); + recStack.add(nodeId); + path.push(nodeId); + for (const target of edges.get(nodeId) ?? []) { + if (!visited.has(target)) { + const cycle = hasCycle(target, recStack, path); + if (cycle) return cycle; + } else if (recStack.has(target)) { + const idx = path.indexOf(target); + return idx >= 0 ? [...path.slice(idx), target] : [nodeId, target]; + } + } + path.pop(); + recStack.delete(nodeId); + return null; + }; + + for (const nodeId of allNodeIds) { + if (!visited.has(nodeId)) { + const cyclePath = hasCycle(nodeId, new Set(), []); + if (cyclePath) { + throw new Error(`Circular dependency detected: ${cyclePath.map(String).join(' -> ')}`); + } + } + } +}; + +const validateLegacyFlowStructure = (phases: Phase[], tasks: Task[]): void => { + const phaseIds = new Set(phases.map((p) => p.id as string | number)); + for (const phase of phases) { + for (const dep of normalizeDependsOn(phase)) { + if (!phaseIds.has(dep)) { + throw new Error(`Phase ${phase.id} depends on non-existent phase ${dep}`); + } + } + } + for (const task of tasks) { + if (!phaseIds.has(task.phase as string | number)) { + throw new Error(`Task ${task.id} references non-existent phase ${task.phase}`); + } + } + const edges = new Map( + phases.map((p) => [p.id as string | number, normalizeDependsOn(p)]), + ); + checkCircularDependencies(edges, phaseIds); +}; + +const reachableIds = ( + start: string, + edges: Map>, + visited: Set, +): Set => { + visited.add(start); + for (const target of edges.get(start) ?? []) { + if (!visited.has(target)) reachableIds(target, edges, visited); + } + return visited; +}; + +const validateConditionalFlowStructure = (phases: Phase[], tasks: Task[]): void => { + // Duplicate phase / task ids. + const countOccurrences = (ids: unknown[]): Map => { + const counter = new Map(); + for (const id of ids) counter.set(id, (counter.get(id) ?? 0) + 1); + return counter; + }; + const phaseCounter = countOccurrences(phases.map((p) => p.id)); + const dupPhases = [...phaseCounter.entries()].filter(([, c]) => c > 1).map(([id]) => id); + if (dupPhases.length > 0) { + throw new Error(`Flow contains duplicate phase IDs: ${JSON.stringify(dupPhases)}.`); + } + const taskCounter = countOccurrences(tasks.map((t) => t.id)); + const dupTasks = [...taskCounter.entries()].filter(([, c]) => c > 1).map(([id]) => id); + if (dupTasks.length > 0) { + throw new Error(`Flow contains duplicate task IDs: ${JSON.stringify(dupTasks)}.`); + } + + const phaseIds = new Set(phases.map((p) => String(p.id))); + for (const task of tasks) { + if (!phaseIds.has(String(task.phase))) { + throw new Error(`Task ${task.id} references non-existent phase ${task.phase}`); + } + } + + const succPhases = new Map>(); + const predPhases = new Map>(); + const endingPhases = new Set(); + const ensure = (map: Map>, key: string): Set => { + let set = map.get(key); + if (!set) { + set = new Set(); + map.set(key, set); + } + return set; + }; + + for (const phase of phases) { + const pid = String(phase.id); + const next = (phase.next as { goto?: string | null }[] | undefined) ?? []; + if (next.length === 0) { + endingPhases.add(pid); + } else { + for (const transition of next) { + if (transition.goto === null || transition.goto === undefined) { + endingPhases.add(pid); + } else { + if (!phaseIds.has(transition.goto)) { + throw new Error( + `Phase ${phase.id} has a transition that references non-existent phase ${transition.goto}`, + ); + } + ensure(succPhases, pid).add(transition.goto); + ensure(predPhases, transition.goto).add(pid); + } + } + } + } + + if (endingPhases.size === 0) { + throw new Error( + 'Flow has no ending phases. Each conditional flow must have at least one ending phase. Any ending phase ' + + 'has either no transitions at all or contains transition with goto: null referencing end of the flow.', + ); + } + + const entryPhases = [...phaseIds].filter((pid) => (predPhases.get(pid)?.size ?? 0) === 0); + if (entryPhases.length === 0) { + throw new Error( + 'Flow has no entry phase. Each conditional flow must have exactly one entry phase. An entry phase has no ' + + 'incoming transitions; no transition from another phase leads to it.', + ); + } + if (entryPhases.length > 1) { + throw new Error( + `Flow has multiple entry phases (${entryPhases.length}): ${JSON.stringify(entryPhases)}. Each conditional flow must have ` + + 'exactly one entry phase. Either merge the entry phases into one or redefine the transitions to form a ' + + 'single entry phase.', + ); + } + + const reachable = reachableIds(entryPhases[0]!, succPhases, new Set()); + if (reachable.size !== phaseIds.size || [...phaseIds].some((id) => !reachable.has(id))) { + const unreachable = [...phaseIds].filter((id) => !reachable.has(id)); + throw new Error( + `Flow has phases that are not reachable from the entry phase (${entryPhases[0]}): ` + + `${JSON.stringify(unreachable)}. All phases must be reachable from the entry phase by a valid path of ` + + 'transitions.', + ); + } + + const edges = new Map(); + for (const [pid, targets] of succPhases) edges.set(pid, [...targets]); + checkCircularDependencies(edges, phaseIds); +}; + +export const validateFlowStructure = ( + flowConfiguration: Record, + flowType: FlowType, +): void => { + const phases = (flowConfiguration.phases as Phase[]) ?? []; + const tasks = (flowConfiguration.tasks as Task[]) ?? []; + if (flowType === ORCHESTRATOR_COMPONENT_ID) { + validateLegacyFlowStructure(phases, tasks); + } else { + validateConditionalFlowStructure(phases, tasks); + } +}; + +export { normalizeDependsOn }; + +// ============================================================================= +// SCHEMA VALIDATION (jsonschema, port of validation.validate_flow_configuration_against_schema) +// ============================================================================= + +// `strict: false` mirrors Python jsonschema's leniency (e.g. `minLength` on integer-or-string +// ids, `$ref` next to sibling keywords); invalid schemas there "continue as valid", so we never +// hard-fail on schema-author mistakes — only on data that violates a usable schema. +const ajv = new Ajv({ strict: false, allErrors: true }); + +export const validateFlowConfigurationAgainstSchema = ( + flow: Record, + schema: Record, +): void => { + let validate; + try { + validate = ajv.compile(schema); + } catch (e) { + // Schema itself is unusable — Python logs and treats the data as valid. + logger.error({ err: e }, 'The validation schema is not valid; skipping schema validation.'); + return; + } + if (!validate(flow)) { + const message = (validate.errors ?? []) + .map((err) => `${err.instancePath || ''} ${err.message}`) + .join('; '); + throw new Error(`Flow configuration does not follow the schema: ${message}`); + } +}; + +/** + * Resolve the JSON schema for a flow type. Legacy orchestrator stays bundled; conditional + * (`keboola.flow`) is fetched live from the Developer Portal (AI catalog) via fetchComponent. + */ +export const resolveFlowSchema = async ( + clients: KeboolaClients, + flowType: FlowType, + loadLegacySchema: () => Record, +): Promise> => { + if (flowType !== CONDITIONAL_FLOW_COMPONENT_ID) { + return loadLegacySchema(); + } + const failureMessage = + 'Could not retrieve the conditional flow (keboola.flow) configuration schema from the ' + + 'Developer Portal. The schema is required to create or validate conditional flows. ' + + 'Please retry; if this persists the keboola.flow component schema may be unavailable on ' + + 'this stack.'; + let component: Record; + try { + component = await fetchComponent(clients, CONDITIONAL_FLOW_COMPONENT_ID); + } catch (e) { + throw new Error(failureMessage, { cause: e }); + } + const schema = (component.configurationSchema ?? component.configuration_schema) as + | Record + | undefined; + if (!schema || Object.keys(schema).length === 0) { + throw new Error(failureMessage); + } + return schema; +}; diff --git a/src/tools/flow/scheduler.ts b/src/tools/flow/scheduler.ts new file mode 100644 index 000000000..f51ca7cd1 --- /dev/null +++ b/src/tools/flow/scheduler.ts @@ -0,0 +1,387 @@ +import type { KeboolaClients } from '@/clients/keboola'; +import { createRawClient, type RawClient } from '@/clients/raw'; +import { deriveServiceUrls } from '@/clients/urls'; +import type { Config } from '@/config'; +import { logger } from '@/logger'; +import type { ScheduleRequest } from './model'; +import { + configurationCreate, + configurationDelete, + configurationDetail, + configurationUpdate, + setCfgCreationMetadata, + setCfgUpdateMetadata, +} from './utils'; + +// Ported from tools/flow/{scheduler,scheduler_model}.py and clients/scheduler.py. +// The scheduler service has no @keboola/api-client subpath, so it is built locally as a +// raw client (see createSchedulerClient below). Scheduler *configurations* are still +// stored as component configs via the typed Storage client (configurationCreate/etc.). + +// ============================================================================= +// SCHEDULER CLIENT (local raw client; no api-client subpath exists) +// ============================================================================= + +export type ScheduleApiResponse = { + id: string; + configurationId?: string; + configuration_id?: string; + schedule: { cronTab?: string; cron_tab?: string; timezone: string; state: string }; + target?: Record; + executions?: { + jobId?: string; + job_id?: string; + executionTime?: string; + execution_time?: string; + }[]; +}; + +export type SchedulerClient = { + activateSchedule: (scheduleConfigId: string) => Promise; + listSchedulesByConfigId: ( + componentId: string, + configurationId: string, + ) => Promise; + deleteSchedule: (scheduleConfigId: string) => Promise; +}; + +/** Builds a Scheduler API client against `deriveServiceUrls(...).scheduler`. */ +export const createSchedulerClient = (config: Config): SchedulerClient => { + const urls = deriveServiceUrls(config.storageApiUrl ?? ''); + const raw: RawClient = createRawClient({ + baseUrl: urls.scheduler, + token: config.bearerToken ? `Bearer ${config.bearerToken}` : config.storageToken, + }); + return { + activateSchedule: (scheduleConfigId) => + raw.post('schedules', { body: { configurationId: scheduleConfigId } }), + listSchedulesByConfigId: (componentId, configurationId) => + raw.get('schedules', { + params: { componentId, configurationId }, + }), + deleteSchedule: async (scheduleConfigId) => { + await raw.delete(`configurations/${scheduleConfigId}`); + }, + }; +}; + +// ============================================================================= +// SCHEDULER MODELS + LOGIC (port of scheduler.py + scheduler_model.py) +// ============================================================================= + +export const toScheduleDetail = (api: ScheduleApiResponse) => ({ + scheduleId: api.configurationId ?? api.configuration_id ?? '', + timezone: api.schedule.timezone, + state: api.schedule.state, + cronTab: api.schedule.cronTab ?? api.schedule.cron_tab ?? '', + target_executions: (api.executions ?? []).map((exec) => ({ + jobId: exec.jobId ?? exec.job_id ?? null, + executionTime: exec.executionTime ?? exec.execution_time ?? null, + })), +}); + +const SCHEDULER_COMPONENT_ID = 'keboola.scheduler'; + +const CRON_TAB_INSTRUCTIONS = ` +Cron Tab Expression should be in the format: \`* * * * *\`. +Field order: +1. Minute (0-59) +2. Hour (0-23) +3. Day of month (1-31, or L for last day of month) +4. Month (1-12) +5. Day of week (0-6, where 0 = Sunday) + +Examples: +1. schedule daily at 1:00 PM and 1:00 AM would be \`0 1,13 * * *\` +2. schedule weekly on Monday at 9:00 AM would be \`0 9 * * 1\` +3. schedule monthly on the 1st and 20th day of the month at 10:00 AM would be \`0 10 1,20 * *\` +4. schedule yearly on the 1st of january and august at 11:00 AM would be \`0 11 1 1,8 *\` +5. schedule hourly every 15 minutes would be \`0,15,30,45 * * * *\` +6. schedule monthly on the last day of the month at 10:00 AM would be \`0 10 L * *\` +`; + +/** Port of scheduler.validate_cron_tab. */ +export const validateCronTab = (cronTab: string | null | undefined): void => { + if (cronTab === null || cronTab === undefined) return; + try { + const parts = cronTab.trim().split(/\s+/); + if (parts.length !== 5) { + throw new Error( + `Cron expression must have exactly 5 parts got: ${cronTab} which has ${parts.length} parts.`, + ); + } + const toIntList = (field: string, allowL = false): { parts: number[]; hasL: boolean } => { + if (field === '*') return { parts: [], hasL: false }; + let hasL = false; + const nums: number[] = []; + for (let x of field.split(',')) { + x = x.trim(); + if (allowL && x.toUpperCase() === 'L') { + hasL = true; + } else if (/^-?\d+$/.test(x)) { + nums.push(Number(x)); + } else { + throw new Error(`Cron expression must have only digits got: ${field} in "${cronTab}".`); + } + } + if (allowL && hasL && nums.length > 0) { + throw new Error('Day of month must use either `L` or numeric values, not both.'); + } + return { parts: nums, hasL }; + }; + + const { parts: minutes } = toIntList(parts[0]!.trim()); + const { parts: hours } = toIntList(parts[1]!.trim()); + const { parts: days, hasL: hasLastDay } = toIntList(parts[2]!.trim(), true); + const { parts: months } = toIntList(parts[3]!.trim()); + const { parts: weekdays } = toIntList(parts[4]!.trim()); + + if (minutes.some((x) => x < 0 || x > 59)) { + throw new Error(`Minutes of hour \`M _ _ _ _\` must be between 0 and 59, got: ${parts[0]}`); + } + if (hours.some((x) => x < 0 || x > 23)) { + throw new Error(`Hours of day \`_ H _ _ _\` must be between 0 and 23, got: ${parts[1]}`); + } + if (days.some((x) => x < 1 || x > 31)) { + throw new Error(`Days of month \`_ _ D _ _\`must be between 1 and 31, got: ${parts[2]}`); + } + if (months.some((x) => x < 1 || x > 12)) { + throw new Error(`Months of year \`_ _ _ M _\` must be between 1 and 12, got: ${parts[3]}`); + } + if (weekdays.some((x) => x < 0 || x > 6)) { + throw new Error( + `Days of week \`_ _ _ _ W\` must be between 0=Sunday and 6=Saturday, got: ${parts[4]}`, + ); + } + if (months.length > 0 && days.length === 0 && !hasLastDay) { + throw new Error( + 'Months of year must be specified with days of month. Example: `35 12 31 1,3 *`', + ); + } + if ((days.length > 0 || hasLastDay) && hours.length === 0) { + throw new Error('Days of month must be specified with hours of day. Example: `55 12 31 * *`'); + } + if (hours.length > 0 && minutes.length === 0) { + throw new Error( + 'Hours of day must be specified with minutes of hour. Example: `55 12 * * *`', + ); + } + if (weekdays.length > 0 && hours.length === 0) { + throw new Error('Days of week must be specified with hours of day. Example: `55 12 * * 0`'); + } + if (weekdays.length > 0 && (days.length > 0 || months.length > 0 || hasLastDay)) { + throw new Error('Days of week must not be specified with days of month nor months of year.'); + } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + throw new Error(`Invalid cron tab expression: ${msg}.\n${CRON_TAB_INSTRUCTIONS}`); + } +}; + +type SimplifiedSchedule = { + scheduleId: string | null; + cronTab: string; + timezone: string; + state: string; +}; + +export const listSchedulesForConfig = async ( + scheduler: SchedulerClient, + componentId: string, + configurationId: string, +): Promise[]> => { + const apiSchedules = await scheduler.listSchedulesByConfigId(componentId, configurationId); + return apiSchedules.map(toScheduleDetail); +}; + +/** Compute original/updated/new schedulers (port of scheduler._update_schedulers_internal). */ +const updateSchedulersInternal = async ( + scheduler: SchedulerClient, + configurationId: string, + componentId: string, + schedules: ScheduleRequest[], +): Promise<{ + original: Map; + updated: Map; + added: SimplifiedSchedule[]; +}> => { + const current = await listSchedulesForConfig(scheduler, componentId, configurationId); + const original = new Map(); + for (const s of current) { + original.set(s.scheduleId, { + scheduleId: s.scheduleId, + cronTab: s.cronTab, + timezone: s.timezone, + state: s.state, + }); + } + const added: SimplifiedSchedule[] = []; + const updated = new Map(); + + for (const request of schedules) { + if (request.action === 'add') { + if (request.cron_tab == null) { + throw new Error('cron_tab is required to add a schedule.'); + } + validateCronTab(request.cron_tab); + added.push({ + scheduleId: request.schedule_id ?? null, + cronTab: request.cron_tab, + timezone: request.timezone ?? 'UTC', + state: request.state ?? 'enabled', + }); + } else if (request.action === 'update') { + const id = request.schedule_id ?? ''; + const existing = original.get(id); + if (!existing) { + throw new Error( + `Schedule (ID: ${request.schedule_id}) cannot be updated because it was not found in the existing schedulers.`, + ); + } + if (request.cron_tab != null) validateCronTab(request.cron_tab); + updated.set(id, { + scheduleId: existing.scheduleId, + cronTab: request.cron_tab ?? existing.cronTab, + timezone: request.timezone ?? existing.timezone, + state: request.state ?? existing.state, + }); + } else if (request.action === 'remove') { + const id = request.schedule_id ?? ''; + if (!original.has(id)) { + throw new Error( + `Schedule (ID: ${request.schedule_id}) cannot be removed because it was not found in the existing schedulers.`, + ); + } + updated.set(id, null); + } else { + throw new Error(`Invalid action for schedulers: ${(request as { action: string }).action}.`); + } + } + return { original, updated, added }; +}; + +const createSchedule = async ( + clients: KeboolaClients, + scheduler: SchedulerClient, + targetComponentId: string, + targetConfigurationId: string, + cronTab: string, + timezone: string, + state: string, +): Promise> => { + const scheduleName = `Schedule for ${targetConfigurationId}`; + const schedulerConfig = { + schedule: { cronTab, timezone, state }, + target: { componentId: targetComponentId, configurationId: targetConfigurationId, mode: 'run' }, + }; + const storageResponse = await configurationCreate( + clients, + SCHEDULER_COMPONENT_ID, + scheduleName, + `Automated schedule for ${targetConfigurationId}`, + schedulerConfig, + ); + const scheduleConfigId = String(storageResponse.id ?? ''); + logger.info(`Created schedule configuration in Storage API: ${scheduleConfigId}`); + const scheduleResponse = await scheduler.activateSchedule(scheduleConfigId); + logger.info(`Activated schedule in Scheduler API: ${scheduleResponse.id}`); + await setCfgCreationMetadata(clients, SCHEDULER_COMPONENT_ID, scheduleConfigId); + return toScheduleDetail(scheduleResponse); +}; + +const updateSchedule = async ( + clients: KeboolaClients, + scheduler: SchedulerClient, + scheduleConfigId: string, + cronTab: string | null, + timezone: string | null, + state: string | null, +): Promise => { + const currentConfig = await configurationDetail( + clients, + SCHEDULER_COMPONENT_ID, + scheduleConfigId, + ); + const schedulerConfig = (currentConfig.configuration as Record) ?? {}; + const schedule = (schedulerConfig.schedule as Record) ?? {}; + if (cronTab !== null) schedule.cronTab = cronTab; + if (timezone !== null) schedule.timezone = timezone; + if (state !== null) schedule.state = state; + schedulerConfig.schedule = schedule; + + const updated = await configurationUpdate( + clients, + SCHEDULER_COMPONENT_ID, + scheduleConfigId, + schedulerConfig, + 'Schedule Updated', + ); + logger.info(`Updated schedule configuration in Storage API: ${scheduleConfigId}`); + await scheduler.activateSchedule(scheduleConfigId); + await setCfgUpdateMetadata( + clients, + SCHEDULER_COMPONENT_ID, + scheduleConfigId, + Number(updated.version ?? 0), + ); +}; + +const removeSchedule = async ( + clients: KeboolaClients, + scheduler: SchedulerClient, + scheduleConfigId: string, +): Promise => { + await scheduler.deleteSchedule(scheduleConfigId); + await configurationDelete(clients, SCHEDULER_COMPONENT_ID, scheduleConfigId); +}; + +/** Port of scheduler.process_schedule_request. */ +export const processScheduleRequest = async ( + clients: KeboolaClients, + scheduler: SchedulerClient, + targetComponentId: string, + targetConfigurationId: string, + requests: ScheduleRequest[], +): Promise => { + const { updated, added } = await updateSchedulersInternal( + scheduler, + targetConfigurationId, + targetComponentId, + requests, + ); + const responses: string[] = []; + try { + for (const [scheduleId, schedule] of updated) { + if (schedule === null) { + await removeSchedule(clients, scheduler, scheduleId); + responses.push(`Removed schedule: ${scheduleId}`); + } else { + await updateSchedule( + clients, + scheduler, + scheduleId, + schedule.cronTab, + schedule.timezone, + schedule.state, + ); + responses.push(`Updated schedule: ${scheduleId}`); + } + } + for (const newScheduler of added) { + const response = await createSchedule( + clients, + scheduler, + targetComponentId, + targetConfigurationId, + newScheduler.cronTab, + newScheduler.timezone, + newScheduler.state, + ); + responses.push(`Created schedule: ${response.scheduleId}`); + } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + throw new Error(`Error processing schedule requests: ${msg}`); + } + return responses; +}; diff --git a/src/tools/flow/tools.ts b/src/tools/flow/tools.ts new file mode 100644 index 000000000..dde2cc0b5 --- /dev/null +++ b/src/tools/flow/tools.ts @@ -0,0 +1,555 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { createKeboolaClients, createLinksManager } from '@/clients/keboola'; +import type { Config } from '@/config'; +import { + CONDITIONAL_FLOW_COMPONENT_ID, + FLOW_TYPES, + type FlowType, + ORCHESTRATOR_COMPONENT_ID, +} from '@/constants'; +import { logger } from '@/logger'; +import { registerTool } from '@/mcp/tool'; +import { + CREATE_CONDITIONAL_FLOW_DESCRIPTION, + CREATE_FLOW_DESCRIPTION, + GET_FLOW_EXAMPLES_DESCRIPTION, + GET_FLOW_SCHEMA_DESCRIPTION, + GET_FLOWS_DESCRIPTION, + MODIFY_FLOW_DESCRIPTION, + UPDATE_FLOW_DESCRIPTION, +} from './descriptions'; +import { + flowTypeSchema, + normalizeScheduleRequests, + type Phase, + type RawConfig, + resolveFlowSchema, + type ScheduleRequest, + scheduleRequestSchema, + type Task, + validateFlowConfigurationAgainstSchema, + validateFlowStructure, +} from './model'; +import { createSchedulerClient, listSchedulesForConfig, processScheduleRequest } from './scheduler'; +import { + assertConditionalAllowed, + buildFlowToolOutput, + buildFolderHint, + clearConfigurationFolderMetadata, + configurationCreate, + configurationDetail, + configurationList, + configurationUpdate, + EXAMPLE_FILES, + flowLabel, + folderFieldDescription, + getConfigFolders, + getFlowConfiguration, + getProjectContext, + loadLegacySchema, + readResource, + resolveFlowById, + setCfgCreationMetadata, + setCfgUpdateMetadata, + setConfigurationFolderMetadata, + toFlowDetail, + toFlowSummary, +} from './utils'; + +// Ported from tools/flow/tools.py: the 7 tool handlers + the shared create/modify cores. + +const getSchemaAsMarkdown = async ( + clients: ReturnType, + flowType: FlowType, +): Promise => { + const schema = await resolveFlowSchema(clients, flowType, loadLegacySchema); + return `\`\`\`json\n${JSON.stringify(schema, null, 2)}\n\`\`\``; +}; + +// ============================================================================= +// CREATE (shared between create_flow and create_conditional_flow) +// ============================================================================= + +const createFlowImpl = async ( + config: Config, + flowType: FlowType, + args: { name: string; description: string; phases: Phase[]; tasks: Task[]; folder: string }, +) => { + const clients = createKeboolaClients(config); + const flowConfiguration = getFlowConfiguration(args.phases, args.tasks, flowType); + + // Structural validation (semantic), then schema validation (syntax). + validateFlowStructure(flowConfiguration, flowType); + const schema = await resolveFlowSchema(clients, flowType, loadLegacySchema); + validateFlowConfigurationAgainstSchema(flowConfiguration, schema); + + const linksManager = await createLinksManager(config, clients); + const newRaw = await configurationCreate( + clients, + flowType, + args.name, + args.description, + flowConfiguration, + ); + const configId = String(newRaw.id ?? ''); + await setCfgCreationMetadata(clients, flowType, configId); + + const folder = args.folder.trim(); + let changeSummary: string | null = null; + if (folder) { + try { + await setConfigurationFolderMetadata(clients, flowType, configId, folder); + } catch { + logger.warn( + `Unable to set folder metadata for component "${flowType}", configuration "${configId}".`, + ); + } + } else { + try { + const { total, folders, lowerBound } = await getConfigFolders(clients, flowType); + changeSummary = buildFolderHint( + total, + folders, + flowLabel(flowType), + 'modify_flow', + lowerBound, + ); + } catch { + logger.warn( + `Unable to fetch flow folders for component "${flowType}" when creating flow "${configId}".`, + ); + } + } + + const flowLinks = linksManager.getFlowLinks(configId, String(newRaw.name ?? ''), flowType); + logger.info( + `Created flow "${args.name}" with configuration ID "${configId}" (type: ${flowType})`, + ); + return buildFlowToolOutput({ + configurationId: configId, + componentId: flowType, + description: (newRaw.description as string) || '', + version: Number(newRaw.version ?? 0), + links: flowLinks, + changeSummary, + }); +}; + +// ============================================================================= +// MODIFY (shared core behind modify_flow and update_flow) +// ============================================================================= + +const modifyFlowImpl = async ( + config: Config, + args: { + configuration_id: string; + flow_type: FlowType; + change_description: string; + phases: Phase[] | null; + tasks: Task[] | null; + name: string; + description: string; + schedules: ScheduleRequest[]; + is_disabled: boolean | null; + folder: string | null; + }, +) => { + const clients = createKeboolaClients(config); + const project = await getProjectContext(clients); + assertConditionalAllowed(args.flow_type, project); + + let responseMessage: string | null = null; + const hasConfigChanges = + Boolean(args.name) || + Boolean(args.description) || + args.phases !== null || + args.tasks !== null || + args.is_disabled !== null; + + let apiConfig: RawConfig; + if (hasConfigChanges) { + logger.info(`Updating flow configuration: ${args.configuration_id} (type: ${args.flow_type})`); + // update_flow_internal: deep-clone the existing config, replace phases/tasks, validate. + const currentConfig = await configurationDetail(clients, args.flow_type, args.configuration_id); + const flowConfiguration = structuredClone( + (currentConfig.configuration as Record) ?? {}, + ); + const updated = getFlowConfiguration(args.phases, args.tasks, args.flow_type); + if ((updated.phases as unknown[]).length > 0) flowConfiguration.phases = updated.phases; + if ((updated.tasks as unknown[]).length > 0) flowConfiguration.tasks = updated.tasks; + + validateFlowStructure(flowConfiguration, args.flow_type); + const schema = await resolveFlowSchema(clients, args.flow_type, loadLegacySchema); + validateFlowConfigurationAgainstSchema(flowConfiguration, schema); + + apiConfig = await configurationUpdate( + clients, + args.flow_type, + args.configuration_id, + flowConfiguration, + args.change_description, + args.name || undefined, + args.description || undefined, + args.is_disabled, + ); + await setCfgUpdateMetadata( + clients, + args.flow_type, + String(apiConfig.id ?? ''), + Number(apiConfig.version ?? 0), + ); + } else { + apiConfig = await configurationDetail(clients, args.flow_type, args.configuration_id); + } + + // Folder handling. + let folderHint: string | null = null; + if (args.folder === null) { + try { + const { total, folders, lowerBound } = await getConfigFolders(clients, args.flow_type); + folderHint = buildFolderHint( + total, + folders, + flowLabel(args.flow_type), + 'modify_flow', + lowerBound, + ); + } catch { + logger.warn( + `Unable to fetch flow folders for component "${args.flow_type}" when updating flow "${args.configuration_id}".`, + ); + } + } else { + const folderStripped = args.folder.trim(); + if (folderStripped) { + await setConfigurationFolderMetadata( + clients, + args.flow_type, + args.configuration_id, + folderStripped, + ); + } else { + await clearConfigurationFolderMetadata(clients, args.flow_type, args.configuration_id); + } + } + + const linksManager = await createLinksManager(config, clients); + const flowLinks = linksManager.getFlowLinks( + String(apiConfig.id ?? ''), + String(apiConfig.name ?? ''), + args.flow_type, + ); + + if (args.schedules.length > 0) { + const scheduler = createSchedulerClient(config); + const responses = await processScheduleRequest( + clients, + scheduler, + args.flow_type, + args.configuration_id, + args.schedules, + ); + responseMessage = 'Schedules request processed successfully: \n' + responses.join('\n'); + logger.info( + `Successfully processed ${args.schedules.length} schedule request(s) for flow ${args.configuration_id}`, + ); + flowLinks.push(linksManager.getSchedulerDetailLink(args.configuration_id, args.flow_type)); + } + + logger.info(`Updated flow configuration: ${apiConfig.id}`); + return buildFlowToolOutput({ + configurationId: String(apiConfig.id ?? ''), + componentId: args.flow_type, + description: (apiConfig.description as string) || '', + version: Number(apiConfig.version ?? 0), + links: flowLinks, + response: responseMessage, + changeSummary: folderHint, + }); +}; + +// ============================================================================= +// REGISTRATION +// ============================================================================= + +export const registerFlowTools = (server: McpServer, config: Config): void => { + registerTool(server, { + name: 'create_flow', + title: 'Create flow', + description: CREATE_FLOW_DESCRIPTION, + annotations: { destructiveHint: false }, + inputSchema: { + name: z.string().describe('A short, descriptive name for the flow.'), + description: z.string().describe('Detailed description of the flow purpose.'), + phases: z.array(z.record(z.string(), z.unknown())).describe('List of phase definitions.'), + tasks: z.array(z.record(z.string(), z.unknown())).describe('List of task definitions.'), + folder: z.string().default('').describe(folderFieldDescription('flow', 'flows')), + }, + handler: (args) => + createFlowImpl(config, ORCHESTRATOR_COMPONENT_ID, { + name: args.name, + description: args.description, + phases: args.phases as Phase[], + tasks: args.tasks as Task[], + folder: args.folder, + }), + }); + + registerTool(server, { + name: 'create_conditional_flow', + title: 'Create conditional flow', + description: CREATE_CONDITIONAL_FLOW_DESCRIPTION, + annotations: { destructiveHint: false }, + inputSchema: { + name: z.string().describe('A short, descriptive name for the flow.'), + description: z.string().describe('Detailed description of the flow purpose.'), + phases: z + .array(z.record(z.string(), z.unknown())) + .describe('List of phase definitions for conditional flows.'), + tasks: z + .array(z.record(z.string(), z.unknown())) + .describe('List of task definitions for conditional flows.'), + folder: z.string().default('').describe(folderFieldDescription('flow', 'flows')), + }, + handler: async (args) => { + // Conditional flows require the feature to be enabled (parity with the Python tool's + // create_conditional_flow, which fails fast when the schema is unavailable). + const clients = createKeboolaClients(config); + const project = await getProjectContext(clients); + assertConditionalAllowed(CONDITIONAL_FLOW_COMPONENT_ID, project); + return createFlowImpl(config, CONDITIONAL_FLOW_COMPONENT_ID, { + name: args.name, + description: args.description, + phases: args.phases as Phase[], + tasks: args.tasks as Task[], + folder: args.folder, + }); + }, + }); + + registerTool(server, { + name: 'get_flows', + title: 'Get flows', + description: GET_FLOWS_DESCRIPTION, + annotations: { readOnlyHint: true }, + inputSchema: { + flow_ids: z + .array(z.string()) + .default([]) + .describe( + 'IDs of flows to retrieve full details for. ' + + 'When provided (non-empty), returns full flow configurations including phases and tasks. ' + + 'When empty [], lists all flows in the project as summaries.', + ), + }, + handler: async ({ flow_ids }) => { + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + const scheduler = createSchedulerClient(config); + + // Case 1: full details for specific flow ids. + if (flow_ids.length > 0) { + const flows = await Promise.all( + flow_ids.map(async (flowId) => { + const { raw, flowType } = await resolveFlowById(clients, flowId); + logger.info(`Found flow ${flowId} under flow type ${flowType}.`); + const configId = String(raw.id ?? ''); + const links = linksManager.getFlowLinks(configId, String(raw.name ?? ''), flowType); + const schedules = await listSchedulesForConfig(scheduler, flowType, configId); + const scheduleLink = linksManager.getSchedulerDetailLink(configId, flowType); + return toFlowDetail(raw, flowType, links, schedules, [scheduleLink]); + }), + ); + logger.info(`Retrieved full details for ${flows.length} flows.`); + return { flows }; + } + + // Case 2: list all flows as summaries. + const flows: ReturnType[] = []; + for (const flowType of FLOW_TYPES) { + const rawFlows = await configurationList(clients, flowType); + const summaries = await Promise.all( + rawFlows.map(async (raw) => { + let nSchedules = 0; + try { + const schedules = await listSchedulesForConfig( + scheduler, + flowType, + String(raw.id ?? ''), + ); + nSchedules = schedules.length; + } catch (e) { + logger.warn({ err: e }, `Failed to fetch schedules for flow ${raw.id}`); + } + return toFlowSummary(raw, flowType, nSchedules); + }), + ); + flows.push(...summaries); + } + logger.info(`Retrieved ${flows.length} flows.`); + return { + flows, + links: [ + linksManager.getFlowsDashboardLink(ORCHESTRATOR_COMPONENT_ID), + linksManager.getFlowsDashboardLink(CONDITIONAL_FLOW_COMPONENT_ID), + ], + }; + }, + }); + + registerTool(server, { + name: 'update_flow', + title: 'Update flow', + description: UPDATE_FLOW_DESCRIPTION, + annotations: { destructiveHint: true }, + inputSchema: { + configuration_id: z.string().describe('ID of the flow configuration.'), + flow_type: flowTypeSchema.describe( + 'The type of flow to update. Use "keboola.flow" for conditional flows or ' + + '"keboola.orchestrator" for legacy flows. This MUST match the existing flow type.', + ), + change_description: z.string().describe('Description of changes made.'), + phases: z + .array(z.record(z.string(), z.unknown())) + .nullish() + .describe('Updated list of phase definitions.'), + tasks: z + .array(z.record(z.string(), z.unknown())) + .nullish() + .describe('Updated list of task definitions.'), + name: z.string().default('').describe('Updated flow name. Only updated if provided.'), + description: z + .string() + .default('') + .describe('Updated flow description. Only updated if provided.'), + is_disabled: z + .boolean() + .nullish() + .describe( + "Enable or disable the flow. Set to True to disable execution (flow won't run), " + + 'False to enable execution (flow will run). Only provide if changing the status, ' + + 'leave as null to preserve current state.', + ), + folder: z.string().nullish().describe(folderFieldDescription('flow', 'flows')), + }, + handler: (args) => + modifyFlowImpl(config, { + configuration_id: args.configuration_id, + flow_type: args.flow_type, + change_description: args.change_description, + phases: (args.phases as Phase[] | null | undefined) ?? null, + tasks: (args.tasks as Task[] | null | undefined) ?? null, + name: args.name, + description: args.description, + schedules: [], + is_disabled: args.is_disabled ?? null, + folder: args.folder ?? null, + }), + }); + + registerTool(server, { + name: 'modify_flow', + title: 'Modify flow', + description: MODIFY_FLOW_DESCRIPTION, + annotations: { destructiveHint: true }, + inputSchema: { + configuration_id: z.string().describe('ID of the flow configuration.'), + flow_type: flowTypeSchema.describe( + 'The type of flow to update. Use "keboola.flow" for conditional flows or ' + + '"keboola.orchestrator" for legacy flows. This MUST match the existing flow type.', + ), + change_description: z.string().describe('Description of changes made.'), + phases: z + .array(z.record(z.string(), z.unknown())) + .nullish() + .describe('Updated list of phase definitions.'), + tasks: z + .array(z.record(z.string(), z.unknown())) + .nullish() + .describe('Updated list of task definitions.'), + name: z.string().default('').describe('Updated flow name. Only updated if provided.'), + description: z + .string() + .default('') + .describe('Updated flow description. Only updated if provided.'), + schedules: z + .array(scheduleRequestSchema) + .default([]) + .describe( + 'Optional sequence of schedule requests to add/update/remove schedules for this flow. ' + + 'Each request must have "action": "add"|"update"|"remove". ' + + 'For add: include "cron_tab", "state" ("enabled"|"disabled"), "timezone". ' + + 'For update/remove: include "schedule_id". ' + + 'Example: [{"action": "add", "cron_tab": "0 8 * * 1-5", "state": "enabled", "timezone": "UTC"}]', + ), + is_disabled: z + .boolean() + .nullish() + .describe( + "Enable or disable the flow. Set to True to disable execution (flow won't run), " + + 'False to enable execution (flow will run). Only provide if changing the status, ' + + 'leave as null to preserve current state.', + ), + folder: z.string().nullish().describe(folderFieldDescription('flow', 'flows')), + }, + handler: (args) => + modifyFlowImpl(config, { + configuration_id: args.configuration_id, + flow_type: args.flow_type, + change_description: args.change_description, + phases: (args.phases as Phase[] | null | undefined) ?? null, + tasks: (args.tasks as Task[] | null | undefined) ?? null, + name: args.name, + description: args.description, + schedules: normalizeScheduleRequests(args.schedules), + is_disabled: args.is_disabled ?? null, + folder: args.folder ?? null, + }), + }); + + registerTool(server, { + name: 'get_flow_schema', + title: 'Get flow schema', + description: GET_FLOW_SCHEMA_DESCRIPTION, + annotations: { readOnlyHint: true }, + inputSchema: { + flow_type: flowTypeSchema.describe('The type of flow for which to fetch schema.'), + }, + handler: async ({ flow_type }) => { + const clients = createKeboolaClients(config); + const project = await getProjectContext(clients); + assertConditionalAllowed(flow_type, project); + logger.info(`Returning flow configuration schema for flow type: ${flow_type}`); + return getSchemaAsMarkdown(clients, flow_type); + }, + }); + + registerTool(server, { + name: 'get_flow_examples', + title: 'Get flow examples', + description: GET_FLOW_EXAMPLES_DESCRIPTION, + annotations: { readOnlyHint: true }, + inputSchema: { + flow_type: flowTypeSchema.describe('The type of the flow to retrieve examples for.'), + }, + handler: async ({ flow_type }) => { + const clients = createKeboolaClients(config); + const project = await getProjectContext(clients); + assertConditionalAllowed(flow_type, project, true); + + const content = readResource(EXAMPLE_FILES[flow_type]); + let markdown = `# Flow Configuration Examples for \`${flow_type}\`\n\n`; + const lines = content.split('\n').filter((line) => line.trim().length > 0); + lines.forEach((line, i) => { + const data = JSON.parse(line); + markdown += `${i + 1}. Flow Configuration:\n\`\`\`json\n${JSON.stringify(data, null, 2)}\n\`\`\`\n\n`; + }); + return markdown; + }, + }); + + // debug, not info: createServer() runs per HTTP request, so this fires on every request. + logger.debug('Flow tools initialized.'); +}; diff --git a/src/tools/flow/utils.ts b/src/tools/flow/utils.ts new file mode 100644 index 000000000..b59e102d2 --- /dev/null +++ b/src/tools/flow/utils.ts @@ -0,0 +1,575 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import type { KeboolaClients } from '@/clients/keboola'; +import { + CONDITIONAL_FLOW_COMPONENT_ID, + FLOW_TYPES, + type FlowType, + MetadataField, + ORCHESTRATOR_COMPONENT_ID, +} from '@/constants'; +import type { Link } from '@/links'; +import { logger } from '@/logger'; +import { resourcePath } from '@/resource-path'; +import { + CREATED_BY_MCP, + type MetadataItem, + normalizeDependsOn, + type Phase, + type RawConfig, + type Task, + UPDATED_BY_MCP_PREFIX, +} from './model'; +import type { toScheduleDetail } from './scheduler'; + +// Ported from tools/flow/utils.py + the storage/metadata helpers from components/utils.py. + +// ============================================================================= +// RESOURCES +// ============================================================================= +// +// The bundled legacy flow schema and the example files are copied from the Python +// `keboola_mcp_server/resources` tree into `src/resources/flow/` (a path this module +// owns) and read at runtime via `fs` relative to the module's own location. This keeps +// them on disk (not inlined) and works under both vitest (running from `src/`) and a +// built `dist/` once the build copies the folder. + +const RESOURCES_DIR = resourcePath('flow'); + +export const readResource = (filename: string): string => + readFileSync(join(RESOURCES_DIR, filename), 'utf-8'); + +let cachedLegacySchema: Record | undefined; +export const loadLegacySchema = (): Record => { + cachedLegacySchema ??= JSON.parse(readResource('flow-schema.json')) as Record; + return cachedLegacySchema; +}; + +export const EXAMPLE_FILES: Record = { + [CONDITIONAL_FLOW_COMPONENT_ID]: 'conditional_flow_examples.jsonl', + [ORCHESTRATOR_COMPONENT_ID]: 'legacy_flow_examples.jsonl', +}; + +// ============================================================================= +// STORAGE CONFIGURATION HELPERS +// ============================================================================= +// +// Component-config CRUD uses the typed `@keboola/api-client` storage client +// (`clients.storage.componentsAndConfigurations.*`). Configuration *metadata* +// (GET/POST/DELETE `.../configs/{id}/metadata`) has no typed subpath in api-client, +// so those calls stay on the raw Storage client (mirrors Python's storage_client). + +export const configurationCreate = ( + clients: KeboolaClients, + componentId: string, + name: string, + description: string, + configuration: Record, +): Promise => + clients.storage.componentsAndConfigurations.createConfiguration({ + branchId: clients.branchId, + componentId, + name, + description, + configuration, + }) as Promise; + +export const configurationUpdate = ( + clients: KeboolaClients, + componentId: string, + configurationId: string, + configuration: Record, + changeDescription: string, + updatedName?: string, + updatedDescription?: string, + isDisabled?: boolean | null, +): Promise => + clients.storage.componentsAndConfigurations.updateConfiguration({ + branchId: clients.branchId, + componentId, + configId: configurationId, + configuration, + changeDescription, + ...(updatedName ? { name: updatedName } : {}), + ...(updatedDescription ? { description: updatedDescription } : {}), + ...(isDisabled !== undefined && isDisabled !== null ? { isDisabled } : {}), + }) as Promise; + +export const configurationDetail = ( + clients: KeboolaClients, + componentId: string, + configurationId: string, +): Promise => + clients.storage.componentsAndConfigurations.getConfiguration({ + branchId: clients.branchId, + componentId, + configId: configurationId, + }) as Promise; + +export const configurationList = ( + clients: KeboolaClients, + componentId: string, +): Promise => + clients.storage.componentsAndConfigurations.getConfigurations({ + branchId: clients.branchId, + componentId, + }) as Promise; + +export const configurationDelete = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, +): Promise => { + await clients.storage.componentsAndConfigurations.deleteConfiguration({ + branchId: clients.branchId, + componentId, + configId: configurationId, + }); +}; + +// --- Configuration metadata (KEPT RAW: no typed api-client subpath for config metadata) --- + +const configBase = (clients: KeboolaClients, componentId: string): string => + `branch/${clients.branchId}/components/${componentId}/configs`; + +const configurationMetadataGet = ( + clients: KeboolaClients, + componentId: string, + configurationId: string, +): Promise => + clients.rawStorage.get( + `${configBase(clients, componentId)}/${configurationId}/metadata`, + ); + +const configurationMetadataUpdate = ( + clients: KeboolaClients, + componentId: string, + configurationId: string, + metadata: Record, +): Promise => + clients.rawStorage.post( + `${configBase(clients, componentId)}/${configurationId}/metadata`, + { body: { metadata: Object.entries(metadata).map(([key, value]) => ({ key, value })) } }, + ); + +const configurationMetadataDelete = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, + metadataId: string, +): Promise => { + await clients.rawStorage.delete( + `${configBase(clients, componentId)}/${configurationId}/metadata/${metadataId}`, + ); +}; + +const componentConfigurationsSearch = async ( + clients: KeboolaClients, + componentId: string | undefined, + metadataKeys: string[], +): Promise => { + if (!componentId && metadataKeys.length === 0) return []; + return clients.storage.componentsAndConfigurations.searchComponentConfigurations( + { branchId: clients.branchId }, + { + ...(componentId ? { componentId } : {}), + ...(metadataKeys.length > 0 ? { metadataKeys } : {}), + }, + ) as Promise; +}; + +export const metadataProperty = ( + metadata: MetadataItem[] | undefined, + key: string, +): string | undefined => { + // Most-recent wins (mirrors get_metadata_property): iterate and keep the last match. + let value: string | undefined; + for (const item of metadata ?? []) { + if (item.key === key) value = item.value; + } + return value; +}; + +// --- MCP creation/update + folder metadata helpers (port of components/utils.py) --- + +export const setCfgCreationMetadata = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, +): Promise => { + try { + await configurationMetadataUpdate(clients, componentId, configurationId, { + [CREATED_BY_MCP]: 'true', + }); + } catch (e) { + logger.error( + { err: e }, + `Failed to set "${CREATED_BY_MCP}" metadata for configuration ${configurationId}`, + ); + } +}; + +export const setCfgUpdateMetadata = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, + configurationVersion: number, +): Promise => { + const key = `${UPDATED_BY_MCP_PREFIX}${configurationVersion}`; + try { + await configurationMetadataUpdate(clients, componentId, configurationId, { [key]: 'true' }); + } catch (e) { + logger.error( + { err: e }, + `Failed to set "${key}" metadata for configuration ${configurationId}`, + ); + } +}; + +const setConfigurationFolderMetadata = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, + folder: string, +): Promise => { + const normalized = folder.trim(); + if (!normalized) return; + await configurationMetadataUpdate(clients, componentId, configurationId, { + [MetadataField.CONFIGURATION_FOLDER_NAME]: normalized, + }); +}; + +const clearConfigurationFolderMetadata = async ( + clients: KeboolaClients, + componentId: string, + configurationId: string, +): Promise => { + const metadata = await configurationMetadataGet(clients, componentId, configurationId); + for (const entry of metadata) { + if (entry.key === MetadataField.CONFIGURATION_FOLDER_NAME) { + if (entry.id === undefined) { + logger.warn( + `Unable to clear folder metadata for component "${componentId}", configuration "${configurationId}": metadata entry is missing "id".`, + ); + continue; + } + await configurationMetadataDelete(clients, componentId, configurationId, entry.id); + } + } +}; + +export { setConfigurationFolderMetadata, clearConfigurationFolderMetadata }; + +export const getConfigFolders = async ( + clients: KeboolaClients, + componentId: string, +): Promise<{ total: number; folders: string[]; lowerBound: boolean }> => { + const folderConfigs = await componentConfigurationsSearch(clients, componentId, [ + MetadataField.CONFIGURATION_FOLDER_NAME, + ]); + const seen = new Set(); + const folders: string[] = []; + for (const cfg of folderConfigs) { + for (const meta of (cfg.metadata as MetadataItem[]) ?? []) { + if (meta.key === MetadataField.CONFIGURATION_FOLDER_NAME) { + const name = (meta.value ?? '').trim(); + if (name && !seen.has(name)) { + seen.add(name); + folders.push(name); + } + } + } + } + if (folderConfigs.length >= 20) { + return { total: folderConfigs.length, folders, lowerBound: true }; + } + const rawConfigs = await configurationList(clients, componentId); + const total = rawConfigs.length; + if (total < 20) return { total, folders: [], lowerBound: false }; + return { total, folders, lowerBound: false }; +}; + +export const folderFieldDescription = (singular: string, plural: string): string => + `Folder name to organize this ${singular} in the Keboola UI. ` + + `Pass an empty string to remove an existing folder assignment. ` + + `Existing folder names are returned in the response change_summary when no folder is provided ` + + `and there are 20 or more ${plural} in the project. ` + + `If there are 20 or more ${plural}, you should assign one of the existing folders or ` + + `create a new one that clearly reflects the ${singular} purpose.`; + +export const buildFolderHint = ( + total: number, + existingFolders: string[], + configLabel: string, + updateTool: string, + lowerBound = false, +): string | null => { + if (total < 20) return null; + const countStr = lowerBound ? `at least ${total}` : String(total); + let hint = `Note: This project already has ${countStr} ${configLabel}. Consider organizing them with folders. `; + if (existingFolders.length > 0) { + hint += + `Existing folders: ${existingFolders.join(', ')}. ` + + `Call ${updateTool} with a folder= parameter to assign this to one.`; + } else { + hint += `No folders have been created yet. Call ${updateTool} with a folder= parameter to start organizing.`; + } + return hint; +}; + +// ============================================================================= +// PROJECT CONTEXT (only the bits the flow tools need: name + conditional_flows) +// ============================================================================= + +export type ProjectContext = { projectName: string; conditionalFlows: boolean }; + +export const getProjectContext = async (clients: KeboolaClients): Promise => { + const token = (await clients.storage.tokens.verify()) as { + owner?: { name?: string; features?: unknown }; + }; + const owner = token.owner ?? {}; + const features = owner.features; + // `conditional_flows` is enabled unless the project carries the `hide-conditional-flows` feature. + let hidden = false; + if (Array.isArray(features)) { + hidden = features.includes('hide-conditional-flows'); + } else if (features && typeof features === 'object') { + hidden = 'hide-conditional-flows' in (features as Record); + } + return { projectName: owner.name ?? '', conditionalFlows: !hidden }; +}; + +export const assertConditionalAllowed = ( + flowType: FlowType, + project: ProjectContext, + examples = false, +): void => { + if (flowType === CONDITIONAL_FLOW_COMPONENT_ID && !project.conditionalFlows) { + throw new Error( + `Conditional flows are not supported in this project. ` + + `Project "${project.projectName}" has conditional_flows=false. ` + + `If you want to use conditional flows, please enable them in your project settings. ` + + `Otherwise, use flow_type="${ORCHESTRATOR_COMPONENT_ID}" for legacy flow${examples ? ' examples' : 's'} instead.`, + ); + } +}; + +// ============================================================================= +// FLOW CONFIGURATION BUILDING (port of utils.get_flow_configuration + ensure_*) +// ============================================================================= + +/** Legacy phase normalization (port of ensure_legacy_phase_ids). */ +const ensureLegacyPhaseIds = (phases: Phase[]): Phase[] => { + const processed: Phase[] = []; + const usedIds = new Set(); + + phases.forEach((phase, i) => { + const data: Phase = { ...phase }; + if (data.id === undefined || data.id === null || data.id === '' || data.id === 0) { + let phaseId = i + 1; + while (usedIds.has(phaseId)) phaseId += 1; + data.id = phaseId; + } + if (data.name === undefined) data.name = `Phase ${data.id}`; + + if (typeof data.name !== 'string' || data.name.length < 1) { + throw new Error(`Invalid phase configuration: phase ${data.id} has an invalid name.`); + } + const normalized: Phase = { + id: data.id, + name: data.name, + description: typeof data.description === 'string' ? data.description : '', + dependsOn: normalizeDependsOn(data), + }; + usedIds.add(normalized.id as string | number); + processed.push(normalized); + }); + + return processed; +}; + +/** Legacy task normalization (port of ensure_legacy_task_ids). */ +const ensureLegacyTaskIds = (tasks: Task[]): Task[] => { + const processed: Task[] = []; + const usedIds = new Set(); + // Phase IDs are small sequential numbers; task IDs start at 20001 to avoid collisions. + let taskCounter = 20001; + + for (const task of tasks) { + const data: Task = { ...task }; + if (data.id === undefined || data.id === null || data.id === '' || data.id === 0) { + while (usedIds.has(taskCounter)) taskCounter += 1; + data.id = taskCounter; + taskCounter += 1; + } + if (data.name === undefined) data.name = `Task ${data.id}`; + if (data.task === undefined) { + throw new Error(`Task ${data.id} missing 'task' configuration`); + } + const taskObj = (data.task as Record) ?? {}; + if (taskObj.componentId === undefined) { + throw new Error(`Task ${data.id} missing componentId in task configuration`); + } + if (taskObj.mode === undefined) taskObj.mode = 'run'; + data.task = taskObj; + + if (data.phase === undefined || data.phase === null) { + throw new Error(`Invalid task configuration: task ${data.id} missing phase.`); + } + + const normalized: Task = { + id: data.id, + name: data.name, + phase: data.phase, + enabled: data.enabled ?? true, + continueOnFailure: + data.continueOnFailure ?? data.continue_on_failure ?? data['continue-on-failure'] ?? false, + task: data.task, + }; + usedIds.add(normalized.id as string | number); + processed.push(normalized); + } + + return processed; +}; + +/** + * Conditional phase `model_dump(exclude_unset=True)` semantics: drop `next` when it is empty + * or a single transition with goto=null (lets the Designer UI render ending phases cleanly). + */ +const dumpConditionalPhase = (phase: Phase): Phase => { + const out: Phase = { ...phase }; + // Normalize a `next` provided as null/undefined to omitted. + if (out.next === undefined || out.next === null) { + delete out.next; + } else if (Array.isArray(out.next)) { + const next = out.next as Record[]; + if (next.length === 0) { + delete out.next; + } else if (next.length === 1 && (next[0]?.goto === null || next[0]?.goto === undefined)) { + delete out.next; + } + } + return out; +}; + +/** Port of utils.get_flow_configuration. */ +export const getFlowConfiguration = ( + phases: Phase[] | null, + tasks: Task[] | null, + flowType: FlowType, +): Record => { + if (flowType === ORCHESTRATOR_COMPONENT_ID) { + return { + phases: ensureLegacyPhaseIds(phases ?? []), + tasks: ensureLegacyTaskIds(tasks ?? []), + }; + } + return { + phases: (phases ?? []).map(dumpConditionalPhase), + tasks: tasks ?? [], + }; +}; + +// ============================================================================= +// READ-PATH MODELS (port of model.Flow / FlowSummary) +// ============================================================================= + +export const toFlowSummary = (raw: RawConfig, flowComponentId: FlowType, nSchedules: number) => { + const config = (raw.configuration as Record) ?? {}; + const metadata = (raw.metadata as MetadataItem[]) ?? []; + return { + component_id: flowComponentId, + configuration_id: String(raw.id ?? ''), + name: raw.name ?? '', + description: raw.description ?? null, + version: raw.version ?? 0, + is_disabled: raw.isDisabled ?? false, + is_deleted: raw.isDeleted ?? false, + phases_count: ((config.phases as unknown[]) ?? []).length, + tasks_count: ((config.tasks as unknown[]) ?? []).length, + schedules_count: nSchedules, + folder: metadataProperty(metadata, MetadataField.CONFIGURATION_FOLDER_NAME) ?? '', + created: raw.created ?? null, + updated: raw.updated ?? null, + }; +}; + +export const toFlowDetail = ( + raw: RawConfig, + flowComponentId: FlowType, + links: Link[], + schedules: ReturnType[], + scheduleLinks: Link[], +) => { + const config = (raw.configuration as Record) ?? {}; + const metadata = (raw.metadata as MetadataItem[]) ?? []; + return { + component_id: flowComponentId, + configuration_id: String(raw.id ?? ''), + name: raw.name ?? '', + description: raw.description ?? null, + version: raw.version ?? 0, + is_disabled: raw.isDisabled ?? false, + is_deleted: raw.isDeleted ?? false, + configuration: { + phases: (config.phases as unknown[]) ?? [], + tasks: (config.tasks as unknown[]) ?? [], + }, + change_description: raw.changeDescription ?? null, + configuration_metadata: metadata, + folder: metadataProperty(metadata, MetadataField.CONFIGURATION_FOLDER_NAME) ?? '', + created: raw.created ?? null, + updated: raw.updated ?? null, + schedules: { + schedules, + n_schedules: schedules.length, + links: scheduleLinks, + }, + links, + }; +}; + +/** Resolve a flow across all flow types (port of utils.resolve_flow_by_id). */ +export const resolveFlowById = async ( + clients: KeboolaClients, + flowId: string, +): Promise<{ raw: RawConfig; flowType: FlowType }> => { + for (const flowType of FLOW_TYPES) { + try { + const raw = await configurationDetail(clients, flowType, flowId); + return { raw, flowType }; + } catch { + continue; + } + } + throw new Error(`Flow configuration "${flowId}" not found`); +}; + +// ============================================================================= +// TOOL OUTPUT BUILDING +// ============================================================================= + +export const buildFlowToolOutput = (opts: { + configurationId: string; + componentId: string; + description: string; + version: number; + links: Link[]; + response?: string | null; + changeSummary?: string | null; +}) => ({ + configuration_id: opts.configurationId, + component_id: opts.componentId, + description: opts.description, + version: opts.version, + timestamp: new Date().toISOString(), + response: opts.response ?? null, + change_summary: opts.changeSummary ?? null, + success: true, + links: opts.links, +}); + +export const flowLabel = (flowType: FlowType): string => + flowType === ORCHESTRATOR_COMPONENT_ID ? 'legacy flows' : 'conditional flows'; diff --git a/src/tools/jobs.ts b/src/tools/jobs.ts new file mode 100644 index 000000000..913e98306 --- /dev/null +++ b/src/tools/jobs.ts @@ -0,0 +1,216 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { createKeboolaClients, createLinksManager } from '@/clients/keboola'; +import type { Config } from '@/config'; +import type { Link } from '@/links'; +import { logger } from '@/logger'; +import { registerTool } from '@/mcp/tool'; + +// Ported from tools/jobs.py. + +const JOB_STATUS = [ + 'waiting', + 'processing', + 'success', + 'error', + 'created', + 'warning', + 'terminating', + 'cancelled', + 'terminated', +] as const; + +const SORT_BY = ['startTime', 'endTime', 'createdTime', 'durationSeconds', 'id'] as const; +const SORT_ORDER = ['asc', 'desc'] as const; +const LOG_EVENT_TYPES = ['info', 'warn', 'error', 'success'] as const; + +type RawJob = Record; + +/** result/config_data must be an object: empty list / null become {} (port of validate_dict_fields). */ +const asDict = (value: unknown): Record => { + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value as Record; + } + if (Array.isArray(value) && value.length > 0) { + throw new Error( + `Field "result" or "config_data" cannot be a list, expecting dictionary, got: ${JSON.stringify(value)}.`, + ); + } + return {}; +}; + +const toJobListItem = (raw: RawJob) => ({ + id: String(raw.id ?? ''), + status: raw.status, + componentId: raw.component ?? raw.componentId ?? null, + configId: raw.config ?? raw.configId ?? null, + isFinished: raw.isFinished ?? false, + createdTime: raw.createdTime ?? null, + startTime: raw.startTime ?? null, + endTime: raw.endTime ?? null, + durationSeconds: raw.durationSeconds ?? null, +}); + +const toJobDetail = (raw: RawJob, links: Link[]) => ({ + ...toJobListItem(raw), + url: raw.url ?? '', + configData: raw.configData != null ? asDict(raw.configData) : null, + configRow: raw.configRow ?? null, + runId: raw.runId ?? null, + result: raw.result != null ? asDict(raw.result) : null, + links, + logs: null as { message: unknown; type: unknown; created: unknown }[] | null, +}); + +export const registerJobTools = (server: McpServer, config: Config): void => { + registerTool(server, { + name: 'get_jobs', + title: 'Get jobs', + description: 'Retrieves job execution information from the Keboola project.', + annotations: { readOnlyHint: true }, + inputSchema: { + job_ids: z + .array(z.string()) + .default([]) + .describe('IDs of jobs to retrieve full details for; empty lists jobs as summaries.'), + status: z + .enum(JOB_STATUS) + .optional() + .describe('Filter listed jobs by status (ignored if job_ids given).'), + component_id: z + .string() + .optional() + .describe('Filter listed jobs by component id (ignored if job_ids given).'), + config_id: z + .string() + .optional() + .describe('Filter listed jobs by configuration id (ignored if job_ids given).'), + limit: z + .number() + .int() + .min(1) + .max(500) + .default(100) + .describe('Number of jobs to list (max 500).'), + offset: z.number().int().min(0).default(0).describe('Offset of jobs to list.'), + sort_by: z.enum(SORT_BY).default('startTime').describe('Field to sort listed jobs by.'), + sort_order: z.enum(SORT_ORDER).default('desc').describe('Sort order for listed jobs.'), + include_logs: z + .boolean() + .default(false) + .describe('Include execution logs (only when job_ids given).'), + log_tail_lines: z + .number() + .int() + .min(1) + .max(500) + .default(50) + .describe('Max log events per job (most recent).'), + log_event_types: z + .array(z.enum(LOG_EVENT_TYPES)) + .optional() + .describe('Filter log events by type (only when include_logs=true).'), + }, + handler: async (args) => { + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + + // MODE 1: full details for specific job ids. + if (args.job_ids.length > 0) { + const jobs = await Promise.all( + args.job_ids.map(async (jobId) => { + const raw = (await clients.queue.getJob(jobId)) as RawJob; + return toJobDetail(raw, linksManager.getJobLinks(jobId)); + }), + ); + + if (args.include_logs) { + await Promise.all( + jobs.map(async (job) => { + if (!job.id) return; + const raw = (await clients.storage.events.getEvents({ + runId: job.id, + limit: args.log_tail_lines, + offset: 0, + forceUuid: 'true', + } as never)) as RawJob[]; + const typeSet = args.log_event_types ? new Set(args.log_event_types) : null; + const events = (Array.isArray(raw) ? raw : []) + .filter((event) => !typeSet || typeSet.has(event.type as string)) + .reverse(); + job.logs = events.map((event) => ({ + message: event.message, + type: event.type, + created: event.created, + })); + }), + ); + } + + logger.info(`Retrieved full details for ${jobs.length} jobs.`); + return { jobs }; + } + + // MODE 2: list summaries with optional filtering. Queue uses the raw branch id + // (omitted on production) — not the storage `default` alias. + const query: Record = { + branchId: config.branchId, + componentId: args.component_id, + configId: args.config_id, + status: args.status ? [args.status] : undefined, + limit: args.limit, + offset: args.offset, + sortBy: args.sort_by, + sortOrder: args.sort_order, + }; + for (const key of Object.keys(query)) { + if (query[key] === undefined) delete query[key]; + } + + const raw = (await clients.queue.searchJobs(query as never)) as unknown; + const items = Array.isArray(raw) ? raw : ((raw as { jobs?: RawJob[] }).jobs ?? []); + logger.info(`Found ${items.length} jobs.`); + return { + jobs: (items as RawJob[]).map(toJobListItem), + links: [linksManager.getJobsDashboardLink()], + }; + }, + }); + + registerTool(server, { + name: 'run_job', + title: 'Run job', + description: 'Starts a new job for a given component or transformation.', + annotations: { destructiveHint: true }, + inputSchema: { + component_id: z + .string() + .describe('The ID of the component or transformation to start a job for.'), + configuration_id: z.string().describe('The ID of the configuration to start a job for.'), + configuration_row_ids: z + .array(z.string()) + .optional() + .describe('Optional configuration row IDs to run; if omitted, all rows are executed.'), + }, + handler: async (args) => { + const clients = createKeboolaClients(config); + + const payload: Record = { + component: args.component_id, + config: args.configuration_id, + mode: 'run', + }; + if (config.branchId) payload.branchId = config.branchId; + if (args.configuration_row_ids?.length) payload.configRowIds = args.configuration_row_ids; + + const raw = await clients.rawQueue.post('jobs', { body: payload }); + const linksManager = await createLinksManager(config, clients); + const job = toJobDetail(raw, linksManager.getJobLinks(String(raw.id ?? ''))); + logger.info( + `Started a new job with id: ${job.id} for component ${args.component_id} and configuration ${args.configuration_id}.`, + ); + return job; + }, + }); +}; diff --git a/src/tools/oauth.ts b/src/tools/oauth.ts new file mode 100644 index 000000000..7649e9010 --- /dev/null +++ b/src/tools/oauth.ts @@ -0,0 +1,40 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { createKeboolaClients } from '@/clients/keboola'; +import type { Config } from '@/config'; +import { registerTool } from '@/mcp/tool'; + +// Ported from tools/oauth.py. + +export const registerOAuthTools = (server: McpServer, config: Config): void => { + registerTool(server, { + name: 'create_oauth_url', + title: 'Create OAuth URL', + description: 'Generates an OAuth authorization URL for a Keboola component configuration.', + inputSchema: { + component_id: z + .string() + .describe('The component ID to grant access to (e.g., "keboola.ex-google-analytics-v4").'), + config_id: z.string().describe('The configuration ID for the component.'), + }, + handler: async ({ component_id, config_id }) => { + const { rawStorage } = createKeboolaClients(config); + + // Short-lived (1h) token scoped to the component, used by the external OAuth page. + const tokenResponse = await rawStorage.post<{ token: string }>('tokens', { + body: { + description: `Short-lived token for OAuth URL - ${component_id}/${config_id}`, + componentAccess: [component_id], + expiresIn: 3600, + }, + }); + + const query = new URLSearchParams({ + token: tokenResponse.token, + sapiUrl: config.storageApiUrl ?? '', + }); + return `https://external.keboola.com/oauth/index.html?${query.toString()}#/${component_id}/${config_id}`; + }, + }); +}; diff --git a/src/tools/project.ts b/src/tools/project.ts new file mode 100644 index 000000000..ceae06815 --- /dev/null +++ b/src/tools/project.ts @@ -0,0 +1,243 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { readFileSync } from 'node:fs'; +import { z } from 'zod'; + +import { createKeboolaClients, createLinksManager, type KeboolaClients } from '@/clients/keboola'; +import type { Config } from '@/config'; +import { MetadataField } from '@/constants'; +import type { Link } from '@/links'; +import { logger } from '@/logger'; +import { registerTool } from '@/mcp/tool'; +import { resourcePath } from '@/resource-path'; +import { createWorkspaceManager } from '@/tools/sql'; + +// Ported from tools/project.py. + +// --- Project system prompt (LLM instruction) --------------------------------- +// The base prompt markdown lives next to this module's source. It is loaded once +// at runtime via fs, resolved relative to this module (import.meta.url), matching +// the Python `get_project_system_prompt` which reads the packaged resource file. +const PROMPT_PATH = resourcePath('prompts', 'project_system_prompt.md'); + +let cachedBasePrompt: string | undefined; +const loadBasePrompt = (): string => { + if (cachedBasePrompt === undefined) { + cachedBasePrompt = readFileSync(PROMPT_PATH, 'utf-8'); + } + return cachedBasePrompt; +}; + +// Port of resources/prompts/__init__.py `_DIALECT_CONFIGS`. +type DialectConfig = { + delimiter: string; + col: string; + fqn: string; + newTable: string; + extra: string[]; +}; + +const DIALECT_CONFIGS: Record = { + BigQuery: { + delimiter: 'backtick (`` ` ``)', + col: '`column_name`', + fqn: '`project`.`dataset`.`table`', + newTable: '`table_name`', + extra: [], + }, + Snowflake: { + delimiter: 'double quote (`"`)', + col: '"column_name"', + fqn: '"DATABASE"."SCHEMA"."TABLE"', + newTable: '"table_name"', + extra: [ + 'Unquoted identifiers and column aliases are auto-uppercased by Snowflake — ' + + 'always use delimited identifiers to preserve case.', + 'Use `LISTAGG` instead of `STRING_AGG`.', + 'In CTEs, use delimited identifiers for every column alias so the name survives ' + + 'into the outer query unchanged.', + ], + }, +}; + +const buildDialectSection = (sqlDialect: string): string => { + const cfg = DIALECT_CONFIGS[sqlDialect]; + if (!cfg) { + logger.warn( + `Unknown SQL dialect ${JSON.stringify(sqlDialect)} — no dialect-specific identifier guidance will be emitted.`, + ); + return `### SQL Identifiers\n\nSQL dialect: **${sqlDialect}**.\n`; + } + const lines = [ + '### SQL Identifiers\n', + `This project uses **${sqlDialect}** SQL dialect.`, + `The delimited identifier character is the ${cfg.delimiter}.`, + '**Always wrap every identifier** (column name, table name, alias) in delimited identifiers:\n', + `- Column reference: ${cfg.col}`, + `- Fully qualified table name: ${cfg.fqn}`, + `- New table in CREATE TABLE (table name only, no FQN): ${cfg.newTable}`, + '- Never mix delimiter styles within a single query.\n', + ]; + for (const note of cfg.extra) { + lines.push(`- ${note}`); + } + return lines.join('\n'); +}; + +/** Port of `get_project_system_prompt`. */ +const getProjectSystemPrompt = (sqlDialect = ''): string => { + const base = loadBasePrompt(); + if (!sqlDialect) { + return base; + } + return `${buildDialectSection(sqlDialect)}\n\n---\n\n${base}`; +}; + +// --- Toolset restrictions (port of `_get_toolset_restrictions`) -------------- +const getToolsetRestrictions = (role: string): string | null => { + const r = role.toLowerCase(); + if (r === 'readonly') { + return ( + `Your Keboola user role is "${r}". ` + + 'Only read-only tools are available. ' + + 'All write operations (creating, updating, or deleting resources) are disabled.' + ); + } + if (!r || r === 'unknown') { + return 'Your Keboola user role is unknown. You can manage flows but cannot set their schedules.'; + } + if (r !== 'admin' && r !== 'share') { + return `Your Keboola user role is "${r}". You can manage flows but cannot set their schedules.`; + } + return null; +}; + +// --- Branch context resolution (port of `_resolve_branch_context`) ----------- +type BranchEntry = { id?: string | number; name?: string; isDefault?: boolean }; + +/** + * Resolves the current branch's id, name, and dev-branch flag from the storage API. + * The effective branch id is `config.branchId` (undefined on the default/production + * branch), so we list branches and pick the matching entry or the `isDefault` one. + */ +const resolveBranchContext = async ( + config: Config, + clients: KeboolaClients, +): Promise<[string | number, string, boolean]> => { + const targetBranchId = config.branchId; + const branches = (await clients.storage.branches.getDevBranches()) as BranchEntry[]; + + let selected: BranchEntry | undefined; + for (const branch of branches) { + if (targetBranchId === undefined) { + if (branch.isDefault === true) { + selected = branch; + break; + } + } else if (String(branch.id) === String(targetBranchId)) { + selected = branch; + break; + } + } + + if (selected === undefined) { + // Should not happen in a healthy project, but stay defensive. + const fallbackId: string | number = targetBranchId !== undefined ? targetBranchId : 'default'; + return [fallbackId, 'unknown', targetBranchId !== undefined]; + } + + const branchId = selected.id ?? (targetBranchId !== undefined ? targetBranchId : 'default'); + const branchName = selected.name ?? 'unknown'; + const isDevelopmentBranch = selected.isDefault !== true; + return [branchId, branchName, isDevelopmentBranch]; +}; + +/** Registers the project tools (Plan §4). Ported from tools/project.py. */ +export const registerProjectTools = (server: McpServer, config: Config): void => { + registerTool(server, { + name: 'update_project_description', + title: 'Update project description', + description: 'Updates the description of the current Keboola project.', + annotations: { destructiveHint: true }, + inputSchema: { + description: z.string().describe('The new project description text.'), + }, + handler: async ({ description }) => { + const clients = createKeboolaClients(config); + await clients.storage.branches.saveDevBranchMetadata(clients.branchId, [ + { key: MetadataField.PROJECT_DESCRIPTION, value: description }, + ]); + logger.info('Project description updated successfully.'); + return { message: 'Project description updated successfully.' }; + }, + }); + + registerTool(server, { + name: 'get_project_info', + title: 'Get project info', + description: + 'Retrieves structured information about the current project, ' + + 'including essential context and base instructions for working with it ' + + '(e.g., transformations, components, workflows, and dependencies).\n\n' + + 'Always call this tool at least once at the start of a conversation ' + + 'to establish the project context before using other tools.', + annotations: { readOnlyHint: true }, + handler: async () => { + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + + const tokenData = (await clients.storage.tokens.verify()) as Record; + const projectData = (tokenData.owner ?? {}) as Record; + const projectId = (projectData.id ?? '') as string | number; + const projectName = (projectData.name ?? '') as string; + + const organizationData = (tokenData.organization ?? {}) as Record; + const organizationId = (organizationData.id ?? '') as string | number; + + const adminData = (tokenData.admin ?? {}) as Record; + const userRole = (adminData.role as string) || 'unknown'; + + const metadata = (await clients.storage.branches.getDevBranchMetadata(clients.branchId)) as { + key: string; + value: string; + }[]; + const description = + metadata.find((item) => item.key === MetadataField.PROJECT_DESCRIPTION)?.value ?? ''; + + // Resolve sql_dialect + workspace_id via the shared WorkspaceManager, which finds the + // MCP read-only workspace (by configured schema or branch metadata) and creates one + // when absent — matching the Python get_project_info / query_data behavior. + const workspaceManager = await createWorkspaceManager(config); + const sqlDialect = await workspaceManager.getSqlDialect(); + const workspaceId = await workspaceManager.getWorkspaceId(); + + const projectFeatures = (projectData.features ?? {}) as Record | unknown[]; + const conditionalFlows = Array.isArray(projectFeatures) + ? !projectFeatures.includes('hide-conditional-flows') + : !('hide-conditional-flows' in projectFeatures); + const links: Link[] = linksManager.getProjectLinks(); + + const [branchId, branchName, isDevelopmentBranch] = await resolveBranchContext( + config, + clients, + ); + + logger.info('Returning unified project info.'); + return { + project_id: projectId, + project_name: projectName, + project_description: description, + organization_id: organizationId, + sql_dialect: sqlDialect, + workspace_id: workspaceId, + conditional_flows: conditionalFlows, + links, + branch_id: branchId, + branch_name: branchName, + is_development_branch: isDevelopmentBranch, + user_role: userRole, + toolset_restrictions: getToolsetRestrictions(userRole), + llm_instruction: getProjectSystemPrompt(sqlDialect), + }; + }, + }); +}; diff --git a/src/tools/search/globalSearch.ts b/src/tools/search/globalSearch.ts new file mode 100644 index 000000000..79a34b9ba --- /dev/null +++ b/src/tools/search/globalSearch.ts @@ -0,0 +1,553 @@ +/** + * Global (server-side) textual search and the client-side enumeration fallback. + * + * Port of the Python `tools/search_global.py` (`_global_textual_search`) and the + * `tools/search.py` enumeration helpers (`_enumeration_search` and friends). + */ + +import type { KeboolaClients } from '@/clients/keboola'; +import { MetadataField } from '@/constants'; +import { logger } from '@/logger'; +import { + type ApiItemType, + DATA_APP_COMPONENT_ID, + getFieldValue, + getMetadataProperty, + GLOBAL_SEARCH_FEATURE, + makeHit, + MAX_GLOBAL_SEARCH_LIMIT, + ORCHESTRATOR_IDS, + type PatternMatch, + type RawDict, + SEARCH_ITEM_TYPE_TO_API_TYPES, + type SearchHit, + type SearchItemType, + type SearchOutput, + type SearchSpec, + setMatches, + WORKSPACE_COMPONENT_ID, +} from './model'; + +// --------------------------------------------------------------------------- +// Global (server-side) textual search (port of search_global._global_textual_search) +// --------------------------------------------------------------------------- + +type GlobalSearchItem = { + id: string; + name: string; + type: string; + fullPath?: RawDict; + componentId?: string | null; + created: string; +}; + +type GlobalSearchResponse = { + all: number; + items: GlobalSearchItem[]; + byType?: Record; +}; + +const apiTypesFor = (itemTypes: SearchItemType[]): ApiItemType[] => { + const apiTypes: ApiItemType[] = []; + for (const itemType of itemTypes) { + for (const apiType of SEARCH_ITEM_TYPE_TO_API_TYPES[itemType] ?? []) { + if (!apiTypes.includes(apiType)) apiTypes.push(apiType); + } + } + return apiTypes; +}; + +const retypeConfiguration = (componentId: string | null | undefined): SearchItemType => { + if (componentId && ORCHESTRATOR_IDS.has(componentId)) return 'flow'; + if (componentId === DATA_APP_COMPONENT_ID) return 'data-app'; + if (componentId === WORKSPACE_COMPONENT_ID) return 'workspace'; + return 'configuration'; +}; + +const branchInfo = (item: GlobalSearchItem): { id: string | null; name: string | null } => { + const branch = item.fullPath?.branch; + if (branch && typeof branch === 'object') { + const b = branch as RawDict; + return { + id: b.id != null ? String(b.id) : null, + name: b.name ? String(b.name) : null, + }; + } + return { id: null, name: null }; +}; + +const globalSearchHit = (item: GlobalSearchItem): SearchHit | null => { + const { id: branch_id, name: branch_name } = branchInfo(item); + const common = { updated: item.created, name: item.name, branch_id, branch_name }; + + if (item.type === 'bucket') { + return makeHit({ bucket_id: item.id, item_type: 'bucket', ...common }); + } + + if (item.type === 'table') { + const bucket = item.fullPath?.bucket; + const bucketId = + bucket && typeof bucket === 'object' && (bucket as RawDict).id + ? String((bucket as RawDict).id) + : null; + return makeHit({ table_id: item.id, bucket_id: bucketId, item_type: 'table', ...common }); + } + + if (item.type === 'configuration-row' || item.type === 'rows') { + const configuration = item.fullPath?.configuration; + const configurationId = + configuration && typeof configuration === 'object' && (configuration as RawDict).id + ? String((configuration as RawDict).id) + : null; + if (!(item.componentId && configurationId)) { + logger.warn( + `Skipping global-search row hit with no parent configuration in fullPath: ${item.id}`, + ); + return null; + } + return makeHit({ + component_id: item.componentId, + configuration_id: configurationId, + configuration_row_id: item.id, + item_type: 'configuration-row', + ...common, + }); + } + + const componentId = + item.componentId ?? (item.type === 'workspace' ? WORKSPACE_COMPONENT_ID : null); + if (!componentId) { + logger.warn(`Skipping global-search hit with no component id: ${item.type} ${item.id}`); + return null; + } + const itemType = + item.type === 'configuration' + ? retypeConfiguration(componentId) + : (item.type as SearchItemType); + return makeHit({ + component_id: componentId, + configuration_id: item.id, + item_type: itemType, + ...common, + }); +}; + +export const globalTextualSearch = async ( + clients: KeboolaClients, + spec: SearchSpec, + limit: number, + offset: number, +): Promise => { + const apiTypes = apiTypesFor(spec.itemTypes); + const requestedTypes = new Set( + spec.itemTypes + .filter((t) => t !== 'component') + .map((t) => (t === 'rows' ? 'configuration-row' : t)), + ); + + const needsOverfetch = requestedTypes.size > 0 && apiTypes.includes('configuration'); + const fetchLimit = needsOverfetch ? MAX_GLOBAL_SEARCH_LIMIT : limit; + + const projectId = await clients.storage.tokens + .verify() + .then((t) => String((t.owner as { id: string | number }).id)); + + const query = async (branchScope: 'current' | 'all'): Promise => { + return Promise.all( + spec.patterns.map((pattern) => { + // Issued via the raw Storage client (not the typed `storage.search.globalSearch`) so array + // params serialize as repeated `projectIds[]=...` keys — the typed client's serializer + // nests them as `projectIds[][0]`, which SAPI rejects. + const params: Record = { + query: pattern, + 'projectIds[]': [projectId], + 'types[]': apiTypes.length > 0 ? apiTypes : undefined, + limit: fetchLimit, + offset: offset || undefined, + }; + if (branchScope === 'current') { + if (clients.branchId === 'default') { + params['branchTypes[]'] = 'production'; + } else { + params['branchTypes[]'] = 'development'; + params['branchIds[]'] = clients.branchId; + } + } + return clients.rawStorage.get('global-search', { params }); + }), + ); + }; + + const collect = (responses: GlobalSearchResponse[]): SearchHit[] => { + const hitsByKey = new Map(); + for (const response of responses) { + for (const item of response.items ?? []) { + const hit = globalSearchHit(item); + if (hit === null) continue; + if (requestedTypes.size > 0 && !requestedTypes.has(hit.item_type)) continue; + const key = `${item.type} ${item.id}`; + if (!hitsByKey.has(key)) hitsByKey.set(key, hit); + } + } + return [...hitsByKey.values()]; + }; + + let branchScope: 'current' | 'all' = 'current'; + let responses = await query(branchScope); + let hits = collect(responses); + if (hits.length === 0 && offset === 0) { + branchScope = 'all'; + responses = await query(branchScope); + hits = collect(responses); + } + + sortHits(hits); + + const byType: Record = {}; + for (const response of responses) { + for (const [type, count] of Object.entries(response.byType ?? {})) { + byType[type] = (byType[type] ?? 0) + count; + } + } + + return { + hits: hits.slice(0, limit), + total: responses.reduce((sum, r) => sum + (r.all ?? 0), 0), + by_type: byType, + branch_scope: branchScope === 'current' ? 'current-branch' : 'all-branches', + }; +}; + +const sortKey = (hit: SearchHit): string => + hit.bucket_id ?? + hit.table_id ?? + hit.component_id ?? + hit.configuration_id ?? + hit.configuration_row_id ?? + ''; + +/** Sorts by (updated, id) descending — same tuple comparison the Python code uses. */ +const sortHits = (hits: SearchHit[]): void => { + hits.sort((a, b) => { + if (a.updated !== b.updated) return a.updated < b.updated ? 1 : -1; + const ka = sortKey(a); + const kb = sortKey(b); + if (ka === kb) return 0; + return ka < kb ? 1 : -1; + }); +}; + +// --------------------------------------------------------------------------- +// Client-side enumeration (port of search._enumeration_search and helpers) +// --------------------------------------------------------------------------- + +const fetchBuckets = async (clients: KeboolaClients, spec: SearchSpec): Promise => { + const buckets = await clients.rawStorage.get(`branch/${clients.branchId}/buckets`, { + params: { include: 'metadata' }, + }); + const hits: SearchHit[] = []; + for (const bucket of buckets ?? []) { + const bucketId = bucket.id ? String(bucket.id) : null; + if (!bucketId) continue; + const name = (bucket.name as string) ?? null; + const displayName = (bucket.displayName as string) ?? null; + const description = getMetadataProperty(bucket.metadata, MetadataField.DESCRIPTION); + + const matches = spec.matchTexts([bucketId, name, displayName, description]); + if (matches.length > 0) { + hits.push( + setMatches( + makeHit({ + bucket_id: bucketId, + item_type: 'bucket', + updated: getFieldValue(bucket, ['lastChangeDate', 'updated', 'created']) ?? '', + name, + display_name: displayName, + description, + }), + matches, + ), + ); + } + } + return hits; +}; + +const checkColumnMatch = (table: RawDict, spec: SearchSpec): PatternMatch[] => { + const colNames = table.columns; + if (Array.isArray(colNames) && colNames.length > 0) { + const matched = spec.matchTexts(colNames as string[]); + if (matched.length > 0) return matched; + } + const colMetadata = table.columnMetadata; + if (colMetadata && typeof colMetadata === 'object') { + const descs = Object.values(colMetadata as RawDict) + .map((meta) => getMetadataProperty(meta, MetadataField.DESCRIPTION)) + .filter((d): d is string => Boolean(d)); + const matched = spec.matchTexts(descs); + if (matched.length > 0) return matched; + } + return []; +}; + +const fetchTables = async (clients: KeboolaClients, spec: SearchSpec): Promise => { + const buckets = await clients.rawStorage.get(`branch/${clients.branchId}/buckets`); + const hits: SearchHit[] = []; + for (const bucket of buckets ?? []) { + const bucketId = bucket.id ? String(bucket.id) : null; + if (!bucketId) continue; + const tables = await clients.rawStorage.get( + `branch/${clients.branchId}/buckets/${bucketId}/tables`, + { params: { include: 'columns,columnMetadata' } }, + ); + for (const table of tables ?? []) { + const tableId = table.id ? String(table.id) : null; + if (!tableId) continue; + const name = (table.name as string) ?? null; + const displayName = (table.displayName as string) ?? null; + const description = getMetadataProperty(table.metadata, MetadataField.DESCRIPTION); + + const matches = spec.matchTexts([tableId, name, displayName, description]); + matches.push(...checkColumnMatch(table, spec)); + if (matches.length > 0) { + hits.push( + setMatches( + makeHit({ + table_id: tableId, + item_type: 'table', + updated: getFieldValue(table, ['lastChangeDate', 'created']) ?? '', + name, + display_name: displayName, + description, + }), + matches, + ), + ); + } + } + } + return hits; +}; + +const fetchConfigsForType = async ( + clients: KeboolaClients, + spec: SearchSpec, + componentType: string | null, +): Promise => { + const params: Record = { include: 'configuration,rows' }; + if (componentType) params.componentType = componentType; + const components = await clients.rawStorage.get( + `branch/${clients.branchId}/components`, + { params }, + ); + + const allowedTransformations = + spec.itemTypes.includes('transformation') || componentType === null; + const allowedComponents = + spec.itemTypes.includes('configuration') || + spec.itemTypes.includes('configuration-row') || + componentType === null; + const allowedFlows = spec.itemTypes.includes('flow') || componentType === null; + const allowedWorkspaces = spec.itemTypes.includes('workspace') || componentType === null; + const allowedDataApps = spec.itemTypes.includes('data-app') || componentType === null; + + const hits: SearchHit[] = []; + for (const component of components ?? []) { + const componentId = component.id ? String(component.id) : null; + if (!componentId) continue; + const currentComponentType = component.type as string | undefined; + + let itemType: SearchItemType; + if (ORCHESTRATOR_IDS.has(componentId)) { + itemType = 'flow'; + if (!allowedFlows) continue; + } else if (currentComponentType === 'transformation') { + itemType = 'transformation'; + if (!allowedTransformations) continue; + } else if (componentId === WORKSPACE_COMPONENT_ID) { + itemType = 'workspace'; + if (!allowedWorkspaces) continue; + } else if (componentId === DATA_APP_COMPONENT_ID) { + itemType = 'data-app'; + if (!allowedDataApps) continue; + } else if ( + currentComponentType === 'extractor' || + currentComponentType === 'writer' || + currentComponentType === 'application' + ) { + itemType = 'configuration'; + if (!allowedComponents) continue; + } else { + itemType = 'configuration'; + } + + for (const config of (component.configurations as RawDict[]) ?? []) { + const configId = config.id ? String(config.id) : null; + if (!configId) continue; + const configName = (config.name as string) ?? null; + const configDescription = (config.description as string) ?? null; + const configUpdated = getFieldValue(config, ['currentVersion.created', 'created']) ?? ''; + + if (spec.searchType === 'textual') { + const matches = spec.matchTexts([configId, configName, configDescription]); + if (matches.length > 0) { + hits.push( + setMatches( + makeHit({ + component_id: componentId, + configuration_id: configId, + item_type: itemType, + updated: configUpdated, + name: configName, + description: configDescription, + }), + matches, + ), + ); + } + } else { + const matches = spec.matchConfigurationScopes(config.configuration); + if (matches.length > 0) { + hits.push( + setMatches( + makeHit({ + component_id: componentId, + configuration_id: configId, + item_type: itemType, + updated: configUpdated, + name: configName, + description: configDescription, + }), + matches, + ), + ); + } + } + + for (const row of (config.rows as RawDict[]) ?? []) { + const rowId = row.id ? String(row.id) : null; + if (!rowId) continue; + const rowName = (row.name as string) ?? null; + const rowDescription = (row.description as string) ?? null; + const rowUpdated = configUpdated || (getFieldValue(row, ['created']) ?? ''); + + if (spec.searchType === 'textual') { + const matches = spec.matchTexts([rowId, rowName, rowDescription]); + if (matches.length > 0) { + hits.push( + setMatches( + makeHit({ + component_id: componentId, + configuration_id: configId, + configuration_row_id: rowId, + item_type: 'configuration-row', + updated: rowUpdated, + name: rowName, + description: rowDescription, + }), + matches, + ), + ); + } + } else { + const matches = spec.matchConfigurationScopes(row.configuration); + if (matches.length > 0) { + hits.push( + setMatches( + makeHit({ + component_id: componentId, + configuration_id: configId, + configuration_row_id: rowId, + item_type: 'configuration-row', + updated: rowUpdated, + name: rowName, + description: rowDescription, + }), + matches, + ), + ); + } + } + } + } + } + return hits; +}; + +const fetchConfigurations = async ( + clients: KeboolaClients, + spec: SearchSpec, +): Promise => { + if (spec.componentTypes.length > 0) { + const all = await Promise.all( + spec.componentTypes.map((componentType) => fetchConfigsForType(clients, spec, componentType)), + ); + return all.flat(); + } + return fetchConfigsForType(clients, spec, null); +}; + +const CONFIG_TYPES = new Set([ + 'configuration', + 'transformation', + 'flow', + 'configuration-row', + 'workspace', + 'data-app', +]); + +export const enumerationSearch = async ( + clients: KeboolaClients, + spec: SearchSpec, + limit: number, + offset: number, +): Promise => { + const typesToFetch = new Set(spec.itemTypes); + const tasks: Promise[] = []; + + if (typesToFetch.size === 0 || typesToFetch.has('bucket')) { + tasks.push(fetchBuckets(clients, spec)); + } + if (typesToFetch.size === 0 || typesToFetch.has('table')) { + tasks.push(fetchTables(clients, spec)); + } + if (typesToFetch.size === 0) { + tasks.push(fetchConfigurations(clients, spec)); + } else if ([...typesToFetch].some((t) => CONFIG_TYPES.has(t))) { + tasks.push(fetchConfigurations(clients, spec)); + } + + const results = await Promise.allSettled(tasks); + let allHits: SearchHit[] = []; + for (const result of results) { + if (result.status === 'rejected') { + logger.warn(`Error fetching items: ${String(result.reason)}`); + continue; + } + allHits.push(...result.value); + } + + if (typesToFetch.size > 0) { + allHits = allHits.filter((hit) => typesToFetch.has(hit.item_type)); + } + + sortHits(allHits); + + const byType: Record = {}; + for (const hit of allHits) { + byType[hit.item_type] = (byType[hit.item_type] ?? 0) + 1; + } + + return { + hits: allHits.slice(offset, offset + limit), + total: allHits.length, + by_type: byType, + branch_scope: 'current-branch', + }; +}; + +export const isGlobalSearchEnabled = async (clients: KeboolaClients): Promise => { + const verified = await clients.storage.tokens.verify(); + const owner = verified.owner as { features?: string[] } | undefined; + return Boolean(owner?.features?.includes(GLOBAL_SEARCH_FEATURE)); +}; diff --git a/src/tools/search/index.ts b/src/tools/search/index.ts new file mode 100644 index 000000000..3e34e1625 --- /dev/null +++ b/src/tools/search/index.ts @@ -0,0 +1,11 @@ +/** + * `search` tool module. Public entry point: `registerSearchTools`, used by `src/server.ts`. + * + * Decomposed into: + * - `tools.ts` — `find_component_id` + `search` handlers and registration + * - `globalSearch.ts` — server-side global textual search + client-side enumeration fallback + * - `model.ts` — types, constants, `SearchSpec` matching model and metadata helpers + * - `jsonpath.ts` — the local JSONPath subset used by config-based search + */ + +export { registerSearchTools } from './tools'; diff --git a/src/tools/search/jsonpath.ts b/src/tools/search/jsonpath.ts new file mode 100644 index 000000000..199a36427 --- /dev/null +++ b/src/tools/search/jsonpath.ts @@ -0,0 +1,122 @@ +/** + * Local JSONPath subset used by config-based search (port of the `_clean_jsonpath_path_str` + * / scope-selection helpers from the Python `tools/search.py`). Kept standalone so the + * matching model in `model.ts` and the tool handlers stay focused. + */ + +export type JsonValue = unknown; + +export type PathNode = { path: string; value: JsonValue }; + +export const escapeRegExp = (value: string): string => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** Deterministic JSON stringification with sorted keys (port of SearchSpec._stringify). */ +export const stringify = (value: JsonValue): string => { + try { + return stableStringify(value); + } catch { + return String(value); + } +}; + +const stableStringify = (value: JsonValue): string => { + const seen = new WeakSet(); + const sort = (val: JsonValue): JsonValue => { + if (val === null || typeof val !== 'object') return val; + if (seen.has(val as object)) return val; + seen.add(val as object); + if (Array.isArray(val)) return val.map(sort); + const out: Record = {}; + for (const key of Object.keys(val as Record).sort()) { + out[key] = sort((val as Record)[key]); + } + return out; + }; + return JSON.stringify(sort(value)); +}; + +/** Resolves a dot/bracket scope to the matching nodes (supports a single `[*]`/`[N]` step). */ +export const selectScope = (root: JsonValue, scope: string): PathNode[] => { + const normalized = scope.startsWith('$') ? scope.slice(1).replace(/^\./, '') : scope; + if (!normalized) return [{ path: '$', value: root }]; + + let nodes: PathNode[] = [{ path: '$', value: root }]; + for (const token of tokenizePath(normalized)) { + const next: PathNode[] = []; + for (const node of nodes) { + if (token.type === 'wildcard') { + next.push(...childEntries(node)); + } else if (token.type === 'index') { + if (Array.isArray(node.value) && token.index < node.value.length) { + next.push({ path: `${node.path}[${token.index}]`, value: node.value[token.index] }); + } + } else { + if (node.value && typeof node.value === 'object' && !Array.isArray(node.value)) { + const obj = node.value as Record; + if (token.key in obj) { + next.push({ path: `${node.path}.${token.key}`, value: obj[token.key] }); + } + } + } + } + nodes = next; + } + return nodes; +}; + +type PathToken = + | { type: 'key'; key: string } + | { type: 'index'; index: number } + | { type: 'wildcard' }; + +const tokenizePath = (path: string): PathToken[] => { + const tokens: PathToken[] = []; + const regex = /\[([^\]]*)\]|([^.[\]]+)/g; + let match: RegExpExecArray | null; + while ((match = regex.exec(path)) !== null) { + if (match[1] !== undefined) { + const inner = match[1].replace(/^['"]|['"]$/g, ''); + if (inner === '*') tokens.push({ type: 'wildcard' }); + else if (/^\d+$/.test(inner)) tokens.push({ type: 'index', index: Number(inner) }); + else tokens.push({ type: 'key', key: inner }); + } else if (match[2] !== undefined) { + if (match[2] === '*') tokens.push({ type: 'wildcard' }); + else tokens.push({ type: 'key', key: match[2] }); + } + } + return tokens; +}; + +const childEntries = (node: PathNode): PathNode[] => { + const { value, path } = node; + if (Array.isArray(value)) { + return value.map((item, i) => ({ path: `${path}[${i}]`, value: item })); + } + if (value && typeof value === 'object') { + return Object.entries(value as Record).map(([k, v]) => ({ + path: `${path}.${k}`, + value: v, + })); + } + return []; +}; + +/** Recursive descent (`$..*`): every descendant node with its full path. */ +export const descendants = (root: JsonValue, basePath: string): PathNode[] => { + const out: PathNode[] = []; + const walk = (node: PathNode): void => { + for (const child of childEntries(node)) { + out.push(child); + walk(child); + } + }; + walk({ path: basePath, value: root }); + return out; +}; + +/** Normalizes a path string: strips the leading `$.`/`$` and `.[` artifacts (port of _clean_jsonpath_path_str). */ +export const cleanJsonPath = (path: string): string => { + let result = path.replace(/^\$\.?/, ''); + result = result.replace(/\.\[/g, '['); + return result; +}; diff --git a/src/tools/search/model.ts b/src/tools/search/model.ts new file mode 100644 index 000000000..84c75a1e6 --- /dev/null +++ b/src/tools/search/model.ts @@ -0,0 +1,355 @@ +/** + * Shared models, type aliases, constants and the matching model (`SearchSpec`) for the + * `search` tool. + * + * Port of the Python `tools/search_models.py` (and the parts of `search_global.py` / + * `search.py` that the global `search` tool depends on). Kept in its own module so the + * tool file stays focused on registration + handler wiring. + */ + +import { + CONDITIONAL_FLOW_COMPONENT_ID, + DATA_APP_COMPONENT_ID, + ORCHESTRATOR_COMPONENT_ID, +} from '@/constants'; +import type { Link } from '@/links'; +import { + cleanJsonPath, + descendants, + escapeRegExp, + type JsonValue, + type PathNode, + selectScope, + stringify, +} from './jsonpath'; + +export const MAX_GLOBAL_SEARCH_LIMIT = 100; +export const DEFAULT_GLOBAL_SEARCH_LIMIT = 50; + +export const GLOBAL_SEARCH_FEATURE = 'global-search'; +export const WORKSPACE_COMPONENT_ID = 'keboola.sandboxes'; + +/** Item types the `search` tool accepts (and reports). */ +export const SEARCH_ITEM_TYPES = [ + 'bucket', + 'table', + 'data-app', + 'flow', + 'transformation', + 'component', + 'configuration', + 'configuration-row', + 'workspace', + 'shared-code', + 'rows', + 'state', +] as const; +export type SearchItemType = (typeof SEARCH_ITEM_TYPES)[number]; + +/** Item types reported by the SAPI global-search endpoint. */ +export type ApiItemType = + | 'flow' + | 'bucket' + | 'table' + | 'transformation' + | 'configuration' + | 'configuration-row' + | 'workspace' + | 'shared-code' + | 'rows' + | 'state'; + +export const SEARCH_TYPES = ['textual', 'config-based'] as const; +export type SearchType = (typeof SEARCH_TYPES)[number]; + +export const SEARCH_PATTERN_MODES = ['regex', 'literal'] as const; +export type SearchPatternMode = (typeof SEARCH_PATTERN_MODES)[number]; + +export type SearchBranchScope = 'current-branch' | 'all-branches'; + +/** Maps a tool item type to the component types fetched during client-side enumeration. */ +export const SEARCH_ITEM_TYPE_TO_COMPONENT_TYPES: Partial> = { + 'data-app': ['other'], + flow: ['other'], + transformation: ['transformation'], + configuration: ['extractor', 'writer', 'application'], + 'configuration-row': ['extractor', 'writer', 'application'], + component: ['extractor', 'writer', 'application'], + workspace: ['other'], +}; + +/** + * Maps the tool's item types to the API types requested from the global-search endpoint. Some tool + * types (data-app, flow, workspace) exist server-side as 'configuration' items distinguished only + * by their component ID, so 'configuration' is over-fetched and narrowed client-side after re-typing. + */ +export const SEARCH_ITEM_TYPE_TO_API_TYPES: Record = { + bucket: ['bucket'], + table: ['table'], + transformation: ['transformation'], + configuration: ['configuration'], + 'configuration-row': ['configuration-row'], + component: ['configuration', 'configuration-row'], + flow: ['flow', 'configuration'], + 'data-app': ['configuration'], + workspace: ['workspace', 'configuration'], + 'shared-code': ['shared-code'], + rows: ['rows'], + state: ['state'], +}; + +export const ORCHESTRATOR_IDS = new Set([ + ORCHESTRATOR_COMPONENT_ID, + CONDITIONAL_FLOW_COMPONENT_ID, +]); +export { DATA_APP_COMPONENT_ID }; + +export type PatternMatch = { + scope: string | null; + patterns: string[]; +}; + +/** A single search result. Shape mirrors the Python `SearchHit`. */ +export type SearchHit = { + bucket_id: string | null; + table_id: string | null; + component_id: string | null; + configuration_id: string | null; + configuration_row_id: string | null; + item_type: SearchItemType; + updated: string; + name: string | null; + display_name: string | null; + description: string | null; + branch_id: string | null; + branch_name: string | null; + matches: PatternMatch[]; + links: Link[]; +}; + +export type SearchOutput = { + hits: SearchHit[]; + total: number; + by_type: Record; + branch_scope: SearchBranchScope; +}; + +/** Builds a SearchHit with the same field defaults the Python model declares. */ +export const makeHit = (init: Partial & { item_type: SearchItemType }): SearchHit => ({ + bucket_id: null, + table_id: null, + component_id: null, + configuration_id: null, + configuration_row_id: null, + updated: '', + name: null, + display_name: null, + description: null, + branch_id: null, + branch_name: null, + matches: [], + links: [], + ...init, +}); + +// --------------------------------------------------------------------------- +// Pattern / configuration matching (port of search_models.SearchSpec) +// --------------------------------------------------------------------------- + +/** Compiled search specification, mirroring the Python `SearchSpec`. */ +export class SearchSpec { + readonly patterns: string[]; + readonly itemTypes: SearchItemType[]; + readonly patternMode: SearchPatternMode; + readonly searchType: SearchType; + readonly searchScopes: string[]; + readonly returnAllMatchedPatterns: boolean; + readonly componentTypes: string[]; + + private readonly compiled: RegExp[]; + + constructor(opts: { + patterns: string[]; + itemTypes: SearchItemType[]; + patternMode?: SearchPatternMode; + searchType?: SearchType; + searchScopes?: string[]; + returnAllMatchedPatterns?: boolean; + }) { + const cleaned = opts.patterns + .filter((p) => p != null) + .map((p) => String(p).trim()) + .filter((p) => p.length > 0); + if (cleaned.length === 0) { + throw new Error('At least one search pattern must be provided.'); + } + this.patterns = cleaned; + this.patternMode = opts.patternMode ?? 'regex'; + this.searchType = opts.searchType ?? 'textual'; + this.searchScopes = [...(opts.searchScopes ?? [])]; + this.returnAllMatchedPatterns = opts.returnAllMatchedPatterns ?? false; + + // _validate_item_types: 'component' expands to configuration + configuration-row. + let itemTypes = [...opts.itemTypes]; + if (itemTypes.includes('component')) { + itemTypes = [ + ...new Set([...itemTypes, 'configuration', 'configuration-row']), + ]; + } + this.itemTypes = itemTypes; + + // _validate_component_args: derive component types fetched during enumeration. + this.componentTypes = [ + ...new Set(itemTypes.flatMap((item) => SEARCH_ITEM_TYPE_TO_COMPONENT_TYPES[item] ?? [])), + ]; + + // Case-insensitive by default (Python `case_sensitive` defaults to False). + const reFlags = 'i'; + this.compiled = cleaned.map((pattern) => + this.patternMode === 'literal' + ? new RegExp(escapeRegExp(pattern), reFlags) + : new RegExp(pattern, reFlags), + ); + } + + /** Returns the patterns that match a string or stringified JSON value. */ + matchPatterns(value: string | JsonValue | null | undefined): string[] { + if (value === null || value === undefined) return []; + const haystack = typeof value === 'string' ? value : stringify(value); + if (!haystack) return []; + + const matches: string[] = []; + for (let i = 0; i < this.patterns.length; i++) { + if (this.compiled[i]!.test(haystack)) { + matches.push(this.patterns[i]!); + if (!this.returnAllMatchedPatterns) break; + } + } + return matches; + } + + /** Matches a list of texts (e.g. id/name/description); scope is null. */ + matchTexts(texts: (string | null | undefined)[]): PatternMatch[] { + const matches: PatternMatch[] = []; + for (const text of texts) { + const matched = this.matchPatterns(text); + if (matched.length > 0) { + matches.push({ scope: null, patterns: matched }); + if (!this.returnAllMatchedPatterns) break; + } + } + return matches; + } + + /** Matches configuration JSON within the configured scopes (or all nodes). */ + matchConfigurationScopes(configuration: JsonValue | null | undefined): PatternMatch[] { + if (configuration === null || configuration === undefined) return []; + + if (this.searchScopes.length > 0) { + const all: PatternMatch[] = []; + const seen = new Set(); + for (const scope of this.searchScopes) { + const selfNodes = selectScope(configuration, scope); + // Scalar matches in the scope node first. + let scopeMatches = this.findMatches(selfNodes, true); + if (scopeMatches.length === 0) { + const descNodes = selfNodes.flatMap((n) => descendants(n.value, n.path)); + scopeMatches = this.findMatches(descNodes, false); + } + for (const match of scopeMatches) { + if (seen.has(match.scope)) continue; + seen.add(match.scope); + all.push(match); + if (!this.returnAllMatchedPatterns) return all; + } + } + return all; + } + + // No scope provided — search all descendants, return exact match paths. + const nodes = descendants(configuration, '$'); + return this.findMatches(nodes, false); + } + + private findMatches(nodes: PathNode[], scalarOnly: boolean): PatternMatch[] { + const matches: PatternMatch[] = []; + for (const node of nodes) { + if (scalarOnly && node.value !== null && typeof node.value === 'object') continue; + const matched = this.matchPatterns(node.value); + if (matched.length > 0) { + matches.push({ scope: cleanJsonPath(node.path), patterns: matched }); + if (!this.returnAllMatchedPatterns) return matches; + } + } + return matches; + } +} + +/** Assigns matches to a hit, keeping only the most specific scopes (port of SearchHit.set_matches). */ +export const setMatches = (hit: SearchHit, matches: PatternMatch[]): SearchHit => { + const patternsByScope = new Map>(); + for (const match of matches) { + if (!match.scope) continue; + if (!patternsByScope.has(match.scope)) patternsByScope.set(match.scope, new Set()); + for (const p of match.patterns) patternsByScope.get(match.scope)!.add(p); + } + const scopes = [...patternsByScope.keys()]; + const mostSpecific = scopes.filter( + (scope) => + !scopes.some( + (other) => + other.startsWith(scope) && + other.length > scope.length && + (other[scope.length] === '.' || other[scope.length] === '['), + ), + ); + hit.matches = mostSpecific.map((scope) => ({ + scope, + patterns: [...patternsByScope.get(scope)!].sort(), + })); + return hit; +}; + +// --------------------------------------------------------------------------- +// Metadata helpers (port of clients.client.get_metadata_property + get_nested) +// --------------------------------------------------------------------------- + +export type RawDict = Record; + +export const getMetadataProperty = (metadata: unknown, key: string): string | null => { + if (!Array.isArray(metadata)) return null; + const filtered = (metadata as RawDict[]).filter((m) => m && m.key === key); + // Most recent by timestamp. + let best: RawDict | undefined; + let bestTs = ''; + for (const m of filtered) { + const ts = (m.timestamp as string) ?? ''; + if (best === undefined || ts >= bestTs) { + best = m; + bestTs = ts; + } + } + const value = best ? best.value : undefined; + return value != null ? String(value) : null; +}; + +const getNested = (obj: RawDict | null | undefined, key: string): unknown => { + let cur: unknown = obj; + for (const part of key.split('.')) { + if (cur && typeof cur === 'object' && !Array.isArray(cur)) { + cur = (cur as RawDict)[part]; + } else { + return null; + } + if (cur === null || cur === undefined) return null; + } + return cur; +}; + +export const getFieldValue = (item: RawDict, fields: string[]): string | null => { + for (const field of fields) { + const value = getNested(item, field); + if (value) return String(value); + } + return null; +}; diff --git a/src/tools/search/tools.ts b/src/tools/search/tools.ts new file mode 100644 index 000000000..5f2c6eb23 --- /dev/null +++ b/src/tools/search/tools.ts @@ -0,0 +1,189 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { getDocsSearch } from '@/clients/docsSearch'; +import { createKeboolaClients, createLinksManager } from '@/clients/keboola'; +import type { Config } from '@/config'; +import { logger } from '@/logger'; +import { registerTool } from '@/mcp/tool'; +import { enumerationSearch, globalTextualSearch, isGlobalSearchEnabled } from './globalSearch'; +import { + DEFAULT_GLOBAL_SEARCH_LIMIT, + MAX_GLOBAL_SEARCH_LIMIT, + SEARCH_ITEM_TYPES, + SEARCH_PATTERN_MODES, + SEARCH_TYPES, + type SearchOutput, + SearchSpec, +} from './model'; + +// Ported from tools/search.py: the `find_component_id` and global `search` tools. + +const SEARCH_DESCRIPTION = + 'Searches for Keboola items (tables, buckets, components, configurations, transformations, flows, ' + + 'data-apps, etc.) in the current project and returns matching IDs and metadata. Supports textual ' + + 'search (matches item names, server-side) and config-based search (matches patterns against the ' + + 'configuration JSON content, optionally narrowed by JSONPath scopes). THIS IS THE PRIMARY DISCOVERY ' + + 'TOOL — use it before any get_* tool when you need to find items by name or configuration content. ' + + 'Multiple patterns work as an OR condition. Textual search prefers the current branch and, when ' + + 'nothing is found there, automatically widens to all branches of the project.'; + +export const registerSearchTools = (server: McpServer, config: Config): void => { + registerTool(server, { + name: 'find_component_id', + title: 'Find component id', + description: 'Returns a list of component IDs that match the given natural-language query.', + annotations: { readOnlyHint: true }, + inputSchema: { + query: z.string().describe('Natural language query to find the requested component.'), + }, + handler: async ({ query }) => { + const docs = getDocsSearch(); + if (!docs) throw new Error('The documentation index is not available.'); + + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + + const results = await docs.recommendComponents(query); + + return results + .map((doc) => { + // Component docs are keyed `component:`; recover the id from the source key. + const componentId = doc.sourceKey.startsWith('component:') + ? doc.sourceKey.slice('component:'.length) + : ''; + return { + component_id: componentId, + score: doc.score, + links: [linksManager.getConfigDashboardLink(componentId, undefined)], + }; + }) + .filter((r) => r.component_id !== ''); + }, + }); + + registerTool(server, { + name: 'search', + title: 'Search', + description: SEARCH_DESCRIPTION, + annotations: { readOnlyHint: true }, + inputSchema: { + patterns: z + .array(z.string()) + .describe( + 'One or more search patterns. For textual search they match item names (server-side, ' + + 'tokenized full-text); for config-based search they match the configuration JSON content. ' + + 'Case-insensitive by default. Examples: ["customer"], ["sales", "revenue"], ["my_bucket"]. ' + + 'Do not use empty strings or empty lists.', + ), + item_types: z + .array(z.enum(SEARCH_ITEM_TYPES)) + .default([]) + .describe( + 'Filter for specific Keboola item types. Common values: "table" (data tables), "bucket" ' + + '(table containers), "transformation" (SQL/Python transformations), "component" ' + + '(extractor/writer/application components), "data-app" (data apps), "flow" (orchestration ' + + "flows). Use when you know what type of item you're looking for or leave empty to search " + + 'all types.', + ), + search_type: z + .enum(SEARCH_TYPES) + .default('textual') + .describe( + 'Search mode: "textual" (name/id/description) or "config-based" (stringified configuration ' + + 'payloads). (default: "textual")', + ), + scopes: z + .array(z.string()) + .default([]) + .describe( + 'JSONPath expressions to narrow config-based search to specific parts of the configuration. ' + + 'Simple dot-notation (e.g. "parameters", "storage.input") and full JSONPath (e.g. ' + + '"$.tasks[*]") are both supported (e.g. "parameters.host", "storage.input[0].source"). ' + + 'Leave empty to search the whole configuration.', + ), + mode: z + .enum(SEARCH_PATTERN_MODES) + .default('literal') + .describe( + 'How to interpret patterns. Applies to config-based search only: "regex" for regular ' + + 'expressions or "literal" for exact text (default: "literal"). Ignored by textual search, ' + + 'which is always a tokenized full-text name query (not typo-corrected) and rejects "regex".', + ), + limit: z + .number() + .default(DEFAULT_GLOBAL_SEARCH_LIMIT) + .describe( + `Maximum number of items to return (default: ${DEFAULT_GLOBAL_SEARCH_LIMIT}, max: ${MAX_GLOBAL_SEARCH_LIMIT}).`, + ), + offset: z + .number() + .default(0) + .describe('Number of matching items to skip for pagination (default: 0).'), + }, + handler: async (args) => { + const spec = new SearchSpec({ + patterns: args.patterns, + itemTypes: args.item_types, + patternMode: args.mode, + searchType: args.search_type, + searchScopes: args.scopes, + returnAllMatchedPatterns: args.search_type === 'config-based', + }); + + const offset = Math.max(0, args.offset); + let limit = args.limit; + if (!(limit > 0 && limit <= MAX_GLOBAL_SEARCH_LIMIT)) { + logger.warn( + `The "limit" parameter is out of range (0, ${MAX_GLOBAL_SEARCH_LIMIT}], setting to default value ${DEFAULT_GLOBAL_SEARCH_LIMIT}.`, + ); + limit = DEFAULT_GLOBAL_SEARCH_LIMIT; + } + + const clients = createKeboolaClients(config); + + let output: SearchOutput; + if (args.search_type === 'textual' && (await isGlobalSearchEnabled(clients))) { + if (args.mode === 'regex') { + throw new Error( + 'Regex patterns are not supported for textual search — it is a tokenized full-text name search. ' + + 'Pass the plain name as the pattern, or use search_type="config-based" for regex matching inside ' + + 'configurations.', + ); + } + // Global search is a fast path with a safety net: fall back to client-side enumeration on any + // error, or when it finds nothing. + try { + output = await globalTextualSearch(clients, spec, limit, offset); + if (output.hits.length === 0 && offset === 0) { + logger.info('Global search returned no hits; falling back to client-side enumeration.'); + output = await enumerationSearch(clients, spec, limit, offset); + } + } catch (error) { + logger.warn( + { err: error }, + 'Global search failed; falling back to client-side enumeration.', + ); + output = await enumerationSearch(clients, spec, limit, offset); + } + } else { + output = await enumerationSearch(clients, spec, limit, offset); + } + + const linksManager = await createLinksManager(config, clients); + for (const hit of output.hits) { + hit.links.push( + ...linksManager.getLinks({ + bucketId: hit.bucket_id ?? undefined, + tableId: hit.table_id ?? undefined, + componentId: hit.component_id ?? undefined, + configurationId: hit.configuration_id ?? undefined, + name: hit.name ?? undefined, + }), + ); + } + + return output; + }, + }); +}; diff --git a/src/tools/semantic/detect.ts b/src/tools/semantic/detect.ts new file mode 100644 index 000000000..b7a9d70f4 --- /dev/null +++ b/src/tools/semantic/detect.ts @@ -0,0 +1,532 @@ +import { + type ConstraintValidationFinding, + type SemanticObjectType, + type SemanticServiceData, + type SemanticServiceDataTypeGroup, + type SemanticValidationServiceOutput, +} from './model'; +import { + getSemanticModelId, + loadValidationContexts, + metaName, + type MetastoreClient, + toSemanticServiceData, +} from './service'; + +// Ported 1:1 from tools/semantic/service.py — SQL-string heuristics, object +// detection, constraint evaluation, and validation orchestration. + +const POST_QUERY_CONSTRAINT_TYPES = new Set([ + 'inequality', + 'equality', + 'range', + 'temporal', + 'conditional', +]); + +// Captures the single column name from a simple aggregate metric SQL expression, +// e.g. SUM("REVENUE_YTD") -> "REVENUE_YTD", AVG(margin_pct) -> "margin_pct". +const AGGREGATE_COLUMN_RE = /^\s*\w+\s*\(\s*"?([A-Za-z_][A-Za-z0-9_]*)"?\s*\)\s*$/; + +// SQL function names / keywords never treated as column identifiers in ON clauses. +const SQL_KEYWORDS_UPPER = new Set([ + 'AND', + 'OR', + 'NOT', + 'IN', + 'IS', + 'NULL', + 'TRUE', + 'FALSE', + 'LEFT', + 'RIGHT', + 'INNER', + 'OUTER', + 'FULL', + 'CROSS', + 'JOIN', + 'ON', + 'WHERE', + 'SELECT', + 'FROM', + 'AS', + 'BY', + 'GROUP', + 'AVG', + 'SUM', + 'COUNT', + 'MIN', + 'MAX', + 'COALESCE', + 'NULLIF', + 'CAST', + 'CONCAT', + 'TRIM', + 'LENGTH', + 'UPPER', + 'LOWER', + 'IFF', + 'CASE', + 'WHEN', + 'THEN', + 'ELSE', + 'END', +]); + +const matchesSql = (sqlQuery: string, candidate: string): boolean => { + if (!candidate) return false; + const candidateLower = candidate.toLowerCase(); + const sqlLower = sqlQuery.toLowerCase(); + if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(candidate)) { + const escaped = candidateLower.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const pattern = new RegExp(`(? { + const validationQuery = constraint.validationQuery; + if (validationQuery == null) return null; + + const dialectKey = (sqlDialect ?? '').trim().toLowerCase(); + if (dialectKey === 'snowflake' && typeof validationQuery.snowflake === 'string') { + return validationQuery.snowflake; + } + if (dialectKey === 'bigquery' && typeof validationQuery.bigquery === 'string') { + return validationQuery.bigquery; + } + const defaultQuery = validationQuery.default; + return typeof defaultQuery === 'string' ? defaultQuery : null; +}; + +const constraintMessage = (constraint: SemanticServiceData, defaultMessage: string): string => { + const err = constraint.errorMessage?.trim(); + const rem = constraint.remediation?.trim(); + if (err) { + return rem ? `${err} Remediation: ${rem}` : err; + } + if (rem) return `${defaultMessage} Remediation: ${rem}`; + return defaultMessage; +}; + +const datasetIdentifiers = (dataset: SemanticServiceData): string[] => + [dataset.fqn] + .filter((c): c is string => typeof c === 'string' && c.trim().length > 0) + .map((c) => c.trim()); + +const extractMetricColumn = (sql: string): string | null => { + const m = AGGREGATE_COLUMN_RE.exec(sql); + return m ? m[1]! : null; +}; + +const metricIdentifiers = (metric: SemanticServiceData): string[] => { + const candidates: string[] = []; + if (metric.sql) { + candidates.push(metric.sql); + const col = extractMetricColumn(metric.sql); + if (col) candidates.push(col); + } + return candidates.map((c) => c.trim()).filter((c) => c.length > 0); +}; + +const detectUsedDatasets = ( + sqlQuery: string, + datasets: SemanticServiceData[], +): SemanticServiceData[] => + datasets.filter((dataset) => + datasetIdentifiers(dataset).some((candidate) => matchesSql(sqlQuery, candidate)), + ); + +const detectUsedMetricsForDatasets = ( + sqlQuery: string, + metrics: SemanticServiceData[], + usedDatasetIds: Set, +): SemanticServiceData[] => { + const matches: SemanticServiceData[] = []; + for (const metric of metrics) { + if (metric.dataset == null || !usedDatasetIds.has(metric.dataset)) continue; + if (metricIdentifiers(metric).some((candidate) => matchesSql(sqlQuery, candidate))) { + matches.push(metric); + } + } + return matches; +}; + +const extractJoinColumns = (onClause: string): string[] => { + const cleaned = onClause.replace(/'[^']*'/g, ''); + const tokens = cleaned.match(/\b[A-Z][A-Z0-9_]{2,}\b/g) ?? []; + const seen = new Set(); + const result: string[] = []; + for (const token of tokens) { + if (!SQL_KEYWORDS_UPPER.has(token) && !seen.has(token)) { + seen.add(token); + result.push(token); + } + } + return result; +}; + +const detectUsedRelationships = ( + sqlQuery: string, + relationships: SemanticServiceData[], + usedDatasetIds: Set, +): SemanticServiceData[] => { + const matches: SemanticServiceData[] = []; + for (const relationship of relationships) { + if (relationship.fromDataset == null || relationship.toDataset == null) continue; + if ( + !usedDatasetIds.has(relationship.fromDataset) || + !usedDatasetIds.has(relationship.toDataset) + ) { + continue; + } + if (relationship.on && relationship.on.trim()) { + const colNames = extractJoinColumns(relationship.on); + if (colNames.length) { + if (!colNames.every((col) => matchesSql(sqlQuery, col))) continue; + } else if (!matchesSql(sqlQuery, relationship.on)) { + continue; + } + } + matches.push(relationship); + } + return matches; +}; + +const constraintIsRelevant = ( + constraint: SemanticServiceData, + usedMetricNames: Set, + usedDatasetIds: Set, +): boolean => { + const constraintMetrics = new Set( + (constraint.metrics ?? []).map((m) => m.trim()).filter((m) => m.length > 0), + ); + const constraintDatasets = new Set( + (constraint.datasets ?? []).map((d) => d.trim()).filter((d) => d.length > 0), + ); + if (constraintMetrics.size && [...usedMetricNames].some((m) => constraintMetrics.has(m))) + return true; + if (constraintDatasets.size && [...usedDatasetIds].some((d) => constraintDatasets.has(d))) + return true; + return constraintMetrics.size === 0 && constraintDatasets.size === 0; +}; + +// --- Detect + evaluate --------------------------------------------------------- + +const emptyGroup = (objectType: SemanticObjectType): SemanticServiceDataTypeGroup => ({ + objectType, + objects: [], +}); + +const detectUsedObjectsFromContext = ( + sqlQuery: string, + contextByType: Map, + usedObjectsByType: Map, +): Map => { + const datasets = contextByType.get('semantic-dataset') ?? emptyGroup('semantic-dataset'); + const metrics = contextByType.get('semantic-metric') ?? emptyGroup('semantic-metric'); + const relationships = + contextByType.get('semantic-relationship') ?? emptyGroup('semantic-relationship'); + + let usedDatasetObjects = detectUsedDatasets(sqlQuery, datasets.objects); + const expectedDatasets = usedObjectsByType.get('semantic-dataset'); + if (expectedDatasets) { + const ids = new Set(usedDatasetObjects.map((o) => o.id)); + usedDatasetObjects = usedDatasetObjects.concat( + expectedDatasets.objects.filter((o) => !ids.has(o.id)), + ); + } + const usedDatasetIds = new Set( + usedDatasetObjects.map((item) => (item.tableId ?? '').trim()).filter((id) => id.length > 0), + ); + + let usedMetricObjects = detectUsedMetricsForDatasets(sqlQuery, metrics.objects, usedDatasetIds); + const expectedMetrics = usedObjectsByType.get('semantic-metric'); + if (expectedMetrics) { + const ids = new Set(usedMetricObjects.map((o) => o.id)); + usedMetricObjects = usedMetricObjects.concat( + expectedMetrics.objects.filter((o) => !ids.has(o.id)), + ); + } + + let usedRelationshipObjects = detectUsedRelationships( + sqlQuery, + relationships.objects, + usedDatasetIds, + ); + const expectedRelationships = usedObjectsByType.get('semantic-relationship'); + if (expectedRelationships) { + const ids = new Set(usedRelationshipObjects.map((o) => o.id)); + usedRelationshipObjects = usedRelationshipObjects.concat( + expectedRelationships.objects.filter((o) => !ids.has(o.id)), + ); + } + + const usedGroups = new Map(); + if (usedDatasetObjects.length) { + usedGroups.set('semantic-dataset', { + objectType: 'semantic-dataset', + objects: usedDatasetObjects, + }); + } + if (usedMetricObjects.length) { + usedGroups.set('semantic-metric', { + objectType: 'semantic-metric', + objects: usedMetricObjects, + }); + } + if (usedRelationshipObjects.length) { + usedGroups.set('semantic-relationship', { + objectType: 'semantic-relationship', + objects: usedRelationshipObjects, + }); + } + return usedGroups; +}; + +const relationshipNames = (objects: SemanticServiceData[]): string[] => + objects.map((item) => item.name || metaName(item.data) || item.id).sort(); + +const evaluateConstraintsFromContext = ( + contextByType: Map, + usedObjectGroupsByType: Map, +): SemanticValidationServiceOutput => { + const modelGroup = contextByType.get('semantic-model') ?? emptyGroup('semantic-model'); + const model = modelGroup.objects[0] ?? null; + const constraints = ( + contextByType.get('semantic-constraint') ?? emptyGroup('semantic-constraint') + ).objects; + + const usedDatasetObjects = usedObjectGroupsByType.get('semantic-dataset')?.objects ?? []; + const usedMetricObjects = usedObjectGroupsByType.get('semantic-metric')?.objects ?? []; + const usedRelationshipObjects = + usedObjectGroupsByType.get('semantic-relationship')?.objects ?? []; + + const usedDatasetIds = new Set( + usedDatasetObjects.map((i) => (i.tableId ?? '').trim()).filter((i) => i.length > 0), + ); + const usedMetricNames = new Set( + usedMetricObjects.map((i) => (i.name ?? '').trim()).filter((i) => i.length > 0), + ); + const matchedRelationships = relationshipNames(usedRelationshipObjects); + + const sqlDialectStr = model ? model.sqlDialect : null; + const violations: ConstraintValidationFinding[] = []; + const postExecutionChecks: ConstraintValidationFinding[] = []; + let hasError = false; + + for (const constraint of constraints) { + if (!constraintIsRelevant(constraint, usedMetricNames, usedDatasetIds)) continue; + + const constraintName = constraint.name || metaName(constraint.data) || constraint.id; + const severity = constraint.severity || 'error'; + const constraintType = constraint.constraintType || 'unknown'; + const validationQuery = pickValidationQuery(constraint, sqlDialectStr); + const constraintMetrics = (constraint.metrics ?? []) + .map((m) => m.trim()) + .filter((m) => m.length > 0); + const constraintDatasets = (constraint.datasets ?? []) + .map((d) => d.trim()) + .filter((d) => d.length > 0); + const preQueryCheck = constraint.preQueryCheck ?? false; + + if (constraintType === 'composition') { + const missingMetrics = constraintMetrics.filter((m) => !usedMetricNames.has(m)); + if (missingMetrics.length) { + if (severity === 'error') hasError = true; + violations.push({ + constraint_id: constraint.id, + constraint_name: constraintName, + severity, + status: 'missing_metrics', + message: constraintMessage( + constraint, + `Constraint "${constraintName}" expects metrics present in the SQL: ${missingMetrics.join(', ')}.`, + ), + validation_query: validationQuery, + }); + } + continue; + } + + if (constraintType === 'exclusion') { + const usedExcludedMetrics = constraintMetrics.filter((m) => usedMetricNames.has(m)); + const usedExcludedDatasets = constraintDatasets.filter((d) => usedDatasetIds.has(d)); + if (usedExcludedMetrics.length > 1 || usedExcludedDatasets.length > 1) { + if (severity === 'error') hasError = true; + violations.push({ + constraint_id: constraint.id, + constraint_name: constraintName, + severity, + status: 'excluded_combination', + message: constraintMessage( + constraint, + `Constraint "${constraintName}" forbids this combination of semantic objects.`, + ), + validation_query: validationQuery, + }); + } + continue; + } + + if (preQueryCheck) { + if (severity === 'error') hasError = true; + violations.push({ + constraint_id: constraint.id, + constraint_name: constraintName, + severity, + status: 'pre_query_check', + message: constraintMessage( + constraint, + `Constraint "${constraintName}" should be explicitly checked before trusting the query result.`, + ), + validation_query: validationQuery, + }); + continue; + } + + if (!POST_QUERY_CONSTRAINT_TYPES.has(constraintType) && validationQuery === null) { + continue; + } + + postExecutionChecks.push({ + constraint_id: constraint.id, + constraint_name: constraintName, + severity, + status: 'post_query_check', + message: constraintMessage( + constraint, + `Constraint "${constraintName}" is relevant for this SQL and should be verified against the result.`, + ), + validation_query: validationQuery, + }); + } + + return { + valid: !hasError, + usedObjectGroups: [...usedObjectGroupsByType.values()], + matchedRelationships, + violations, + postExecutionChecks, + }; +}; + +const mergeContexts = ( + contexts: Map[], +): Map => { + const merged = new Map(); + for (const context of contexts) { + for (const [objectType, group] of context) { + const list = merged.get(objectType) ?? []; + list.push(...group.objects); + merged.set(objectType, list); + } + } + return new Map([...merged].map(([objectType, objects]) => [objectType, { objectType, objects }])); +}; + +const filterUsedObjectsByModel = ( + usedObjectGroupsByType: Map, + modelId: string, +): Map => { + const filtered = new Map(); + for (const [objectType, group] of usedObjectGroupsByType) { + const modelObjects = group.objects.filter((obj) => getSemanticModelId(obj) === modelId); + if (modelObjects.length) filtered.set(objectType, { objectType, objects: modelObjects }); + } + return filtered; +}; + +const mergeUsedObjectGroups = ( + usedObjectGroups: SemanticServiceDataTypeGroup[], +): Map => { + const merged = new Map(); + for (const group of usedObjectGroups) { + const list = merged.get(group.objectType) ?? []; + list.push(...group.objects); + merged.set(group.objectType, list); + } + return new Map([...merged].map(([objectType, objects]) => [objectType, { objectType, objects }])); +}; + +const evaluateUsedObjectsForContexts = ( + semanticModelIds: readonly string[], + contextsPerModel: Map[], + usedObjectGroupsByType: Map, +): SemanticValidationServiceOutput => { + const allViolations: ConstraintValidationFinding[] = []; + const allPostChecks: ConstraintValidationFinding[] = []; + let hasError = false; + + for (let i = 0; i < semanticModelIds.length; i++) { + const modelId = semanticModelIds[i]!; + const contextByType = contextsPerModel[i]!; + const modelUsedObjects = filterUsedObjectsByModel(usedObjectGroupsByType, modelId); + const perModelResult = evaluateConstraintsFromContext(contextByType, modelUsedObjects); + allViolations.push(...perModelResult.violations); + allPostChecks.push(...perModelResult.postExecutionChecks); + if (!perModelResult.valid) hasError = true; + } + + const usedRelationships = + usedObjectGroupsByType.get('semantic-relationship') ?? emptyGroup('semantic-relationship'); + const matchedRelationships = relationshipNames(usedRelationships.objects); + + return { + valid: !hasError, + usedObjectGroups: [...usedObjectGroupsByType.values()], + matchedRelationships, + violations: allViolations, + postExecutionChecks: allPostChecks, + }; +}; + +export const validateSemanticQueryWithUsedObjects = async ( + client: MetastoreClient, + sqlQuery: string, + semanticModelIds: readonly string[], + opts: { + usedObjectGroups?: SemanticServiceDataTypeGroup[]; + contextsPerModel?: Map[]; + } = {}, +): Promise => { + if (!sqlQuery.trim()) { + throw new Error('sql_query must not be empty.'); + } + const cleanedModelIds = [ + ...new Set(semanticModelIds.map((m) => m.trim()).filter((m) => m.length > 0)), + ]; + if (!cleanedModelIds.length) { + throw new Error('At least one semantic_model_id must be provided.'); + } + + const contextsPerModel = + opts.contextsPerModel ?? (await loadValidationContexts(client, cleanedModelIds)); + const usedObjectGroups = opts.usedObjectGroups ?? []; + + const mergedContext = mergeContexts(contextsPerModel); + const usedByType = detectUsedObjectsFromContext( + sqlQuery, + mergedContext, + mergeUsedObjectGroups(usedObjectGroups), + ); + return evaluateUsedObjectsForContexts(cleanedModelIds, contextsPerModel, usedByType); +}; + +export const getObjectById = async ( + client: MetastoreClient, + objectType: SemanticObjectType, + objectId: string, +): Promise => { + const rawObj = await client.getObject(objectType, objectId); + if (rawObj.type !== objectType) { + throw new Error( + `Expected object "${objectId}" to be of type "${objectType}", got "${rawObj.type}" from the Metastore API.`, + ); + } + return toSemanticServiceData(objectType, rawObj); +}; diff --git a/src/tools/semantic/index.ts b/src/tools/semantic/index.ts new file mode 100644 index 000000000..21ec99737 --- /dev/null +++ b/src/tools/semantic/index.ts @@ -0,0 +1,2 @@ +// Semantic tools module — mirrors the Python tools/semantic/{tools,service,model}.py layout. +export { registerSemanticTools } from './tools'; diff --git a/src/tools/semantic/model.ts b/src/tools/semantic/model.ts new file mode 100644 index 000000000..35839f277 --- /dev/null +++ b/src/tools/semantic/model.ts @@ -0,0 +1,114 @@ +import { z } from 'zod'; + +// Ported from tools/semantic/model.py. + +/** Semantic object types handled by the semantic tools. */ +export const SEMANTIC_OBJECT_TYPE = [ + 'semantic-model', + 'semantic-dataset', + 'semantic-metric', + 'semantic-relationship', + 'semantic-glossary', + 'semantic-constraint', +] as const; + +export type SemanticObjectType = (typeof SEMANTIC_OBJECT_TYPE)[number]; + +export const SemanticObjectTypeEnum = z.enum(SEMANTIC_OBJECT_TYPE); + +/** Semantic object type selection used by semantic tools (object_type + optional ids). */ +export const SemanticObjectTypeSelectionSchema = z.object({ + object_type: SemanticObjectTypeEnum.describe('Semantic object type to load.'), + ids: z + .array(z.string()) + .default([]) + .describe( + 'Specific object UUIDs to include. Empty list [] means include all objects of this type.', + ), +}); + +export type SemanticObjectTypeSelection = z.infer; + +/** Typed semantic object reference. */ +export type SemanticObjectRef = { + object_type: SemanticObjectType; + id: string; +}; + +// --- Metastore object shape (JSON:API `data` envelope item) -------------------- + +export type MetaObjectMeta = { + name?: string | null; + [key: string]: unknown; +}; + +/** Single object from the Metastore JSON:API response (port of `MetastoreObject`). */ +export type MetastoreObject = { + type?: string | null; + id?: string | null; + attributes?: Record | null; + relationships?: Record | null; + meta?: MetaObjectMeta | null; +}; + +// --- Typed service objects (port of SemanticServiceData hierarchy) ------------- + +export type SemanticServiceData = { + semanticType: SemanticObjectType; + id: string; + data: MetastoreObject; + attributes: Record; + /** display_name: own `name`, else meta.name (glossary overrides with term). */ + displayName: string | null; + // Type-specific fields used by the service heuristics: + name?: string | null; + sqlDialect?: string | null; + tableId?: string | null; + fqn?: string | null; + modelUuid?: string | null; + sql?: string | null; + dataset?: string | null; + fromDataset?: string | null; + toDataset?: string | null; + on?: string | null; + term?: string | null; + description?: string | null; + constraintType?: string | null; + severity?: string | null; + metrics?: string[]; + datasets?: string[]; + errorMessage?: string | null; + remediation?: string | null; + preQueryCheck?: boolean; + validationQuery?: Record | null; +}; + +export type SemanticServiceDataTypeGroup = { + objectType: SemanticObjectType; + objects: SemanticServiceData[]; +}; + +export type ConstraintValidationFinding = { + constraint_id: string; + constraint_name: string; + severity: string; + status: string; + message: string; + validation_query: string | null; +}; + +export type SemanticValidationServiceOutput = { + valid: boolean; + usedObjectGroups: SemanticServiceDataTypeGroup[]; + matchedRelationships: string[]; + violations: ConstraintValidationFinding[]; + postExecutionChecks: ConstraintValidationFinding[]; +}; + +export type SemanticSearchHit = { + objectType: SemanticObjectType; + object: SemanticServiceData; + semanticModelId: string; + matchedPatterns: string[]; + matchedPaths: string[]; +}; diff --git a/src/tools/semantic/service.ts b/src/tools/semantic/service.ts new file mode 100644 index 000000000..74ffe86ec --- /dev/null +++ b/src/tools/semantic/service.ts @@ -0,0 +1,412 @@ +import type { RawClient } from '@/clients/raw'; +import { + type MetastoreObject, + SEMANTIC_OBJECT_TYPE, + type SemanticObjectType, + type SemanticSearchHit, + type SemanticServiceData, + type SemanticServiceDataTypeGroup, +} from './model'; + +// Ported 1:1 from tools/semantic/service.py. +// +// The semantic tools read the Metastore service (the semantic-layer repository). +// The Python `MetastoreClient` issues plain JSON:API requests; we build an +// equivalent raw client locally from the derived Metastore service URL so we keep +// exact response-shape parity (`data` envelope with per-item `attributes` + `meta`). +// +// NOTE: we deliberately KEEP this local raw client instead of the typed +// `@keboola/api-client` metastore client. The typed `getMetaObjects` exposes only a +// `filter` query parameter (no `limit`/`offset`), but these heuristics rely on +// aggressive offset paging with small per-type limits to avoid 500s on large +// responses; and the typed response nests user attributes under `attributes.data` +// and drops the `meta.name` field, which would break the shapes these heuristics +// (and the JSON:API `data` envelope parsing) depend on. + +// --- Metastore raw client ------------------------------------------------------ + +const parseList = (response: unknown): MetastoreObject[] => { + if (!response || typeof response !== 'object' || Array.isArray(response)) { + throw new Error('Unexpected metastore response format: expected JSON object with "data" key.'); + } + const data = (response as { data?: unknown }).data; + if (!Array.isArray(data)) { + throw new Error('Unexpected metastore response format: "data" is not an array.'); + } + return data as MetastoreObject[]; +}; + +const parseObject = (response: unknown): MetastoreObject => { + if (!response || typeof response !== 'object') { + throw new Error('Unexpected metastore response format: expected JSON object.'); + } + const data = (response as { data?: unknown }).data ?? response; + if (!data || typeof data !== 'object' || Array.isArray(data)) { + throw new Error('Unexpected metastore response format: "data" is not an object.'); + } + return data as MetastoreObject; +}; + +export type MetastoreClient = { + getSchema: (objectType: string) => Promise>; + listObjects: ( + objectType: string, + opts?: { limit?: number; offset?: number }, + ) => Promise; + getObject: (objectType: string, uuid: string) => Promise; +}; + +export const createMetastoreClient = (raw: RawClient): MetastoreClient => ({ + getSchema: async (objectType) => { + const response = await raw.get(`api/v1/schema/${objectType}`); + if (!response || typeof response !== 'object' || Array.isArray(response)) { + throw new Error('Unexpected metastore schema response format.'); + } + return response as Record; + }, + listObjects: async (objectType, opts = {}) => { + const params: Record = {}; + if (opts.limit !== undefined) params.limit = opts.limit; + if (opts.offset !== undefined) params.offset = opts.offset; + const response = await raw.get(`api/v1/repository/${objectType}`, { params }); + return parseList(response); + }, + getObject: async (objectType, uuid) => { + const response = await raw.get(`api/v1/repository/${objectType}/${uuid}`); + return parseObject(response); + }, +}); + +// --- Service constants --------------------------------------------------------- + +const SEMANTIC_OBJECT_TYPES: readonly SemanticObjectType[] = SEMANTIC_OBJECT_TYPE; + +export const VALIDATION_OBJECT_TYPES: readonly SemanticObjectType[] = [ + 'semantic-model', + 'semantic-dataset', + 'semantic-metric', + 'semantic-relationship', + 'semantic-constraint', +]; + +// Some metastore endpoints return 500 for large responses unless paged aggressively. +const DEFAULT_PAGE_LIMIT = 20; +const DEFAULT_PAGE_LIMITS: Partial> = { + 'semantic-dataset': 1, + 'semantic-metric': 5, +}; + +// --- Object mapping ------------------------------------------------------------ + +const asString = (value: unknown): string | null => (typeof value === 'string' ? value : null); + +const asStringArray = (value: unknown): string[] => + Array.isArray(value) + ? value.filter((v): v is string => typeof v === 'string' && v.length > 0) + : []; + +export const metaName = (obj: MetastoreObject): string | null => { + const name = obj.meta?.name; + return typeof name === 'string' && name ? name : null; +}; + +export const toSemanticServiceData = ( + objectType: SemanticObjectType, + obj: MetastoreObject, +): SemanticServiceData => { + const attributes = obj.attributes ?? {}; + const id = obj.id ?? ''; + const ownName = asString(attributes.name) || metaName(obj); + + const base: SemanticServiceData = { + semanticType: objectType, + id, + data: obj, + attributes, + displayName: ownName || null, + }; + + switch (objectType) { + case 'semantic-model': + return { + ...base, + name: ownName, + description: asString(attributes.description), + sqlDialect: asString(attributes.sql_dialect), + }; + case 'semantic-dataset': + return { + ...base, + name: ownName, + tableId: asString(attributes.tableId), + fqn: asString(attributes.fqn), + description: asString(attributes.description), + modelUuid: asString(attributes.modelUUID), + }; + case 'semantic-metric': + return { + ...base, + name: ownName, + sql: asString(attributes.sql), + dataset: asString(attributes.dataset), + description: asString(attributes.description), + modelUuid: asString(attributes.modelUUID), + }; + case 'semantic-relationship': + return { + ...base, + name: ownName, + fromDataset: asString(attributes.from), + toDataset: asString(attributes.to), + on: asString(attributes.on), + modelUuid: asString(attributes.modelUUID), + }; + case 'semantic-glossary': { + const term = asString(attributes.term); + return { + ...base, + // Glossary display name overrides with term. + displayName: term || base.displayName, + term, + modelUuid: asString(attributes.modelUUID), + }; + } + case 'semantic-constraint': { + const ai = attributes.ai; + const validationQuery = attributes.validationQuery; + return { + ...base, + name: ownName, + description: asString(attributes.description), + constraintType: asString(attributes.constraintType), + severity: asString(attributes.severity), + modelUuid: asString(attributes.modelUUID), + metrics: asStringArray(attributes.metrics), + datasets: asStringArray(attributes.datasets), + errorMessage: asString(attributes.errorMessage), + remediation: asString(attributes.remediation), + preQueryCheck: + typeof ai === 'object' && + ai !== null && + (ai as { preQueryCheck?: unknown }).preQueryCheck === true, + validationQuery: + validationQuery && typeof validationQuery === 'object' && !Array.isArray(validationQuery) + ? (validationQuery as Record) + : null, + }; + } + default: + throw new Error(`Unsupported semantic object type "${objectType}".`); + } +}; + +// --- Helpers ------------------------------------------------------------------- + +export const getSemanticModelId = (obj: SemanticServiceData): string => { + if (obj.semanticType === 'semantic-model') return obj.id; + return obj.modelUuid ?? ''; +}; + +/** Model id from a raw metastore object (used during paged listing). */ +const getModelIdFromMeta = (obj: MetastoreObject): string => { + if (obj.type === 'semantic-model') return obj.id ?? ''; + const modelId = (obj.attributes ?? {}).modelUUID; + return modelId ? String(modelId) : ''; +}; + +const stringifyValue = (value: unknown): string => { + if (typeof value === 'string') return value; + try { + return stableStringify(value); + } catch { + return String(value); + } +}; + +/** JSON stringify with sorted keys (port of json.dumps(..., sort_keys=True)). */ +const stableStringify = (value: unknown): string => { + return JSON.stringify(value, (_key, val) => { + if (val && typeof val === 'object' && !Array.isArray(val)) { + return Object.keys(val as Record) + .sort() + .reduce>((acc, k) => { + acc[k] = (val as Record)[k]; + return acc; + }, {}); + } + return val; + }); +}; + +/** Walk every scalar leaf of an object/array, yielding [dottedPath, value]. */ +function* walkLeaves(node: unknown, path: string): Generator<[string, unknown]> { + if (Array.isArray(node)) { + for (let i = 0; i < node.length; i++) { + yield* walkLeaves(node[i], `${path}[${i}]`); + } + } else if (node && typeof node === 'object') { + for (const [key, value] of Object.entries(node as Record)) { + yield* walkLeaves(value, path ? `${path}.${key}` : key); + } + } else { + yield [path, node]; + } +} + +const findMatches = ( + semanticObject: SemanticServiceData, + compiledPatterns: RegExp[], +): { matchedPaths: string[]; matchedPatterns: string[] } => { + const matchedPaths = new Set(); + const matchedPatterns = new Set(); + + if (semanticObject.displayName) { + for (const compiled of compiledPatterns) { + if (compiled.test(semanticObject.displayName)) { + matchedPaths.add('meta.name'); + matchedPatterns.add(compiled.source); + } + } + } + + const attrs = semanticObject.attributes ?? {}; + const attrsStringified = stringifyValue(attrs); + if (compiledPatterns.some((compiled) => compiled.test(attrsStringified))) { + for (const [path, value] of walkLeaves(attrs, '')) { + if (value && typeof value === 'object') continue; + const haystack = stringifyValue(value); + if (!haystack) continue; + for (const compiled of compiledPatterns) { + if (compiled.test(haystack)) { + matchedPaths.add(path); + matchedPatterns.add(compiled.source); + } + } + } + } + + return { + matchedPaths: [...matchedPaths].sort(), + matchedPatterns: [...matchedPatterns].sort(), + }; +}; + +const listSemanticTypeObjects = async ( + client: MetastoreClient, + objectType: SemanticObjectType, + semanticModelIds?: readonly string[] | null, +): Promise => { + const limit = DEFAULT_PAGE_LIMITS[objectType] ?? DEFAULT_PAGE_LIMIT; + let offset = 0; + const data: SemanticServiceData[] = []; + const modelIdSet = semanticModelIds && semanticModelIds.length ? new Set(semanticModelIds) : null; + + for (;;) { + const page = await client.listObjects(objectType, { limit, offset }); + for (const obj of page) { + if (modelIdSet === null || modelIdSet.has(getModelIdFromMeta(obj))) { + data.push(toSemanticServiceData(objectType, obj)); + } + } + if (page.length < limit) return data; + offset += limit; + } +}; + +// --- Service: search ----------------------------------------------------------- + +export const searchSemanticContext = async ( + client: MetastoreClient, + patterns: string[], + opts: { + semanticTypes?: readonly SemanticObjectType[]; + semanticModelIds?: readonly string[] | null; + caseSensitive?: boolean; + maxResults?: number; + }, +): Promise => { + const cleanedPatterns = patterns.map((p) => p.trim()).filter((p) => p.length > 0); + if (cleanedPatterns.length === 0) { + throw new Error('At least one regex pattern must be provided.'); + } + const maxResults = opts.maxResults ?? 50; + if (maxResults <= 0) { + throw new Error('max_results must be a positive integer.'); + } + + const targetTypes = + opts.semanticTypes && opts.semanticTypes.length ? opts.semanticTypes : SEMANTIC_OBJECT_TYPES; + const flags = opts.caseSensitive ? '' : 'i'; + const compiledPatterns: RegExp[] = []; + for (const pattern of cleanedPatterns) { + try { + compiledPatterns.push(new RegExp(pattern, flags)); + } catch (e) { + throw new Error( + `Invalid regex pattern "${pattern}": ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + + const matches: SemanticSearchHit[] = []; + for (const objectType of targetTypes) { + if (matches.length >= maxResults) break; + const objects = await listSemanticTypeObjects(client, objectType, opts.semanticModelIds); + for (const semanticObject of objects) { + if (matches.length >= maxResults) break; + const { matchedPaths, matchedPatterns } = findMatches(semanticObject, compiledPatterns); + if (matchedPatterns.length === 0) continue; + matches.push({ + objectType, + semanticModelId: getSemanticModelId(semanticObject), + object: semanticObject, + matchedPatterns: [...matchedPatterns].sort(), + matchedPaths: [...matchedPaths].sort(), + }); + } + } + return matches.slice(0, maxResults); +}; + +// --- Service: load context ----------------------------------------------------- + +export const loadSemanticContextForType = async ( + client: MetastoreClient, + objectType: SemanticObjectType, + opts: { ids?: readonly string[]; semanticModelIds?: readonly string[] | null } = {}, +): Promise => { + let objects: SemanticServiceData[]; + if (opts.ids && opts.ids.length) { + const raw = await Promise.all(opts.ids.map((id) => client.getObject(objectType, id))); + objects = raw.map((obj) => toSemanticServiceData(objectType, obj)); + } else { + objects = await listSemanticTypeObjects(client, objectType, opts.semanticModelIds); + } + return { objectType, objects }; +}; + +const loadSemanticContextForModel = async ( + client: MetastoreClient, + semanticModelId: string, +): Promise> => { + const groups = await Promise.all( + VALIDATION_OBJECT_TYPES.map((objectType) => + loadSemanticContextForType(client, objectType, { semanticModelIds: [semanticModelId] }), + ), + ); + return new Map(groups.map((g) => [g.objectType, g])); +}; + +// --- Service: validation contexts ---------------------------------------------- + +export const loadValidationContexts = async ( + client: MetastoreClient, + semanticModelIds: readonly string[], +): Promise[]> => { + if (!semanticModelIds.length) { + throw new Error('At least one semantic_model_id must be provided.'); + } + return Promise.all( + semanticModelIds.map((modelId) => loadSemanticContextForModel(client, modelId)), + ); +}; diff --git a/src/tools/semantic/tools.ts b/src/tools/semantic/tools.ts new file mode 100644 index 000000000..d8bc32ab5 --- /dev/null +++ b/src/tools/semantic/tools.ts @@ -0,0 +1,607 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { createRawClient } from '@/clients/raw'; +import { deriveServiceUrls } from '@/clients/urls'; +import type { Config } from '@/config'; +import { registerTool } from '@/mcp/tool'; +import { getObjectById, validateSemanticQueryWithUsedObjects } from './detect'; +import { + type SemanticObjectRef, + type SemanticObjectType, + SemanticObjectTypeEnum, + type SemanticObjectTypeSelection, + SemanticObjectTypeSelectionSchema, + type SemanticServiceData, + type SemanticServiceDataTypeGroup, + type SemanticValidationServiceOutput, +} from './model'; +import { + createMetastoreClient, + loadSemanticContextForType, + loadValidationContexts, + type MetastoreClient, + searchSemanticContext, +} from './service'; + +// Ported 1:1 from tools/semantic/tools.py. + +const asString = (value: unknown): string | null => (typeof value === 'string' ? value : null); + +// --- Tool-facing compact views (port of model classes in tools.py) ------------- + +const compactSemanticObject = (obj: SemanticServiceData): Record => { + const a = obj.attributes; + switch (obj.semanticType) { + case 'semantic-model': + return { + id: obj.id, + name: obj.displayName, + description: asString(a.description), + sql_dialect: asString(a.sql_dialect), + }; + case 'semantic-dataset': + return { + id: obj.id, + name: obj.displayName, + tableId: asString(a.tableId), + description: asString(a.description), + model_uuid: asString(a.modelUUID), + fqn: asString(a.fqn), + }; + case 'semantic-metric': + return { + id: obj.id, + name: obj.displayName, + description: asString(a.description), + dataset: asString(a.dataset), + model_uuid: asString(a.modelUUID), + }; + case 'semantic-relationship': + return { + id: obj.id, + name: obj.displayName, + from_dataset: asString(a.from), + to_dataset: asString(a.to), + type: asString(a.type), + on: asString(a.on), + model_uuid: asString(a.modelUUID), + }; + case 'semantic-glossary': + return { + id: obj.id, + name: obj.displayName, + term: asString(a.term), + definition: asString(a.definition), + model_uuid: asString(a.modelUUID), + }; + case 'semantic-constraint': + return { + id: obj.id, + name: obj.displayName, + description: asString(a.description), + type: asString(a.constraintType), + rule: asString(a.rule), + severity: asString(a.severity), + model_uuid: asString(a.modelUUID), + }; + default: + throw new Error(`Unsupported semantic object type "${obj.semanticType}"`); + } +}; + +const fullSemanticObject = (obj: SemanticServiceData): Record => ({ + id: obj.id, + name: obj.displayName, + attributes: obj.attributes ?? {}, +}); + +const compactName = (compact: Record): string => + (compact.name as string | null) || (compact.id as string); + +const usedDatasetView = (obj: SemanticServiceData): Record => ({ + id: obj.id, + name: obj.name || '', + tableId: obj.tableId || '', + description: obj.description || '', + fqn: obj.fqn || '', +}); + +const usedMetricView = (obj: SemanticServiceData): Record => ({ + id: obj.id, + name: obj.name || '', + description: obj.description || '', + sql: obj.sql || '', + dataset: obj.dataset || '', +}); + +const formatValidationResult = ( + rawResult: SemanticValidationServiceOutput, + opts: { models?: SemanticServiceData[]; summaryNotes?: string[] } = {}, +): Record => { + const models = opts.models ?? []; + const summaryNotes = opts.summaryNotes ?? []; + + let usedDatasetObjects: SemanticServiceData[] = []; + let usedMetricObjects: SemanticServiceData[] = []; + for (const group of rawResult.usedObjectGroups) { + if (group.objectType === 'semantic-dataset') usedDatasetObjects = group.objects; + else if (group.objectType === 'semantic-metric') usedMetricObjects = group.objects; + } + + const usedDatasets = usedDatasetObjects.map(usedDatasetView); + const usedMetrics = usedMetricObjects.map(usedMetricView); + + const semanticModelOutputs = models.map((m) => compactSemanticObject(m)); + const sqlDialects = [ + ...new Set(models.map((m) => m.sqlDialect).filter((d): d is string => Boolean(d))), + ].sort(); + + const summaryParts: string[] = []; + if (sqlDialects.length > 1) { + summaryParts.push( + `Warning: semantic models use different SQL dialects (${sqlDialects.join(', ')}). ` + + 'The query may not be portable across all models.', + ); + } + if (rawResult.violations.length) { + summaryParts.push( + 'Semantic validation found pre-execution issues that should be fixed before running.', + ); + } + if (rawResult.postExecutionChecks.length) { + summaryParts.push('Some checks should be verified after execution.'); + } + summaryParts.push(...summaryNotes); + + const summary = summaryParts.length + ? summaryParts.join('\n') + : 'Semantic validation finished without relevant findings.'; + + return { + valid: rawResult.valid, + semantic_models: semanticModelOutputs, + sql_dialects: sqlDialects, + used_datasets: usedDatasets, + used_metrics: usedMetrics, + matched_relationships: rawResult.matchedRelationships, + violations: rawResult.violations, + post_execution_checks: rawResult.postExecutionChecks, + summary, + }; +}; + +const compareExpectedAndDetectedObjects = ( + expectedSemanticObjects: SemanticObjectTypeSelection[], + usedObjectGroups: SemanticServiceDataTypeGroup[], +): { + matched: SemanticObjectRef[]; + missing: SemanticObjectRef[]; + unexpected: Record[]; +} => { + if (!expectedSemanticObjects.length) return { matched: [], missing: [], unexpected: [] }; + + const expectedIdsByType = new Map>(); + for (const selection of expectedSemanticObjects) { + if (selection.ids.length) { + const set = expectedIdsByType.get(selection.object_type) ?? new Set(); + for (const id of selection.ids) set.add(id); + expectedIdsByType.set(selection.object_type, set); + } + } + const expectedTypes = new Set(expectedSemanticObjects.map((s) => s.object_type)); + + const matched: SemanticObjectRef[] = []; + const missing: SemanticObjectRef[] = []; + const unexpected: Record[] = []; + + for (const [objectType, expectedIds] of expectedIdsByType) { + const detectedIds = new Set( + usedObjectGroups + .filter((g) => g.objectType === objectType) + .flatMap((g) => g.objects.map((o) => o.id)), + ); + const both = [...expectedIds].filter((id) => detectedIds.has(id)).sort(); + const onlyExpected = [...expectedIds].filter((id) => !detectedIds.has(id)).sort(); + matched.push(...both.map((id) => ({ object_type: objectType, id }))); + missing.push(...onlyExpected.map((id) => ({ object_type: objectType, id }))); + } + + for (const group of usedObjectGroups) { + const selectionIds = expectedIdsByType.get(group.objectType); + let unexpectedObjects: Record[]; + if (!expectedTypes.has(group.objectType)) { + unexpectedObjects = group.objects.map(compactSemanticObject); + } else if (selectionIds && selectionIds.size) { + unexpectedObjects = group.objects + .filter((obj) => !selectionIds.has(obj.id)) + .map(compactSemanticObject); + } else { + unexpectedObjects = []; + } + if (unexpectedObjects.length) { + unexpected.push({ object_type: group.objectType, objects: unexpectedObjects }); + } + } + + return { matched, missing, unexpected }; +}; + +// --- Tool registration --------------------------------------------------------- + +export const registerSemanticTools = (server: McpServer, config: Config): void => { + const buildClient = (): MetastoreClient => { + if (!config.storageApiUrl) throw new Error('Storage API URL is not configured.'); + if (!config.storageToken) throw new Error('Storage API token is not configured.'); + const urls = deriveServiceUrls(config.storageApiUrl); + const token = config.bearerToken ? `Bearer ${config.bearerToken}` : config.storageToken; + return createMetastoreClient(createRawClient({ baseUrl: urls.metastore, token })); + }; + + registerTool(server, { + name: 'search_semantic_context', + title: 'Search semantic context', + annotations: { readOnlyHint: true }, + description: + 'Searches semantic models and semantic objects using regex patterns matched against their names, descriptions and\n' + + 'stringified JSON attributes.\n\n' + + 'Returns compact matches grouped by semantic model. Each match includes the semantic object type,\n' + + 'the paths where the patterns matched, and compact object view.\n\n' + + 'CONSIDERATIONS:\n' + + '- The search is case-insensitive by default. Use `case_sensitive=True` when exact casing matters.\n' + + '- The search is performed against semantic object names and data attributes which are stringified JSON objects\n' + + 'following their corresponding JSON schema.\n' + + '- The search can be scoped to specific semantic models or semantic object types but prefer broader search without\n' + + 'scoping unless required by the context.\n\n' + + 'WHEN TO USE:\n' + + '- When you need to discover which semantic objects are relevant to a user request.\n' + + '- When you know business terms, column names, metric fragments, or rule names, but not exact object UUIDs.\n' + + '- When you need to find semantic objects by keyword or values used in their attributes.\n\n' + + 'WHEN NOT TO USE:\n' + + '- When you know the exact IDs.\n\n' + + 'EXAMPLES:\n' + + '- Find semantic objects by business concepts for revenue or sales:\n' + + ' `patterns=["revenue", "sales"]`\n' + + '- Find semantic objects using a Keboola table ID:\n' + + ' `patterns=["out.c-sales-main.fact_orders"]`\n' + + '- Find semantic dataset for a certain table:\n' + + ' `patterns=["in.c-sales-main.fact_orders"], semantic_types=["semantic-dataset"]`\n' + + '- Find semantic datasets that mention a column name:\n' + + ' `patterns=["column_name"], semantic_types=["semantic-dataset"]`\n' + + '- Search semantic objects e.g. semantic metrics, relationships, and constraints using a certain semantic dataset:\n' + + ' `patterns=["table-id-of-the-dataset"], semantic_types=["semantic-metric",`\n' + + ' `"semantic-relationship", "semantic-constraint"]`\n' + + '- Search semantic constraints using e.g. certain semantic metrics and certain semantic datasets:\n' + + ' `patterns=["metric-name-1", "metric-name-2", "table-id-from-the-dataset"],`\n' + + ' `semantic_types=["semantic-metric", "semantic-relationship"]`\n' + + '- Search something within specific semantic models only:\n' + + ' `patterns=["something"], semantic_model_ids=["", ""]`', + inputSchema: { + patterns: z + .array(z.string()) + .describe( + 'One or more regex patterns used to search semantic metadata. ' + + 'The search checks semantic model names plus semantic object names and nested attribute values. ' + + 'Use multiple patterns when you need to find objects related to several business terms at once.', + ), + semantic_types: z + .array(SemanticObjectTypeEnum) + .default([]) + .describe( + 'Optional semantic object types to search. ' + + 'Empty list [] means ALL semantic object types are searched. ' + + 'Use this to narrow the search when you already know whether you want datasets, metrics, ' + + 'relationships, glossary terms, constraints, or models.', + ), + semantic_model_ids: z + .array(z.string()) + .default([]) + .describe( + 'Optional list of semantic model IDs to restrict the search to specific models. ' + + 'Empty list [] means search across all semantic models.', + ), + case_sensitive: z + .boolean() + .default(false) + .describe( + 'Whether regex matching should be case-sensitive. ' + + 'Leave false for normal discovery; set true only when exact casing matters.', + ), + max_results: z + .number() + .int() + .default(100) + .describe( + 'Maximum number of matched semantic objects to return. ' + + 'Use a smaller value for quick discovery and a larger value only when you need a broader result set.', + ), + }, + handler: async (args) => { + const cleanedPatterns = args.patterns.filter((p) => p && p.trim()).map((p) => p.trim()); + if (!cleanedPatterns.length) throw new Error('At least one regex pattern must be provided.'); + if (args.max_results <= 0) throw new Error('max_results must be a positive integer.'); + + const client = buildClient(); + const hits = await searchSemanticContext(client, cleanedPatterns, { + semanticTypes: args.semantic_types, + semanticModelIds: args.semantic_model_ids.length ? args.semantic_model_ids : null, + caseSensitive: args.case_sensitive, + maxResults: args.max_results, + }); + + const grouped = new Map[]>(); + for (const hit of hits) { + const list = grouped.get(hit.semanticModelId) ?? []; + list.push({ + object_type: hit.objectType, + matched_paths: hit.matchedPaths, + data: compactSemanticObject(hit.object), + }); + grouped.set(hit.semanticModelId, list); + } + + const modelResults = [...grouped.entries()].map(([modelId, matches]) => ({ + semantic_model_id: modelId, + matches: [...matches].sort((a, b) => + compactName(a.data as Record).localeCompare( + compactName(b.data as Record), + ), + ), + })); + modelResults.sort((a, b) => a.semantic_model_id.localeCompare(b.semantic_model_id)); + return modelResults; + }, + }); + + registerTool(server, { + name: 'get_semantic_context', + title: 'Get semantic context', + annotations: { readOnlyHint: true }, + description: + 'Loads semantic objects grouped by semantic object type.\n\n' + + 'CONSIDERATIONS:\n' + + '- If a selection has empty `ids`, the tool returns all objects of that type in compact form.\n' + + '- If a selection has non-empty `ids`, the tool returns only those specific objects with full attributes.\n' + + '- `semantic_model_ids` optionally narrows the lookup to specific semantic models.\n\n' + + 'WHEN TO USE:\n' + + '- When you already know IDs of the semantic objects you want to load and want to inspect them in detail.\n' + + '- When you want to list all semantic objects of certain types or specific semantic models.\n' + + '- When you want to list semantic models.\n\n' + + 'WHEN NOT TO USE:\n' + + '- When you need to discover semantic objects.\n\n' + + 'EXAMPLES:\n' + + '- List all semantic models:\n' + + ' `semantic_objects=[{"object_type": "semantic-model"}]`\n' + + '- List semantic datasets and metrics for specific semantic models:\n' + + ' `semantic_objects=[{"object_type": "semantic-dataset"}, {"object_type": "semantic-metric"}],`\n' + + ' `semantic_model_ids=["model-uuid-1", "model-uuid-2"]`\n' + + '- Get detailed context for specific semantic objects by their id:\n' + + ' `semantic_objects=[{"object_type": "semantic-dataset", "ids": ["dataset-uuid-1"]},`\n' + + ' `{"object_type": "semantic-metric", "ids": ["metric-uuid-1", "metric-uuid-2"]}]`\n' + + '- List all constraints for specific semantic models:\n' + + ' `semantic_objects=[{"object_type": "semantic-constraint"}], semantic_model_ids=["model-uuid-1"]`', + inputSchema: { + semantic_objects: z + .array(SemanticObjectTypeSelectionSchema) + .describe( + 'List of semantic object selections to load. ' + + 'Each item contains "object_type" and optional "ids". ' + + 'If "ids" is empty, all objects of that type are returned in compact form. ' + + 'If "ids" is non-empty, only those objects are returned with full attributes.', + ), + semantic_model_ids: z + .array(z.string()) + .default([]) + .describe( + 'Optional list of semantic model IDs to restrict loading to specific models. ' + + 'Empty list [] means load across all semantic models.', + ), + }, + handler: async (args) => { + if (!args.semantic_objects.length) { + throw new Error('At least one semantic object type must be provided.'); + } + const client = buildClient(); + const modelIds = args.semantic_model_ids.length ? args.semantic_model_ids : null; + + const groups = await Promise.all( + args.semantic_objects.map((selection) => + loadSemanticContextForType(client, selection.object_type, { + semanticModelIds: modelIds, + ids: selection.ids, + }), + ), + ); + + return args.semantic_objects.map((selection, i) => { + const context = groups[i]!; + if (selection.ids.length) { + return { + object_type: context.objectType, + objects: context.objects.map(fullSemanticObject), + }; + } + return { + object_type: context.objectType, + objects: context.objects.map(compactSemanticObject), + }; + }); + }, + }); + + registerTool(server, { + name: 'get_semantic_schema', + title: 'Get semantic schema', + annotations: { readOnlyHint: true }, + description: + 'Returns JSON schemas for the requested semantic object types.\n\n' + + 'WHEN TO USE:\n' + + '- When you want to know the JSON schema of a semantic object type, e.g. before searching something specific.', + inputSchema: { + semantic_types: z + .array(SemanticObjectTypeEnum) + .describe( + 'List of semantic object types for which JSON schemas should be returned. ' + + 'Each returned item contains the requested semantic type and its metastore schema.', + ), + }, + handler: async (args) => { + if (!args.semantic_types.length) { + throw new Error('At least one semantic type must be provided.'); + } + const client = buildClient(); + const schemas = await Promise.all( + args.semantic_types.map((semanticType) => client.getSchema(semanticType)), + ); + return args.semantic_types.map((semanticType, i) => ({ + semantic_type: semanticType, + schema: schemas[i]!, + })); + }, + }); + + registerTool(server, { + name: 'validate_semantic_query', + title: 'Validate semantic query', + annotations: { readOnlyHint: true }, + description: + 'Performs best-effort semantic validation of an SQL query against one or more semantic models and compares it with\n' + + 'the expected semantic objects provided.\n\n' + + 'RETURNS:\n' + + '- `validation_auto_detected`: semantic validation built from objects heuristically detected in the SQL\n' + + '- `validation_detected_from_expected`: semantic validation built only from explicitly provided expected object IDs\n' + + '- expected semantic objects that were matched or missing in the auto-detected result\n' + + '- unexpected auto-detected objects outside the expected semantic scope\n\n' + + 'LIMITATIONS:\n' + + '- Detection is heuristic and based on string matching over SQL and semantic metadata.\n' + + '- The tool does not parse SQL semantically and does not execute the query.\n' + + '- Auto-detected objects, missing objects, and relationship matches may therefore be imperfect.\n' + + '- Use the result as a best-effort semantic check, not as a formal proof that the query is correct.\n\n' + + 'CONSIDERATIONS:\n' + + '- Prefer calling this tool before executing any SQL that touches semantic objects.\n' + + '- This tool confirms the SQL dialect, surfaces semantic constraint violations, and provides post-execution checks.\n' + + '- Only proceed to query_data once this tool returns valid=True and violations is empty. If violations are found,\n' + + 'fix the query first or consider the limitations of this tool.\n\n' + + 'WHEN TO USE:\n' + + '- Before generating or approving a query that should follow a semantic model.\n' + + '- When you want to validate a SQL query against the semantic objects before executing it using "query_data" tool\n' + + 'or creating a new SQL transformation out of it, especially when investigating data quality issues.\n' + + '- When you want to verify that a query uses the intended semantic objects.\n' + + '- When you need to surface semantic business-rule violations or follow-up checks.\n\n' + + 'EXAMPLES:\n' + + '- Validate a SQL query against one semantic model:\n' + + ' `sql_query="SELECT SUM(\\"REVENUE\\") FROM ...", semantic_model_ids=["semantic-model-uuid"],`\n' + + ' `expected_semantic_objects=[{"object_type": "semantic-dataset"}]`\n' + + '- Validate a cross-model query against two semantic models:\n' + + ' `sql_query="SELECT * FROM ...", semantic_model_ids=["model-uuid-1", "model-uuid-2"],`\n' + + ' `expected_semantic_objects=[{"object_type": "semantic-dataset", "ids": ["dataset-uuid-1"]}]`\n' + + '- Validate a query and compare it against expected objects:\n' + + ' `sql_query="SELECT SUM(\\"REVENUE\\") FROM ...", semantic_model_ids=["semantic-model-uuid"],`\n' + + ' `expected_semantic_objects=[{"object_type": "semantic-metric", "ids": ["metric-uuid-1"]}]`', + inputSchema: { + sql_query: z + .string() + .describe( + 'SQL query that should be checked against the semantic layer. ' + + 'The query is not executed; the tool performs best-effort semantic detection and rule validation ' + + 'using heuristic string matching, so the detected objects may be incomplete or imperfect.', + ), + semantic_model_ids: z + .array(z.string()) + .describe( + 'One or more semantic model IDs against which the SQL should be validated. ' + + 'Contexts from all models are merged into a single universe for object detection. ' + + 'Constraint evaluation is performed per model to avoid cross-model rule contamination.', + ), + expected_semantic_objects: z + .array(SemanticObjectTypeSelectionSchema) + .default([]) + .describe( + 'Optional semantic object selections that define the expected semantic scope of the query. ' + + 'These expectations are compared with the objects actually detected in the SQL. ' + + 'Use `ids` when you want to assert that specific semantic objects should be present.', + ), + }, + handler: async (args) => { + if (!args.sql_query.trim()) throw new Error('sql_query must not be empty.'); + const cleanedModelIds = [ + ...new Set(args.semantic_model_ids.filter((m) => m && m.trim()).map((m) => m.trim())), + ]; + if (!cleanedModelIds.length) { + throw new Error('At least one semantic_model_id must be provided.'); + } + + const client = buildClient(); + + const models = await Promise.all( + cleanedModelIds.map((modelId) => getObjectById(client, 'semantic-model', modelId)), + ); + + // Pre-load contexts once when both validation paths will run. + let preLoadedContexts: Map[] | undefined; + if (args.expected_semantic_objects.length) { + preLoadedContexts = await loadValidationContexts(client, cleanedModelIds); + } + + const rawAutoDetected = await validateSemanticQueryWithUsedObjects( + client, + args.sql_query, + cleanedModelIds, + { contextsPerModel: preLoadedContexts }, + ); + + let matched: SemanticObjectRef[] = []; + let missing: SemanticObjectRef[] = []; + let unexpected: Record[] = []; + let rawFromExpected: SemanticValidationServiceOutput | null = null; + + if (args.expected_semantic_objects.length) { + ({ matched, missing, unexpected } = compareExpectedAndDetectedObjects( + args.expected_semantic_objects, + rawAutoDetected.usedObjectGroups, + )); + const modelIds = args.semantic_model_ids.length ? args.semantic_model_ids : null; + const expectedObjectGroups = await Promise.all( + args.expected_semantic_objects.map((selection) => + loadSemanticContextForType(client, selection.object_type, { + semanticModelIds: modelIds, + ids: selection.ids, + }), + ), + ); + if (expectedObjectGroups.length) { + rawFromExpected = await validateSemanticQueryWithUsedObjects( + client, + args.sql_query, + cleanedModelIds, + { usedObjectGroups: expectedObjectGroups, contextsPerModel: preLoadedContexts }, + ); + } + } + + const autoDetectedSummaryNotes: string[] = []; + if (missing.length) { + autoDetectedSummaryNotes.push( + 'Some expected semantic objects were not detected in the SQL query.', + ); + } + if (unexpected.length) { + autoDetectedSummaryNotes.push( + 'Some detected semantic objects fall outside the expected semantic scope.', + ); + } + + return { + validation_auto_detected: formatValidationResult(rawAutoDetected, { + models, + summaryNotes: autoDetectedSummaryNotes, + }), + validation_detected_from_expected: + rawFromExpected !== null ? formatValidationResult(rawFromExpected, { models }) : null, + matched_expected_objects: matched, + missing_expected_objects: missing, + unexpected_detected_objects: unexpected, + }; + }, + }); +}; diff --git a/src/tools/sql.ts b/src/tools/sql.ts new file mode 100644 index 000000000..c98fe158f --- /dev/null +++ b/src/tools/sql.ts @@ -0,0 +1,163 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { createKeboolaClients } from '@/clients/keboola'; +import { createRawClient } from '@/clients/raw'; +import { deriveServiceUrls } from '@/clients/urls'; +import type { Config } from '@/config'; +import { logger } from '@/logger'; +import { registerTool } from '@/mcp/tool'; +import { type SqlSelectData, WorkspaceManager } from '@/workspace'; + +// Ported from tools/sql.py. + +const MAX_ROWS = 1_000; +const MAX_CHARS = 50_000; + +const QUERY_DATA_DESCRIPTION = ` + Executes an SQL SELECT query to get the data from the underlying database. + + BEFORE QUERYING: + * Always verify the table has a non-null fullyQualifiedName from get_tables tool. + If it does not, the table is not SQL-accessible from this workspace — do not attempt the query and inform user. + + CRITICAL SQL REQUIREMENTS: + + * ALWAYS check the SQL dialect before constructing queries. + * Do not include any comments in the SQL code + * Use delimited identifiers and FQN format for the current SQL dialect. + + TABLE AND COLUMN REFERENCES: + * Always use fully qualified table names in the exact FQN format provided by table information tools + * Follow the identifier structure exactly as shown by table info tools for the current SQL dialect + * Always use delimited identifiers when referring to table columns + + CTE (WITH CLAUSE) RULES: + * ALL column references in main query MUST match exact case used in the CTE + * If you alias a column in a CTE, reference it under the aliased name in the subsequent queries + * Define all column aliases explicitly in CTEs + * Use delimited identifiers in both CTE definition and references to preserve case + + FUNCTION COMPATIBILITY: + * Check data types before using date functions (DATE_TRUNC, EXTRACT require proper date/timestamp types) + * Cast VARCHAR columns to appropriate types before using in date/numeric functions + + ERROR PREVENTION: + * Never pass empty strings ('') where numeric or date values are expected + * Use NULLIF or CASE statements to handle empty values + * Always use TRY_CAST or similar safe casting functions when converting data types + * Check for division by zero using NULLIF(denominator, 0) + * Always use the LIMIT clause in your SELECT statements when fetching data. There are hard limits imposed + by this tool on the maximum number of rows that can be fetched and the maximum number of characters. + The tool will truncate the data if those limits are exceeded. + + DATA VALIDATION: + * When querying columns with categorical values, use query_data tool to inspect distinct values beforehand + * Ensure valid filtering by checking actual data values first + `; + +/** + * Serializes rows to CSV, matching Python's `csv.DictWriter` defaults: + * comma delimiter, `\r\n` line terminator, minimal quoting (quote a field only + * when it contains the delimiter, a quote, CR, or LF; double embedded quotes). + */ +const toCsv = (data: SqlSelectData): string => { + const needsQuote = (field: string): boolean => + field.includes(',') || field.includes('"') || field.includes('\n') || field.includes('\r'); + const formatField = (value: unknown): string => { + const s = value === null || value === undefined ? '' : String(value); + return needsQuote(s) ? `"${s.replaceAll('"', '""')}"` : s; + }; + const lines: string[] = []; + lines.push(data.columns.map(formatField).join(',')); + for (const row of data.rows) { + lines.push(data.columns.map((col) => formatField(row[col])).join(',')); + } + return lines.map((line) => `${line}\r\n`).join(''); +}; + +/** + * Builds the WorkspaceManager from the resolved config. The query service / + * workspace-discovery clients are built locally (the shared `createKeboolaClients` + * does not expose a query service client), rooted at `query.`. + */ +export const createWorkspaceManager = async (config: Config): Promise => { + const clients = createKeboolaClients(config); + const urls = deriveServiceUrls(config.storageApiUrl ?? ''); + // Query Service host: no dedicated key in deriveServiceUrls — derive `query.` + // from the storage host, matching the Python `query.` derivation. + const suffix = new URL(urls.storage).hostname.slice('connection.'.length); + const queryServiceUrl = `https://query.${suffix}`; + + // Query Service prefers the OAuth bearer token, falling back to the SAPI token. + const queryServiceToken = config.bearerToken + ? `Bearer ${config.bearerToken}` + : (config.storageToken ?? ''); + + // A production-branch raw storage client for the legacy / default-branch fallback path. + const storageToken = config.bearerToken + ? `Bearer ${config.bearerToken}` + : (config.storageToken ?? ''); + const makeProdRawStorage = () => + createRawClient({ baseUrl: `${urls.storage}/v2/storage`, token: storageToken }); + + return WorkspaceManager.create(config, { + rawStorage: clients.rawStorage, + makeProdRawStorage, + queryServiceUrl, + queryServiceToken, + }); +}; + +export const registerSqlTools = (server: McpServer, config: Config): void => { + registerTool(server, { + name: 'query_data', + title: 'Query data', + description: QUERY_DATA_DESCRIPTION, + annotations: { readOnlyHint: true }, + inputSchema: { + sql_query: z.string().describe('SQL SELECT query to run.'), + query_name: z + .string() + .describe( + 'A concise, human-readable name for this query based on its purpose and what data it retrieves. ' + + 'Use normal words with spaces (e.g., "Customer Orders Last Month", "Top Selling Products", ' + + '"User Activity Summary").', + ), + }, + handler: async (args) => { + const workspaceManager = await createWorkspaceManager(config); + + const result = await workspaceManager.executeQuery(args.sql_query, { + maxRows: MAX_ROWS, + maxChars: MAX_CHARS, + }); + + if (result.status === 'ok') { + logger.info( + [`Query "${args.query_name}" executed successfully.`, result.message] + .filter(Boolean) + .join(' '), + ); + const data: SqlSelectData = result.data + ? result.data + : // Non-SELECT query (should not happen for this SELECT-only tool). + { columns: ['message'], rows: [{ message: result.message }] }; + + return { + query_name: args.query_name, + csv_data: toCsv(data), + message: result.message ?? null, + }; + } + + // Surface cancellation cleanly without the generic "Failed to run SQL query" prefix. + if (result.message === 'Query was cancelled') { + logger.info(`Query "${args.query_name}" was cancelled.`); + throw new Error('Query was cancelled'); + } + logger.warn([`Query "${args.query_name}" failed.`, result.message].filter(Boolean).join(' ')); + throw new Error(`Failed to run SQL query, error: ${result.message}`); + }, + }); +}; diff --git a/src/keboola_mcp_server/resources/storage-schema.json b/src/tools/storage-schema.json similarity index 100% rename from src/keboola_mcp_server/resources/storage-schema.json rename to src/tools/storage-schema.json diff --git a/src/tools/storage/index.ts b/src/tools/storage/index.ts new file mode 100644 index 000000000..f0ea41dfb --- /dev/null +++ b/src/tools/storage/index.ts @@ -0,0 +1,13 @@ +/** + * Public entry point for the storage tool module. + * + * Preserves the import path `@/tools/storage` for downstream consumers + * (`registerStorageTools` in src/server.ts). + * + * The model/serialization layer (bucket/table types, metadata accessors, dialect-aware + * FQN/quoting helpers) and the lineage-usage helpers are re-exported so they remain + * reachable through `@/tools/storage`. + */ +export { registerStorageTools } from './tools'; +export * from './model'; +export * from './usage'; diff --git a/src/tools/storage/model.ts b/src/tools/storage/model.ts new file mode 100644 index 000000000..556c32513 --- /dev/null +++ b/src/tools/storage/model.ts @@ -0,0 +1,410 @@ +// Ported from tools/storage/tools.py (BucketDetail, TableSummary, TableDetail, TableColumnInfo) +// and tools/storage_helpers.py / clients/client.py metadata accessors. The dialect-aware FQN +// builder is a port of workspace.py `_SnowflakeWorkspace.get_table_info` / +// `_BigQueryWorkspace.get_table_info`. + +import { MetadataField } from '@/constants'; +import type { Link } from '@/links'; +import { type ComponentUsageReference, getCreatedBy, getLastUpdatedBy } from './usage'; + +// --------------------------------------------------------------------------- +// Metadata helpers (ports of clients/client.py get_metadata_property and +// tools/components/utils.py get_nested, utils.py parse_iso_timestamp). +// --------------------------------------------------------------------------- + +export type RawObj = Record; + +const FAKE_DEVELOPMENT_BRANCH = 'KBC.fakeDevelopmentBranch'; +const SHARED_DESCRIPTION = 'KBC.sharedDescription'; +const DATATYPE_BASETYPE = 'KBC.datatype.basetype'; +const DATATYPE_TYPE = 'KBC.datatype.type'; +const DATATYPE_NULLABLE = 'KBC.datatype.nullable'; + +/** Parse an ISO 8601 timestamp into epoch millis, accepting `Z` and `+HHMM` offsets. */ +export const parseIsoTimestamp = (ts: string): number => { + const normalized = ts.replace('Z', '+00:00').replace(/([+-]\d{2})(\d{2})$/, '$1:$2'); + const millis = Date.parse(normalized); + if (Number.isNaN(millis)) throw new Error(`Invalid ISO timestamp: ${ts}`); + return millis; +}; + +/** Port of get_metadata_property: most-recent value for `key`, optionally provider-ranked. */ +export const getMetadataProperty = ( + metadata: unknown, + key: string, + preferredProviders?: string[], +): string | null => { + if (!Array.isArray(metadata)) return null; + const filtered = (metadata as RawObj[]).filter((m) => m && m.key === key); + if (filtered.length === 0) return null; + const sortKey = (m: RawObj): [number, string] => { + const ts = (m.timestamp as string) ?? ''; + if (preferredProviders) { + const p = m.provider as string | undefined; + const idx = + p && preferredProviders.includes(p) + ? preferredProviders.indexOf(p) + : preferredProviders.length; + return [-1 * idx, ts]; + } + return [0, ts]; + }; + let best: RawObj | undefined; + let bestKey: [number, string] | undefined; + for (const m of filtered) { + const k = sortKey(m); + if (!bestKey || k[0] > bestKey[0] || (k[0] === bestKey[0] && k[1] > bestKey[1])) { + best = m; + bestKey = k; + } + } + const value = best?.value; + return value != null ? String(value) : null; +}; + +/** Port of get_nested: dot-path lookup through nested objects. */ +export const getNested = (obj: unknown, path: string): unknown => { + let cur: unknown = obj; + for (const part of path.split('.')) { + if (cur && typeof cur === 'object' && !Array.isArray(cur)) { + cur = (cur as RawObj)[part]; + } else { + return undefined; + } + if (cur == null) return undefined; + } + return cur; +}; + +/** Most recent of the given ISO timestamps (string-preserving), or null. */ +export const maxTimestamp = (...timestamps: (string | null | undefined)[]): string | null => { + const valid = timestamps.filter((ts): ts is string => Boolean(ts)); + if (valid.length === 0) return null; + const score = (ts: string): [number, number | string] => { + try { + return [1, parseIsoTimestamp(ts)]; + } catch { + return [0, ts]; + } + }; + let best = valid[0]!; + let bestScore = score(best); + for (const ts of valid.slice(1)) { + const s = score(ts); + if (s[0] > bestScore[0] || (s[0] === bestScore[0] && s[1] > bestScore[1])) { + best = ts; + bestScore = s; + } + } + return best; +}; + +const asNumberOrNull = (value: unknown): number | null => { + if (value == null) return null; + const n = Number(value); + return Number.isNaN(n) ? null : n; +}; + +// --------------------------------------------------------------------------- +// SQL dialect + fully-qualified-name / quoting helpers (port of workspace.py +// `get_backend_path`, `TableFqn`, and the per-backend `get_table_info`). +// +// The project backend is resolved from the verified token's `owner.defaultBackend` +// (see tools.ts), avoiding a live workspace round-trip. +// --------------------------------------------------------------------------- + +export type Dialect = 'snowflake' | 'bigquery'; + +/** Quote a single identifier for the given dialect. */ +export const quotedName = (name: string, dialect: Dialect): string => + dialect === 'bigquery' ? `\`${name}\`` : `"${name}"`; + +/** Default native type for a column with no `KBC.datatype.type` metadata. */ +const defaultNativeType = (dialect: Dialect): string => + dialect === 'bigquery' ? 'STRING' : 'VARCHAR'; + +const getBackendPath = (rawTable: RawObj): string[] | null => { + const bucket = rawTable.bucket; + const backendPath = + bucket && typeof bucket === 'object' ? (bucket as RawObj).backendPath : undefined; + return Array.isArray(backendPath) ? (backendPath as string[]) : null; +}; + +/** + * Build a table's fully qualified name from its bucket backendPath, dialect-aware. + * + * - Snowflake: database.schema.table → `"db"."schema"."name"` (port of + * `_SnowflakeWorkspace.get_table_info`; requires backendPath length >= 2). + * - BigQuery: dataset.table → `` `dataset`.`name` `` (port of + * `_BigQueryWorkspace.get_table_info`). There is no cross-project (database) tier, + * backendPath[0] is the dataset name (separators normalized to `_`), and a table that + * is an alias in its source project is not materialized into this dataset → no FQN. + * + * Returns null when no FQN can be constructed (the table is then not queryable). + */ +export const tableFqn = (rawTable: RawObj, dialect: Dialect): string | null => { + const name = String(rawTable.name ?? ''); + if (dialect === 'bigquery') { + const sourceTable = rawTable.sourceTable as RawObj | undefined; + if (sourceTable && sourceTable.isAlias) return null; + const bp = getBackendPath(rawTable); + if (!bp || bp.length < 1) return null; + const dataset = String(bp[0]).replace(/[.-]/g, '_'); + return [dataset, name].map((p) => quotedName(p, dialect)).join('.'); + } + const bp = getBackendPath(rawTable); + if (!bp || bp.length < 2) return null; + return [bp[0], bp[1], name].map((p) => quotedName(String(p), dialect)).join('.'); +}; + +// --------------------------------------------------------------------------- +// Bucket / table models (ports of BucketDetail, TableSummary, TableDetail). +// --------------------------------------------------------------------------- + +export type Bucket = { + id: string; + name: string; + displayName: string; + description: string | null; + stage: string; + created: string; + updated: string | null; + dataSizeBytes: number | null; + tablesCount: number | null; + links: Link[] | null; + source_project: string | null; + created_by: ComponentUsageReference | null; + last_updated_by: ComponentUsageReference | null; + // internal, excluded from output + branch_id: string | null; + prod_id: string; +}; + +export const validateBucket = (raw: RawObj): Bucket => { + const id = String(raw.id ?? ''); + const metadata = raw.metadata; + + const branchId = getMetadataProperty(metadata, FAKE_DEVELOPMENT_BRANCH); + const prodId = branchId ? id.replace(`c-${branchId}-`, 'c-') : id; + + const description = + getMetadataProperty(metadata, SHARED_DESCRIPTION) || + getMetadataProperty(metadata, MetadataField.DESCRIPTION) || + (raw.description as string | undefined) || + null; + + const tables = raw.tables; + const tablesCount = Array.isArray(tables) ? tables.length : null; + + let sourceProject: string | null = null; + const sp = getNested(raw, 'sourceBucket.project') as RawObj | undefined; + if (sp) sourceProject = `${sp.name} (ID: ${sp.id})`; + + const updated = (raw.updated as string | undefined) || maxTimestamp(raw.lastChangeDate as string); + + return { + id, + name: String(raw.name ?? ''), + displayName: String(raw.displayName ?? raw.display_name ?? ''), + description: description || null, + stage: String(raw.stage ?? ''), + created: String(raw.created ?? ''), + updated: updated || null, + dataSizeBytes: asNumberOrNull(raw.dataSizeBytes), + tablesCount, + links: null, + source_project: sourceProject, + created_by: null, + last_updated_by: null, + branch_id: branchId || null, + prod_id: prodId, + }; +}; + +/** Port of BucketDetail.with_lineage_metadata. */ +export const withBucketLineage = (bucket: Bucket, raw: RawObj): Bucket => { + const metadata = raw.metadata; + if (!Array.isArray(metadata) || metadata.length === 0) return bucket; + const lastUpdatedBy = getLastUpdatedBy(metadata); + return { + ...bucket, + created_by: getCreatedBy(metadata), + last_updated_by: lastUpdatedBy, + updated: maxTimestamp(bucket.updated, lastUpdatedBy?.timestamp ?? null), + }; +}; + +export type TableColumnInfo = { + name: string; + quotedName: string; + database_native_type: string; + nullable: boolean; + keboola_base_type: string | null; + description: string | null; +}; + +export type Table = { + id: string; + name: string; + displayName: string; + description: string | null; + primaryKey: string | null; // serialized as '|'-joined string (port of serialize_primary_key) + created: string | null; + updated: string | null; + rowsCount: number | null; + dataSizeBytes: number | null; + links: Link[] | null; + source_project: string | null; + // detail-only fields (absent on summaries) + columns?: TableColumnInfo[] | null; + fullyQualifiedName?: string | null; + used_by?: ComponentUsageReference[] | null; + created_by?: ComponentUsageReference | null; + last_updated_by?: ComponentUsageReference | null; + // internal + branch_id: string | null; + prod_id: string; + isDetail: boolean; +}; + +export const validateTableCommon = (raw: RawObj): Omit => { + const id = String(raw.id ?? ''); + const metadata = raw.metadata; + + const branchId = getMetadataProperty(metadata, FAKE_DEVELOPMENT_BRANCH); + const prodId = branchId ? id.replace(`c-${branchId}-`, 'c-') : id; + + const description = + getMetadataProperty(metadata, MetadataField.DESCRIPTION) || + getMetadataProperty(getNested(raw, 'sourceTable.metadata') ?? [], MetadataField.DESCRIPTION) || + (raw.description as string | undefined) || + null; + + let sourceProject: string | null = null; + const sp = getNested(raw, 'sourceTable.project') as RawObj | undefined; + if (sp) sourceProject = `${sp.name} (ID: ${sp.id})`; + + const updated = + (raw.updated as string | undefined) || + maxTimestamp(raw.lastChangeDate as string, raw.lastImportDate as string); + + const pk = raw.primaryKey; + const primaryKey = Array.isArray(pk) && pk.length ? (pk as string[]).join('|') : null; + + return { + id, + name: String(raw.name ?? ''), + displayName: String(raw.displayName ?? raw.display_name ?? ''), + description: description || null, + primaryKey, + created: (raw.created as string | undefined) ?? null, + updated: updated || null, + rowsCount: asNumberOrNull(raw.rowsCount), + dataSizeBytes: asNumberOrNull(raw.dataSizeBytes), + links: null, + source_project: sourceProject, + branch_id: branchId || null, + prod_id: prodId, + }; +}; + +export const withTableLineage = (table: Table, raw: RawObj): Table => { + const metadata = raw.metadata; + if (!Array.isArray(metadata) || metadata.length === 0) return table; + const lastUpdatedBy = getLastUpdatedBy(metadata); + return { + ...table, + created_by: getCreatedBy(metadata), + last_updated_by: lastUpdatedBy, + updated: maxTimestamp(table.updated, lastUpdatedBy?.timestamp ?? null), + }; +}; + +/** + * Build the detail column listing for a table (port of the per-column loop in `_get_table`). + * Native type defaults are dialect-aware (Snowflake VARCHAR, BigQuery STRING). + */ +export const buildTableColumns = (rawTable: RawObj, dialect: Dialect): TableColumnInfo[] => { + const rawColumns = Array.isArray(rawTable.columns) ? (rawTable.columns as string[]) : []; + const columnMetadata = (rawTable.columnMetadata as Record) ?? {}; + const sourceColumnMetadata = + (getNested(rawTable, 'sourceTable.columnMetadata') as Record) ?? {}; + + return rawColumns.map((colName) => { + const colMeta = columnMetadata[colName] ?? []; + const srcMeta = sourceColumnMetadata[colName] ?? []; + + const description = + getMetadataProperty(colMeta, MetadataField.DESCRIPTION) || + getMetadataProperty(srcMeta, MetadataField.DESCRIPTION) || + null; + const baseType = + getMetadataProperty(colMeta, DATATYPE_BASETYPE, ['user']) || + getMetadataProperty(srcMeta, DATATYPE_BASETYPE, ['user']) || + null; + let nativeType = + getMetadataProperty(colMeta, DATATYPE_TYPE) || getMetadataProperty(srcMeta, DATATYPE_TYPE); + const nullableStr = + getMetadataProperty(colMeta, DATATYPE_NULLABLE) || + getMetadataProperty(srcMeta, DATATYPE_NULLABLE); + + if (nativeType === null) { + nativeType = defaultNativeType(dialect); + } + const nullable = + nullableStr != null ? ['1', 'true'].includes(String(nullableStr).toLowerCase()) : false; + + return { + name: colName, + quotedName: quotedName(colName, dialect), + database_native_type: nativeType, + nullable, + keboola_base_type: baseType, + description, + }; + }); +}; + +// Strip internal/absent fields before emitting (mirrors pydantic exclude + the +// TableSummary vs TableDetail field split). isDetail tables keep their detail fields. +export const serializeBucket = (b: Bucket): RawObj => ({ + id: b.id, + name: b.name, + displayName: b.displayName, + description: b.description, + stage: b.stage, + created: b.created, + updated: b.updated, + dataSizeBytes: b.dataSizeBytes, + tablesCount: b.tablesCount, + links: b.links, + source_project: b.source_project, + created_by: b.created_by, + last_updated_by: b.last_updated_by, +}); + +export const serializeTable = (t: Table): RawObj => { + const base: RawObj = { + id: t.id, + name: t.name, + displayName: t.displayName, + description: t.description, + primaryKey: t.primaryKey, + created: t.created, + updated: t.updated, + rowsCount: t.rowsCount, + dataSizeBytes: t.dataSizeBytes, + links: t.links, + source_project: t.source_project, + }; + if (t.isDetail) { + base.columns = t.columns ?? null; + base.fullyQualifiedName = t.fullyQualifiedName ?? null; + base.used_by = t.used_by ?? null; + base.created_by = t.created_by ?? null; + base.last_updated_by = t.last_updated_by ?? null; + } + return base; +}; + +export const FAKE_DEVELOPMENT_BRANCH_KEY = FAKE_DEVELOPMENT_BRANCH; diff --git a/src/tools/storage/tools.ts b/src/tools/storage/tools.ts new file mode 100644 index 000000000..f728d38cb --- /dev/null +++ b/src/tools/storage/tools.ts @@ -0,0 +1,568 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; + +import { createKeboolaClients, createLinksManager } from '@/clients/keboola'; +import { type RawClient, RawHttpError } from '@/clients/raw'; +import type { Config } from '@/config'; +import { MetadataField } from '@/constants'; +import type { Link, ProjectLinksManager } from '@/links'; +import { logger } from '@/logger'; +import { registerTool } from '@/mcp/tool'; +import { + type Bucket, + buildTableColumns, + type Dialect, + FAKE_DEVELOPMENT_BRANCH_KEY, + getMetadataProperty, + type RawObj, + serializeBucket, + serializeTable, + type Table, + tableFqn, + validateBucket, + validateTableCommon, + withBucketLineage, + withTableLineage, +} from './model'; + +// Ported from tools/storage/tools.py (get_buckets, get_tables, update_descriptions), +// tools/storage_helpers.py and tools/storage/usage.py. The model/serialization layer lives in +// ./model, lineage references in ./usage. + +// --------------------------------------------------------------------------- +// Project backend resolution. The SQL dialect is needed to build dialect-aware +// fully-qualified names; it is read from the verified token's owner.defaultBackend, +// which is far cheaper than provisioning a live workspace. Snowflake is assumed +// when the field is absent (the dominant backend / legacy projects). +// --------------------------------------------------------------------------- + +const resolveDialect = async (raw: RawClient): Promise => { + try { + const tokenInfo = await raw.get('tokens/verify'); + const owner = (tokenInfo.owner ?? {}) as RawObj; + return owner.defaultBackend === 'bigquery' ? 'bigquery' : 'snowflake'; + } catch (error) { + logger.warn( + `get_tables: failed to resolve project backend (${String(error)}); defaulting to snowflake.`, + ); + return 'snowflake'; + } +}; + +// --------------------------------------------------------------------------- +// Branch-aware fetch helpers (port of storage_helpers.py, production-branch +// path). The TS client surface has no branchId; createKeboolaClients exposes a +// production-only `default` branch alias, matching has_storage_branches=false. +// --------------------------------------------------------------------------- + +const safeBucketDetail = async ( + raw: RawClient, + bucketId: string, + branchId: string, +): Promise => { + try { + return await raw.get(`branch/${branchId}/buckets/${bucketId}`); + } catch (error) { + if (error instanceof RawHttpError && error.status === 404) return null; + throw error; + } +}; + +// --------------------------------------------------------------------------- +// Links packing (port of GetBucketsOutput.pack_links / GetTablesOutput.pack_links). +// --------------------------------------------------------------------------- + +const sortLinks = (links: Link[]): Link[] => + [...links].sort((a, b) => + a.type < b.type ? -1 : a.type > b.type ? 1 : a.title < b.title ? -1 : a.title > b.title ? 1 : 0, + ); + +const dedupeLinks = (links: Link[]): Link[] => { + const seen = new Set(); + const out: Link[] = []; + for (const link of links) { + const key = `${link.type} ${link.title} ${link.url}`; + if (!seen.has(key)) { + seen.add(key); + out.push(link); + } + } + return out; +}; + +// --------------------------------------------------------------------------- +// Table listing / detail (ports of _list_tables and _get_table, production path). +// --------------------------------------------------------------------------- + +const TABLE_LIST_INCLUDES = [ + 'metadata', + 'columnMetadata', + 'sourceMetadata', + 'sourceColumnMetadata', +]; + +const listTables = async ( + raw: RawClient, + branchId: string, + bucketIds: string[], + linksManager: ProjectLinksManager, +): Promise => { + const tablesByProdId = new Map(); + + for (const bucketId of bucketIds) { + const prodRaw = await safeBucketDetail(raw, bucketId, branchId); + if (!prodRaw) continue; + const prodBucket = validateBucket(prodRaw); + if (prodBucket.branch_id) continue; // production path + + const rawTables = await raw.get( + `branch/${branchId}/buckets/${prodBucket.id}/tables`, + { + params: { include: TABLE_LIST_INCLUDES.join(',') }, + }, + ); + for (const rawTable of rawTables) { + const tableName = String(rawTable.name ?? ''); + const summary: Table = { + ...validateTableCommon(rawTable), + isDetail: false, + links: [linksManager.getTableDetailLink(prodBucket.id, tableName)], + }; + tablesByProdId.set(summary.id, summary); + } + } + + return [...tablesByProdId.values()]; +}; + +const getTableDetail = async ( + raw: RawClient, + branchId: string, + tableId: string, + linksManager: ProjectLinksManager, + dialect: Dialect, +): Promise => { + let rawTable: RawObj | null; + try { + rawTable = await raw.get(`branch/${branchId}/tables/${tableId}`); + } catch (error) { + if (error instanceof RawHttpError && error.status === 404) return null; + throw error; + } + if (!rawTable) return null; + + // production path: a table carrying branch metadata is not a prod table. + if (getMetadataProperty(rawTable.metadata, FAKE_DEVELOPMENT_BRANCH_KEY)) return null; + + const columns = buildTableColumns(rawTable, dialect); + + const bucketInfo = (rawTable.bucket as RawObj) ?? {}; + const bucketId = String(bucketInfo.id ?? ''); + const tableName = String(rawTable.name ?? ''); + + const common = validateTableCommon(rawTable); + let table: Table = { + ...common, + isDetail: true, + columns, + fullyQualifiedName: tableFqn(rawTable, dialect), + used_by: null, + created_by: null, + last_updated_by: null, + links: [linksManager.getTableDetailLink(bucketId, tableName)], + }; + table = withTableLineage(table, rawTable); + // collapse dev id to prod id (no-op on production path) + return { ...table, id: table.prod_id, branch_id: null }; +}; + +type ItemType = 'bucket' | 'table' | 'column'; +type ParsedItemId = { + itemType: ItemType; + bucketId?: string; + tableId?: string; + columnName?: string; +}; +type UpdateItemResult = { item_id: string; success: boolean; error?: string; timestamp?: string }; +type MetadataEntry = { key?: string; value?: string; timestamp?: string }; + +/** Parse "in.c-bucket[.table[.column]]" into its parts (port of _parse_item_id). */ +const parseItemId = (itemId: string): ParsedItemId => { + if (!itemId.startsWith('in.') && !itemId.startsWith('out.')) { + throw new Error(`Invalid item_id format: ${itemId} - must start with in. or out.`); + } + const parts = itemId.split('.'); + if (parts.length === 2) { + return { itemType: 'bucket', bucketId: itemId }; + } + if (parts.length === 3) { + return { itemType: 'table', bucketId: `${parts[0]}.${parts[1]}`, tableId: itemId }; + } + if (parts.length === 4) { + return { + itemType: 'column', + bucketId: `${parts[0]}.${parts[1]}`, + tableId: `${parts[0]}.${parts[1]}.${parts[2]}`, + columnName: parts[3], + }; + } + throw new Error(`Invalid item_id format: ${itemId}`); +}; + +const findDescriptionEntry = (entries: MetadataEntry[]): MetadataEntry | undefined => + entries.find((entry) => entry.key === MetadataField.DESCRIPTION); + +const updateBucketDescription = async ( + raw: RawClient, + bucketId: string, + description: string, +): Promise => { + try { + const response = await raw.post(`buckets/${bucketId}/metadata`, { + body: { + provider: 'user', + metadata: [{ key: MetadataField.DESCRIPTION, value: description }], + }, + }); + return { + item_id: bucketId, + success: true, + timestamp: findDescriptionEntry(response)?.timestamp, + }; + } catch (error) { + return { + item_id: bucketId, + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } +}; + +const updateTableDescription = async ( + raw: RawClient, + tableId: string, + description: string, +): Promise => { + try { + const response = await raw.post<{ metadata?: MetadataEntry[] }>(`tables/${tableId}/metadata`, { + body: { + provider: 'user', + metadata: [{ key: MetadataField.DESCRIPTION, value: description }], + }, + }); + return { + item_id: tableId, + success: true, + timestamp: findDescriptionEntry(response.metadata ?? [])?.timestamp, + }; + } catch (error) { + return { + item_id: tableId, + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } +}; + +const updateColumnDescriptions = async ( + raw: RawClient, + tableId: string, + columnUpdates: Record, +): Promise => { + try { + const columnsMetadata: Record = {}; + for (const [columnName, description] of Object.entries(columnUpdates)) { + columnsMetadata[columnName] = [ + { key: MetadataField.DESCRIPTION, value: description, columnName } as MetadataEntry, + ]; + } + + const response = await raw.post<{ columnsMetadata?: Record }>( + `tables/${tableId}/metadata`, + { body: { provider: 'user', columnsMetadata } }, + ); + + const returned = response.columnsMetadata ?? {}; + return Object.keys(columnUpdates).map((columnName) => { + const entry = findDescriptionEntry(returned[columnName] ?? []); + return entry + ? { item_id: `${tableId}.${columnName}`, success: true, timestamp: entry.timestamp } + : { + item_id: `${tableId}.${columnName}`, + success: false, + error: 'No description metadata returned.', + }; + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return Object.keys(columnUpdates).map((columnName) => ({ + item_id: `${tableId}.${columnName}`, + success: false, + error: message, + })); + } +}; + +export const registerStorageTools = (server: McpServer, config: Config): void => { + registerTool(server, { + name: 'get_buckets', + title: 'Get buckets', + description: `Lists buckets or retrieves full details of specific buckets, including descriptions, +lineage references (created/updated by), and links. + +WHEN NOT TO USE: +- Do NOT call with \`bucket_ids=[]\` just to find a bucket by name. Use \`search\` with + item_types=["bucket"] instead. +- Only use \`bucket_ids=[]\` when you need a complete inventory of all buckets in the project. + +EXAMPLES: +- \`bucket_ids=[]\` → summaries of all buckets in the project +- \`bucket_ids=["id1", ...]\` → full details of the buckets with the specified IDs`, + annotations: { readOnlyHint: true }, + inputSchema: { + bucket_ids: z.array(z.string()).default([]).describe('Filter by specific bucket IDs.'), + }, + handler: async ({ bucket_ids }) => { + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + const raw = clients.rawStorage; + const branchId = clients.branchId; // 'default' on production + + const bucketDetailLink = (id: string, name: string): Link => + linksManager.getBucketDetailLink(id, name); + + const buckets: Bucket[] = []; + const missingIds: string[] = []; + let bucketCounts: { + total_buckets: number; + input_buckets: number; + output_buckets: number; + } | null = null; + + if (bucket_ids.length > 0) { + const results = await Promise.all( + bucket_ids.map(async (bucketId): Promise => { + const prodRaw = await safeBucketDetail(raw, bucketId, branchId); + if (!prodRaw) return bucketId; + const bucket = withBucketLineage(validateBucket(prodRaw), prodRaw); + // production path: only prod buckets (no branch metadata) are surfaced. + if (bucket.branch_id) return bucketId; + return { + ...bucket, + links: [bucketDetailLink(bucket.id, bucket.name || bucket.id)], + }; + }), + ); + for (const r of results) { + if (typeof r === 'string') missingIds.push(r); + else buckets.push(r); + } + } else { + const rawList = await raw.get(`branch/${branchId}/buckets`, { + params: { include: 'metadata,linkedBuckets' }, + }); + for (const item of rawList) { + const bucket = validateBucket(item); + if (bucket.branch_id) continue; // skip other-branch buckets (production path) + buckets.push({ + ...bucket, + links: [bucketDetailLink(bucket.id, bucket.name || bucket.id)], + }); + } + const total = buckets.length; + const input = buckets.filter((b) => b.stage === 'in').length; + bucketCounts = { + total_buckets: total, + input_buckets: input, + output_buckets: total - input, + }; + } + + // pack_links: hoist per-bucket links to the output level, deduped + sorted. + const allLinks: Link[] = [linksManager.getBucketDashboardLink()]; + for (const b of buckets) { + if (b.links) allLinks.push(...b.links); + } + const packedBuckets = buckets.map((b) => serializeBucket({ ...b, links: null })); + + return { + buckets: packedBuckets, + links: sortLinks(dedupeLinks(allLinks)), + buckets_not_found: missingIds.length ? missingIds : null, + bucket_counts: bucketCounts, + }; + }, + }); + + registerTool(server, { + name: 'get_tables', + title: 'Get tables', + description: `Lists tables in buckets or retrieves full details of specific tables, including fully qualified database name, +column definitions, lineage references (created/updated by) and links. + +WHEN NOT TO USE: +- Do NOT list tables across buckets just to find a table by name. Use \`search\` with + item_types=["table"] instead — it also matches column names and descriptions. +- Only use \`bucket_ids\` listing when you need all tables in specific known buckets. + +RETURNS: +- With \`bucket_ids\`: Summaries of tables (ID, name, description, primary key). +- With \`table_ids\`: Full details including columns, data types, and fully qualified database names. +- With \`table_ids\` and \`include_usage\`: Full details plus components / transformations that use the tables + in their input / output mappings. Use only when explicitly needed or evident from context; usage calculation + might be demanding in big projects. + +COLUMN DATA TYPES: +- database_native_type: The actual type in the storage backend (Snowflake, BigQuery, etc.) + with precision, scale, and other implementation details +- keboola_base_type: Standardized type indicating the semantic data type. May not always be + available. When present, it reveals the actual type of data stored in the column - for example, + a column with database_native_type VARCHAR might have keboola_base_type INTEGER, indicating + it stores integer values despite being stored as text in the backend. + +QUERYABILITY RULE: +- A table is directly queryable via query_data tool only if fullyQualifiedName is present and non-null + in the response. +- If fullyQualifiedName is absent or null (e.g. for linked/alias tables from other projects), + the table cannot be queried via SQL from this workspace. +- Do not attempt to construct or guess the FQN — it will not work. In that case, + inform the user of the limitation immediately. + +EXAMPLES: +- \`bucket_ids=["id1", ...]\` → summary info of the tables in the buckets with the specified IDs +- \`table_ids=["id1", ...]\` → detailed info of the tables specified by their IDs +- \`bucket_ids=[]\` and \`table_ids=[]\` → empty list; you have to specify at least one filter`, + annotations: { readOnlyHint: true }, + inputSchema: { + bucket_ids: z.array(z.string()).default([]).describe('Filter by specific bucket IDs.'), + table_ids: z.array(z.string()).default([]).describe('Filter by specific table IDs.'), + include_usage: z + .boolean() + .default(false) + .describe('Show components / transformations where each table is used.'), + }, + handler: async ({ bucket_ids, table_ids, include_usage }) => { + const clients = createKeboolaClients(config); + const linksManager = await createLinksManager(config, clients); + const raw = clients.rawStorage; + const branchId = clients.branchId; + + const tablesById = new Map(); + const missingIds: string[] = []; + + if (bucket_ids.length > 0) { + for (const t of await listTables(raw, branchId, bucket_ids, linksManager)) { + tablesById.set(t.id, t); + } + } + + if (table_ids.length > 0) { + // Resolve the SQL dialect once so detail FQNs / native-type defaults are + // backend-correct (Snowflake double-quote 3-part vs BigQuery backtick 2-part). + const dialect = await resolveDialect(raw); + const results = await Promise.all( + table_ids.map(async (tableId): Promise
=> { + const t = await getTableDetail(raw, branchId, tableId, linksManager, dialect); + return t ?? tableId; + }), + ); + for (const r of results) { + if (typeof r === 'string') missingIds.push(r); + else tablesById.set(r.id, r); + } + + if (include_usage) { + // find_id_usage depends on the search subsystem (tools/search.py), which is + // not yet ported to TypeScript. Initialize empty used_by for parity of shape. + for (const t of tablesById.values()) { + if (t.isDetail) t.used_by = []; + } + logger.warn( + 'get_tables: include_usage requested but the search subsystem is not yet ported; returning empty usage.', + ); + } + } + + const tables = [...tablesById.values()]; + const allLinks: Link[] = [linksManager.getBucketDashboardLink()]; + for (const t of tables) { + if (t.links) allLinks.push(...t.links); + } + const packed = tables.map((t) => serializeTable({ ...t, links: null })); + + return { + tables: packed, + links: sortLinks(dedupeLinks(allLinks)), + tables_not_found: missingIds.length ? missingIds : null, + }; + }, + }); + + registerTool(server, { + name: 'update_descriptions', + title: 'Update descriptions', + description: 'Updates the description for Keboola storage items (buckets, tables, or columns).', + inputSchema: { + updates: z + .array( + z.object({ + item_id: z + .string() + .describe( + 'Storage item: "bucket_id", "bucket_id.table_id", or "bucket_id.table_id.column_name".', + ), + description: z.string().describe('New description to set.'), + }), + ) + .describe('List of description updates to apply.'), + }, + handler: async ({ updates }) => { + const { rawStorage } = createKeboolaClients(config); + const results: UpdateItemResult[] = []; + + // Group valid updates by type; record invalid item_ids up front. + const bucketUpdates: Record = {}; + const tableUpdates: Record = {}; + const columnUpdatesByTable: Record> = {}; + + for (const update of updates) { + let parsed: ParsedItemId; + try { + parsed = parseItemId(update.item_id); + } catch (error) { + results.push({ + item_id: update.item_id, + success: false, + error: `Invalid item_id format: ${error instanceof Error ? error.message : String(error)}`, + }); + continue; + } + if (parsed.itemType === 'bucket') { + bucketUpdates[parsed.bucketId!] = update.description; + } else if (parsed.itemType === 'table') { + tableUpdates[parsed.tableId!] = update.description; + } else { + (columnUpdatesByTable[parsed.tableId!] ??= {})[parsed.columnName!] = update.description; + } + } + + for (const [bucketId, description] of Object.entries(bucketUpdates)) { + results.push(await updateBucketDescription(rawStorage, bucketId, description)); + } + for (const [tableId, description] of Object.entries(tableUpdates)) { + results.push(await updateTableDescription(rawStorage, tableId, description)); + } + for (const [tableId, columnUpdates] of Object.entries(columnUpdatesByTable)) { + results.push(...(await updateColumnDescriptions(rawStorage, tableId, columnUpdates))); + } + + const successful = results.filter((result) => result.success).length; + return { + results, + total_processed: results.length, + successful, + failed: results.length - successful, + }; + }, + }); +}; diff --git a/src/tools/storage/usage.ts b/src/tools/storage/usage.ts new file mode 100644 index 000000000..ae8cf1cf6 --- /dev/null +++ b/src/tools/storage/usage.ts @@ -0,0 +1,81 @@ +// Ported from tools/storage/usage.py (get_created_by / get_last_updated_by). +// +// find_id_usage depends on the search subsystem (tools/search.py), which is not yet +// ported to TypeScript. include_usage therefore returns empty usage; see tools.ts. + +import { getMetadataProperty, parseIsoTimestamp, type RawObj } from './model'; + +const CREATED_BY_COMPONENT_ID = 'KBC.createdBy.component.id'; +const CREATED_BY_CONFIGURATION_ID = 'KBC.createdBy.configuration.id'; +const CREATED_BY_CONFIGURATION_ROW_ID = 'KBC.createdBy.configurationRow.id'; +const UPDATED_BY_COMPONENT_ID = 'KBC.lastUpdatedBy.component.id'; +const UPDATED_BY_CONFIGURATION_ID = 'KBC.lastUpdatedBy.configuration.id'; +const UPDATED_BY_CONFIGURATION_ROW_ID = 'KBC.lastUpdatedBy.configurationRow.id'; + +export type ComponentUsageReference = { + component_id: string; + configuration_id: string; + configuration_row_id: string | null; + configuration_name: string | null; + used_in: string | null; + timestamp: string | null; +}; + +const latestMetadataTimestamp = (metadata: RawObj[], keys: string[]): string | null => { + let latest: number | null = null; + let latestRaw: string | null = null; + for (const item of metadata) { + if (!keys.includes(item.key as string)) continue; + const rawTs = item.timestamp; + if (typeof rawTs !== 'string') continue; + let parsed: number; + try { + parsed = parseIsoTimestamp(rawTs); + } catch { + continue; + } + if (latest === null || parsed > latest) { + latest = parsed; + latestRaw = rawTs; + } + } + return latestRaw; +}; + +const lineageReference = ( + metadata: unknown, + componentKey: string, + configKey: string, + rowKey: string, +): ComponentUsageReference | null => { + if (!Array.isArray(metadata)) return null; + const items = metadata as RawObj[]; + const componentId = getMetadataProperty(items, componentKey); + const configurationId = getMetadataProperty(items, configKey); + const rowId = getMetadataProperty(items, rowKey); + if (componentId === null || configurationId === null) return null; + return { + component_id: String(componentId), + configuration_id: String(configurationId), + configuration_row_id: rowId ? String(rowId) : null, + configuration_name: null, + used_in: null, + timestamp: latestMetadataTimestamp(items, [componentKey, configKey, rowKey]), + }; +}; + +export const getCreatedBy = (metadata: unknown): ComponentUsageReference | null => + lineageReference( + metadata, + CREATED_BY_COMPONENT_ID, + CREATED_BY_CONFIGURATION_ID, + CREATED_BY_CONFIGURATION_ROW_ID, + ); + +export const getLastUpdatedBy = (metadata: unknown): ComponentUsageReference | null => + lineageReference( + metadata, + UPDATED_BY_COMPONENT_ID, + UPDATED_BY_CONFIGURATION_ID, + UPDATED_BY_CONFIGURATION_ROW_ID, + ); diff --git a/src/tools/validation/index.ts b/src/tools/validation/index.ts new file mode 100644 index 000000000..eaa2e8b7e --- /dev/null +++ b/src/tools/validation/index.ts @@ -0,0 +1,29 @@ +/** + * Public entry point for the tolerant JSON-schema validator module. + * + * Preserves the import path `@/tools/validation` for downstream consumers: + * - `src/tools/components/tools.ts` (the validate* entry points) + * - `src/tools/components/utils.ts` (ComponentForValidation, JsonDict types) + * - `__tests__/tools.components.test.ts` (validate* entry points + `__testing`) + * + * The split modules are: + * - `types.ts` — shared types + RecoverableValidationError / SchemaError + * - `sanitize.ts` — the tolerant schema sanitizer + * - `validate.ts` — the draft-07 subset validator + * - `model.ts` — the public validate* entry points + */ +import { sanitizeSchema } from './sanitize'; +import { validateJsonAgainstSchema } from './validate'; + +export type { ComponentForValidation, JsonDict, JsonSchema, ValidationContext } from './types'; +export { RecoverableValidationError } from './types'; +export { + validateProcessorsConfiguration, + validateRootParametersConfiguration, + validateRootStorageConfiguration, + validateRowParametersConfiguration, + validateRowStorageConfiguration, +} from './model'; + +// Exported for unit testing. +export const __testing = { sanitizeSchema, validateJsonAgainstSchema }; diff --git a/src/tools/validation/model.ts b/src/tools/validation/model.ts new file mode 100644 index 000000000..3f52840c9 --- /dev/null +++ b/src/tools/validation/model.ts @@ -0,0 +1,242 @@ +// --------------------------------------------------------------------------- +// Public validators (ported from validation.py public functions). +// --------------------------------------------------------------------------- + +import storageSchema from '../storage-schema.json' with { type: 'json' }; + +import { logger } from '@/logger'; +import { + type ComponentForValidation, + type JsonDict, + type JsonSchema, + type ValidationContext, +} from './types'; +import { validateJsonAgainstSchema } from './validate'; + +const SNOWFLAKE_TRANSFORMATION_ID = 'keboola.snowflake-transformation'; +const BIGQUERY_TRANSFORMATION_ID = 'keboola.google-bigquery-transformation'; + +const STORAGE_VALIDATION_INITIAL_MESSAGE = + 'The provided storage configuration input does not follow the storage schema.\n'; +const ROOT_PARAMETERS_INITIAL = (componentId: string) => + `The provided Root parameters configuration input does not follow the Root parameter json schema for component ` + + `id: ${componentId}.\n`; +const ROW_PARAMETERS_INITIAL = (componentId: string) => + `The provided Row parameters configuration input does not follow the Row parameter json schema for component ` + + `id: ${componentId}.\n`; + +const validateStorageConfigurationAgainstSchema = ( + storage: JsonDict, + initialMessage?: string, + validationContext?: ValidationContext, +): JsonDict => { + validateJsonAgainstSchema({ + jsonData: storage, + schema: storageSchema as JsonSchema, + initialMessage, + validationContext, + }); + return storage; +}; + +const validateStorageConfiguration = ( + storage: JsonDict | null | undefined, + component: ComponentForValidation, + initialMessage: string | undefined, + opts: { + isRowStorage: boolean; + configurationId?: string | null; + configurationRowId?: string | null; + }, +): JsonDict => { + // Normalize to {'storage': storage | {} } — the agent may pass {storage: …} or just the inner object. + let storageCfg: JsonDict | null; + if (storage) { + const inner = (storage as JsonDict).storage; + storageCfg = (inner !== undefined ? inner : storage) as JsonDict | null; + } else { + storageCfg = {}; + } + + if (storageCfg === null || storageCfg === undefined) { + logger.warn( + `No "storage" configuration provided for component ${component.component_id} of type ${component.component_type}.`, + ); + storageCfg = {}; + } + + if ( + component.component_id === SNOWFLAKE_TRANSFORMATION_ID || + component.component_id === BIGQUERY_TRANSFORMATION_ID + ) { + if (!storageCfg.input && !storageCfg.output) { + throw new Error( + `The "storage" must contain either "input" or "output" mappings in the configuration of the SQL ` + + `transformation "${component.component_id}".`, + ); + } + } + + if (component.component_type === 'writer' && component.capabilities.is_row_based) { + if (!opts.isRowStorage && Object.keys(storageCfg).length > 0) { + throw new Error( + `The "storage" must be empty for root configuration of the writer component ` + + `"${component.component_id}" since it is row-based. In this case, storage should only be defined ` + + 'in its outgoing row configurations.', + ); + } else if (opts.isRowStorage && !storageCfg.input) { + throw new Error( + `The "storage" must contain "input" mappings for the row configuration of the writer component ` + + `"${component.component_id}".`, + ); + } + } + + if (component.component_type === 'writer' && !component.capabilities.is_row_based) { + if (opts.isRowStorage) { + logger.warn( + `Validating "storage" for row configuration of non-row-based writer ${component.component_id} is not ` + + 'semantically correct. Possible cause: agent error or wrong component flag. Proceeding with validation.', + ); + } + if (!storageCfg.input) { + throw new Error( + `The "storage" must contain "input" mappings for the root configuration of the writer component ` + + `"${component.component_id}".`, + ); + } + } + + const fullInitial = (initialMessage ?? '') + '\n' + STORAGE_VALIDATION_INITIAL_MESSAGE; + const validationContext: ValidationContext = { + component_id: component.component_id, + configuration_id: opts.configurationId, + configuration_row_id: opts.configurationRowId, + scope: 'storage', + }; + const normalized = validateStorageConfigurationAgainstSchema( + { storage: storageCfg }, + fullInitial, + validationContext, + ); + return (normalized.storage ?? {}) as JsonDict; +}; + +export const validateRootStorageConfiguration = ( + storage: JsonDict | null | undefined, + component: ComponentForValidation, + initialMessage?: string, + configurationId?: string | null, +): JsonDict => + validateStorageConfiguration(storage, component, initialMessage, { + isRowStorage: false, + configurationId, + }); + +export const validateRowStorageConfiguration = ( + storage: JsonDict | null | undefined, + component: ComponentForValidation, + initialMessage?: string, + configurationId?: string | null, + configurationRowId?: string | null, +): JsonDict => + validateStorageConfiguration(storage, component, initialMessage, { + isRowStorage: true, + configurationId, + configurationRowId, + }); + +const validateParametersConfiguration = ( + parameters: JsonDict, + schema: JsonSchema | null | undefined, + componentId: string, + initialMessage: string | undefined, + configurationId?: string | null, + configurationRowId?: string | null, +): JsonDict => { + // Agent may pass {parameters: …} or just the inner object. + const inner = (parameters as JsonDict).parameters; + const expected = (inner !== undefined ? inner : parameters) as JsonDict; + + if (!schema || Object.keys(schema).length === 0) { + logger.warn(`No schema provided for component ${componentId}, skipping validation.`); + return expected; + } + + validateJsonAgainstSchema({ + jsonData: expected, + schema, + initialMessage, + validationContext: { + component_id: componentId, + configuration_id: configurationId, + configuration_row_id: configurationRowId, + scope: 'parameters', + }, + sanitize: true, + }); + return expected; +}; + +export const validateRootParametersConfiguration = ( + parameters: JsonDict, + component: ComponentForValidation, + initialMessage?: string, + configurationId?: string | null, +): JsonDict => + validateParametersConfiguration( + parameters, + component.configuration_schema, + component.component_id, + (initialMessage ?? '') + '\n' + ROOT_PARAMETERS_INITIAL(component.component_id), + configurationId, + ); + +export const validateRowParametersConfiguration = ( + parameters: JsonDict, + component: ComponentForValidation, + initialMessage?: string, + configurationId?: string | null, + configurationRowId?: string | null, +): JsonDict => + validateParametersConfiguration( + parameters, + component.configuration_row_schema, + component.component_id, + (initialMessage ?? '') + '\n' + ROW_PARAMETERS_INITIAL(component.component_id), + configurationId, + configurationRowId, + ); + +/** + * Validates a list of processors against their component schemas. Skips processors + * with no schema or whose schema is from the template (the `print_hello` marker). + * Port of `validate_processors_configuration`. + */ +export const validateProcessorsConfiguration = async ( + fetchComponent: (componentId: string) => Promise, + processors: JsonDict[], + initialMessage?: string, +): Promise => { + for (const processor of processors) { + const definition = (processor.definition as JsonDict) ?? {}; + const processorId = definition.component as string; + const processorInfo = await fetchComponent(processorId); + + const schema = processorInfo.configuration_schema; + if (!schema) continue; + const required = Array.isArray((schema as JsonDict).required) + ? ((schema as JsonDict).required as string[]) + : []; + if (required.includes('print_hello')) continue; + + validateJsonAgainstSchema({ + jsonData: processor.parameters, + schema, + initialMessage: `${initialMessage}\nThe configuration of "${processorId}" processor is not valid.`, + validationContext: { component_id: processorId }, + sanitize: true, + }); + } + return processors; +}; diff --git a/src/tools/validation/sanitize.ts b/src/tools/validation/sanitize.ts new file mode 100644 index 000000000..2bb7fa788 --- /dev/null +++ b/src/tools/validation/sanitize.ts @@ -0,0 +1,105 @@ +// --------------------------------------------------------------------------- +// Schema sanitization — port of KeboolaParametersValidator.sanitize_schema. +// --------------------------------------------------------------------------- + +import { isObject, type JsonDict, type JsonSchema, SchemaError } from './types'; + +/** + * Normalizes a JSON schema *in place* and returns it. Mirrors the Python + * `_sanitize_node`: strips empty `enum`, converts boolean-ish `required` flags into + * the list form (propagating up to the parent), and turns `properties: []` into `{}`. + * Returns `[schema, isCurrentRequired]`. + */ +const sanitizeNode = (schema: unknown): [unknown, boolean | null] => { + if (!isObject(schema)) { + return [schema, false]; + } + + if ('enum' in schema && Array.isArray(schema.enum) && schema.enum.length === 0) { + delete schema.enum; + } + + let isCurrentRequired: boolean | null = null; + let required = schema.required; + if (!Array.isArray(required)) { + if (required !== undefined) { + isCurrentRequired = String(required).toLowerCase() === 'true'; + } + required = []; + } + const requiredList = required as string[]; + + let properties = schema.properties; + if (properties !== undefined && properties !== null) { + if (Array.isArray(properties) && properties.length === 0) { + properties = {}; + } else if (!isObject(properties)) { + throw new SchemaError(`properties must be a dictionary, got ${typeof properties}`); + } + const props = properties as JsonDict; + for (const propertyName of Object.keys(props)) { + const [sanitized, isChildRequired] = sanitizeNode(props[propertyName]); + props[propertyName] = sanitized; + if (isChildRequired === true && !requiredList.includes(propertyName)) { + requiredList.push(propertyName); + } else if (isChildRequired === false && requiredList.includes(propertyName)) { + requiredList.splice(requiredList.indexOf(propertyName), 1); + } + } + schema.properties = props; + } + + if (requiredList.length > 0) { + schema.required = [...requiredList]; + } else { + delete schema.required; + } + + if ('items' in schema) { + const items = schema.items; + if (isObject(items)) { + schema.items = sanitizeNode(items)[0]; + } else if (Array.isArray(items)) { + schema.items = items.map((item) => (isObject(item) ? sanitizeNode(item)[0] : item)); + } + } + + for (const keyword of ['allOf', 'anyOf', 'oneOf'] as const) { + if (keyword in schema && Array.isArray(schema[keyword])) { + schema[keyword] = (schema[keyword] as unknown[]).map((s) => + isObject(s) ? sanitizeNode(s)[0] : s, + ); + } + } + + for (const keyword of ['not', 'if', 'then', 'else'] as const) { + if (keyword in schema && isObject(schema[keyword])) { + schema[keyword] = sanitizeNode(schema[keyword])[0]; + } + } + + if ('additionalProperties' in schema && isObject(schema.additionalProperties)) { + schema.additionalProperties = sanitizeNode(schema.additionalProperties)[0]; + } + + if ('patternProperties' in schema && isObject(schema.patternProperties)) { + const pp = schema.patternProperties as JsonDict; + for (const pattern of Object.keys(pp)) { + if (isObject(pp[pattern])) pp[pattern] = sanitizeNode(pp[pattern])[0]; + } + } + + for (const keyword of ['definitions', '$defs'] as const) { + if (keyword in schema && isObject(schema[keyword])) { + const defs = schema[keyword] as JsonDict; + for (const name of Object.keys(defs)) { + if (isObject(defs[name])) defs[name] = sanitizeNode(defs[name])[0]; + } + } + } + + return [schema, isCurrentRequired]; +}; + +export const sanitizeSchema = (schema: JsonSchema): JsonSchema => + sanitizeNode(structuredClone(schema))[0] as JsonSchema; diff --git a/src/tools/validation/types.ts b/src/tools/validation/types.ts new file mode 100644 index 000000000..bee66b1f0 --- /dev/null +++ b/src/tools/validation/types.ts @@ -0,0 +1,112 @@ +/** + * Shared types and helpers for the tolerant JSON-schema validator. + * + * Ported from `tools/validation.py`. There is no `ajv` (or any JSON-schema library) + * in this project's dependencies, so the validator is hand-written. It is + * deliberately *tolerant* of the schema inconsistencies the Keboola Developer Portal + * UI schemas exhibit (boolean `required`, empty `enum`, `properties: []`, the UI-only + * `button` type, …) and — matching the Python behaviour — when the schema itself is + * invalid or absent, validation is skipped (logged + continue) so a broken upstream + * schema never blocks a write. + */ + +export type JsonDict = Record; +export type JsonSchema = Record; + +/** Minimal Component shape the validators need. */ +export type ComponentForValidation = { + component_id: string; + component_type: string; + capabilities: { is_row_based: boolean }; + configuration_schema?: JsonSchema | null; + configuration_row_schema?: JsonSchema | null; +}; + +export type ValidationContext = { + component_id: string; + configuration_id?: string | null; + configuration_row_id?: string | null; + scope?: 'parameters' | 'storage' | string | null; +}; + +export const isObject = (value: unknown): value is JsonDict => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const contextToString = (ctx: ValidationContext): string => { + let s = `component_id=${ctx.component_id}`; + if (ctx.configuration_id) s += `, configuration_id=${ctx.configuration_id}`; + if (ctx.configuration_row_id) s += `, configuration_row_id=${ctx.configuration_row_id}`; + if (ctx.scope) s += `, scope=${ctx.scope}`; + return s; +}; + +/** + * Raised when an instance is invalid under a schema. Carries a recoverable, agent- + * friendly message — the port of Python's RecoverableValidationError. + */ +export class RecoverableValidationError extends Error { + readonly validator: string; + readonly validatorValue: unknown; + readonly schemaPath: (string | number)[]; + readonly instancePath: (string | number)[]; + readonly instance: unknown; + readonly baseMessage: string; + initialMessage?: string; + validationContext?: ValidationContext; + + constructor(opts: { + message: string; + validator: string; + validatorValue: unknown; + schemaPath: (string | number)[]; + instancePath: (string | number)[]; + instance: unknown; + initialMessage?: string; + validationContext?: ValidationContext; + }) { + super(opts.message); + this.name = 'RecoverableValidationError'; + this.baseMessage = opts.message; + this.validator = opts.validator; + this.validatorValue = opts.validatorValue; + this.schemaPath = opts.schemaPath; + this.instancePath = opts.instancePath; + this.instance = opts.instance; + this.initialMessage = opts.initialMessage; + this.validationContext = opts.validationContext; + this.message = this.format(); + } + + private format(): string { + let s = `${this.baseMessage}\n`; + if (this.validator && this.validatorValue !== undefined && this.validatorValue !== null) { + const schemaPath = this.schemaPath.map((p) => `[${JSON.stringify(p)}]`).join(''); + s += `Failed validating ${JSON.stringify(this.validator)} in schema${schemaPath}:\n`; + s += ` ${JSON.stringify({ [this.validator]: this.validatorValue }, null, 2)}\n`; + } + if (this.instancePath.length > 0) { + const instancePath = this.instancePath.map((p) => `[${JSON.stringify(p)}]`).join(''); + s += `On instance${instancePath}:\n`; + s += ` ${JSON.stringify(this.instance, null, 4)}\n`; + } + if (this.initialMessage) s += `${this.initialMessage}\n`; + if (this.validationContext) { + s += `Validation component context: ${contextToString(this.validationContext)}\n`; + } + if ( + this.validator === 'required' && + Array.isArray(this.validatorValue) && + this.validationContext?.scope === 'parameters' + ) { + const requiredFields = (this.validatorValue as string[]).map((f) => `\`${f}\``).join(', '); + s += + `HINT: Ensure ALL of the following required fields are present in \`parameters\`: ${requiredFields}. ` + + 'Call `get_components` to retrieve the full schema and `get_config_examples` for real-world examples.' + + '\n'; + } + return s.replace(/\s+$/, ''); + } +} + +/** A schema that is structurally invalid (port of jsonschema.SchemaError). */ +export class SchemaError extends Error {} diff --git a/src/tools/validation/validate.ts b/src/tools/validation/validate.ts new file mode 100644 index 000000000..1839a1c59 --- /dev/null +++ b/src/tools/validation/validate.ts @@ -0,0 +1,402 @@ +// --------------------------------------------------------------------------- +// Core validator — a tolerant subset of JSON Schema draft-07. +// --------------------------------------------------------------------------- + +import { logger } from '@/logger'; +import { sanitizeSchema } from './sanitize'; +import { + isObject, + type JsonSchema, + RecoverableValidationError, + type ValidationContext, +} from './types'; + +type Fail = ( + validator: string, + validatorValue: unknown, + message: string, + schemaPath: (string | number)[], + instancePath: (string | number)[], + instance: unknown, +) => never; + +const jsTypeMatches = (type: string, instance: unknown): boolean => { + switch (type) { + case 'object': + return isObject(instance); + case 'array': + return Array.isArray(instance); + case 'string': + return typeof instance === 'string'; + case 'boolean': + return typeof instance === 'boolean'; + case 'null': + return instance === null; + case 'number': + return typeof instance === 'number'; + case 'integer': + return typeof instance === 'number' && Number.isInteger(instance); + case 'button': + // UI-only construct accepted as valid (port of check_button_type). + return isObject(instance) && instance.type === 'button'; + default: + // Unknown type keyword: be tolerant and accept. + return true; + } +}; + +const deepEqual = (a: unknown, b: unknown): boolean => { + if (a === b) return true; + if (typeof a !== typeof b) return false; + if (Array.isArray(a) && Array.isArray(b)) { + return a.length === b.length && a.every((x, i) => deepEqual(x, b[i])); + } + if (isObject(a) && isObject(b)) { + const ka = Object.keys(a); + const kb = Object.keys(b); + return ka.length === kb.length && ka.every((k) => deepEqual(a[k], b[k])); + } + return false; +}; + +/** Returns true if `instance` is valid under `schema` (no error thrown). */ +const isValid = (schema: unknown, instance: unknown): boolean => { + try { + validateNode(schema, instance, [], [], () => { + throw new Error('invalid'); + }); + return true; + } catch { + return false; + } +}; + +const validateNode = ( + schema: unknown, + instance: unknown, + schemaPath: (string | number)[], + instancePath: (string | number)[], + fail: Fail, +): void => { + if (typeof schema === 'boolean') { + if (!schema) + fail('schema', false, 'False schema rejects all values', schemaPath, instancePath, instance); + return; + } + if (!isObject(schema)) return; + + // type + if ('type' in schema) { + const types = Array.isArray(schema.type) ? (schema.type as string[]) : [schema.type as string]; + if (!types.some((t) => jsTypeMatches(t, instance))) { + fail( + 'type', + schema.type, + `${JSON.stringify(instance)} is not of type ${types.map((t) => JSON.stringify(t)).join(', ')}`, + [...schemaPath, 'type'], + instancePath, + instance, + ); + } + } + + // enum + if ('enum' in schema && Array.isArray(schema.enum)) { + if (!schema.enum.some((v) => deepEqual(v, instance))) { + fail( + 'enum', + schema.enum, + `${JSON.stringify(instance)} is not one of ${JSON.stringify(schema.enum)}`, + [...schemaPath, 'enum'], + instancePath, + instance, + ); + } + } + + // const + if ('const' in schema && !deepEqual(schema.const, instance)) { + fail( + 'const', + schema.const, + `${JSON.stringify(instance)} was expected`, + [...schemaPath, 'const'], + instancePath, + instance, + ); + } + + // numeric constraints + if (typeof instance === 'number') { + if (typeof schema.minimum === 'number' && instance < schema.minimum) { + fail( + 'minimum', + schema.minimum, + `${instance} is less than the minimum of ${schema.minimum}`, + [...schemaPath, 'minimum'], + instancePath, + instance, + ); + } + if (typeof schema.maximum === 'number' && instance > schema.maximum) { + fail( + 'maximum', + schema.maximum, + `${instance} is greater than the maximum of ${schema.maximum}`, + [...schemaPath, 'maximum'], + instancePath, + instance, + ); + } + } + + // string constraints + if (typeof instance === 'string') { + if (typeof schema.minLength === 'number' && instance.length < schema.minLength) { + fail( + 'minLength', + schema.minLength, + `${JSON.stringify(instance)} is too short`, + [...schemaPath, 'minLength'], + instancePath, + instance, + ); + } + if (typeof schema.maxLength === 'number' && instance.length > schema.maxLength) { + fail( + 'maxLength', + schema.maxLength, + `${JSON.stringify(instance)} is too long`, + [...schemaPath, 'maxLength'], + instancePath, + instance, + ); + } + } + + // array constraints + if (Array.isArray(instance)) { + if (typeof schema.minItems === 'number' && instance.length < schema.minItems) { + fail( + 'minItems', + schema.minItems, + `${JSON.stringify(instance)} is too short`, + [...schemaPath, 'minItems'], + instancePath, + instance, + ); + } + if (typeof schema.maxItems === 'number' && instance.length > schema.maxItems) { + fail( + 'maxItems', + schema.maxItems, + `${JSON.stringify(instance)} is too long`, + [...schemaPath, 'maxItems'], + instancePath, + instance, + ); + } + const items = schema.items; + if (isObject(items) || typeof items === 'boolean') { + instance.forEach((item, i) => + validateNode(items, item, [...schemaPath, 'items'], [...instancePath, i], fail), + ); + } else if (Array.isArray(items)) { + instance.forEach((item, i) => { + if (i < items.length) { + validateNode(items[i], item, [...schemaPath, 'items', i], [...instancePath, i], fail); + } + }); + } + } + + // object constraints + if (isObject(instance)) { + if (Array.isArray(schema.required)) { + const missing = (schema.required as string[]).filter((key) => !(key in instance)); + if (missing.length > 0) { + fail( + 'required', + schema.required, + `${JSON.stringify(missing[0])} is a required property`, + [...schemaPath, 'required'], + instancePath, + instance, + ); + } + } + + const properties = isObject(schema.properties) ? schema.properties : {}; + for (const key of Object.keys(properties)) { + if (key in instance) { + validateNode( + properties[key], + instance[key], + [...schemaPath, 'properties', key], + [...instancePath, key], + fail, + ); + } + } + + const patternProperties = isObject(schema.patternProperties) ? schema.patternProperties : {}; + const patternKeys = Object.keys(patternProperties); + for (const key of Object.keys(instance)) { + for (const pattern of patternKeys) { + let re: RegExp | null = null; + try { + re = new RegExp(pattern); + } catch { + re = null; + } + if (re && re.test(key)) { + validateNode( + patternProperties[pattern], + instance[key], + [...schemaPath, 'patternProperties', pattern], + [...instancePath, key], + fail, + ); + } + } + } + + if (isObject(schema.additionalProperties)) { + const declared = new Set(Object.keys(properties)); + for (const key of Object.keys(instance)) { + if (declared.has(key)) continue; + if (patternKeys.some((p) => safeTest(p, key))) continue; + validateNode( + schema.additionalProperties, + instance[key], + [...schemaPath, 'additionalProperties'], + [...instancePath, key], + fail, + ); + } + } else if (schema.additionalProperties === false) { + const declared = new Set(Object.keys(properties)); + for (const key of Object.keys(instance)) { + if (declared.has(key)) continue; + if (patternKeys.some((p) => safeTest(p, key))) continue; + fail( + 'additionalProperties', + false, + `Additional properties are not allowed (${JSON.stringify(key)} was unexpected)`, + [...schemaPath, 'additionalProperties'], + instancePath, + instance, + ); + } + } + } + + // allOf + if (Array.isArray(schema.allOf)) { + schema.allOf.forEach((sub, i) => + validateNode(sub, instance, [...schemaPath, 'allOf', i], instancePath, fail), + ); + } + + // anyOf + if (Array.isArray(schema.anyOf)) { + if (!schema.anyOf.some((sub) => isValid(sub, instance))) { + fail( + 'anyOf', + schema.anyOf, + `${JSON.stringify(instance)} is not valid under any of the given schemas`, + [...schemaPath, 'anyOf'], + instancePath, + instance, + ); + } + } + + // oneOf + if (Array.isArray(schema.oneOf)) { + const matches = schema.oneOf.filter((sub) => isValid(sub, instance)).length; + if (matches !== 1) { + fail( + 'oneOf', + schema.oneOf, + `${JSON.stringify(instance)} is valid under ${matches} of the given schemas (expected exactly 1)`, + [...schemaPath, 'oneOf'], + instancePath, + instance, + ); + } + } + + // not + if (schema.not !== undefined && isValid(schema.not, instance)) { + fail( + 'not', + schema.not, + `${JSON.stringify(instance)} is not allowed`, + [...schemaPath, 'not'], + instancePath, + instance, + ); + } + + // if / then / else + if (schema.if !== undefined) { + if (isValid(schema.if, instance)) { + if (schema.then !== undefined) { + validateNode(schema.then, instance, [...schemaPath, 'then'], instancePath, fail); + } + } else if (schema.else !== undefined) { + validateNode(schema.else, instance, [...schemaPath, 'else'], instancePath, fail); + } + } +}; + +const safeTest = (pattern: string, value: string): boolean => { + try { + return new RegExp(pattern).test(value); + } catch { + return false; + } +}; + +/** + * Validates `jsonData` against `schema`. On a validation failure, throws a + * RecoverableValidationError. On a structurally invalid schema, logs and returns + * (continue as if valid) — the parity behaviour of `_validate_json_against_schema`. + */ +export const validateJsonAgainstSchema = (opts: { + jsonData: unknown; + schema: JsonSchema; + initialMessage?: string; + validationContext?: ValidationContext; + sanitize?: boolean; +}): void => { + let schema = opts.schema; + try { + if (opts.sanitize) schema = sanitizeSchema(opts.schema); + } catch (error) { + logger.warn({ err: error }, 'The validation schema is not valid; skipping validation.'); + return; + } + + const fail: Fail = (validator, validatorValue, message, schemaPath, instancePath, instance) => { + throw new RecoverableValidationError({ + message, + validator, + validatorValue, + schemaPath, + instancePath, + instance, + initialMessage: opts.initialMessage, + validationContext: opts.validationContext, + }); + }; + + try { + validateNode(schema, opts.jsonData, [], [], fail); + } catch (error) { + if (error instanceof RecoverableValidationError) throw error; + // Treat any non-validation error as an invalid schema → skip (continue). + logger.warn({ err: error }, 'The validation schema is not valid; skipping validation.'); + } +}; diff --git a/src/transports/http.ts b/src/transports/http.ts new file mode 100644 index 000000000..1ae111077 --- /dev/null +++ b/src/transports/http.ts @@ -0,0 +1,256 @@ +import { type HttpBindings, serve } from '@hono/node-server'; +import { RESPONSE_ALREADY_SENT } from '@hono/node-server/utils/response'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { Hono } from 'hono'; + +import type { Config } from '@/config'; +import { logger } from '@/logger'; +import { + buildAuthorizationServerMetadata, + buildProtectedResourceMetadata, + InvalidRedirectUriError, + OAuthHttpError, + SimpleOAuthProvider, + validateRedirectUri, +} from '@/oauth'; +import { parsePreviewRequest, PreviewHttpError, runPreviewConfigDiff } from '@/preview'; +import { createServer, SERVER_NAME, SERVER_VERSION } from '@/server'; + +type Bindings = HttpBindings; + +/** The endpoint where the OAuth server redirects back after the user authorizes. */ +const OAUTH_CALLBACK_ENDPOINT = '/oauth/callback'; + +/** + * Builds the per-request Config: the base (env/CLI) config with HTTP headers + * layered on top. `X-*` headers map onto Config fields by name; the OAuth bearer + * token is taken from the standard `Authorization: Bearer ` header. + */ +const configFromHeaders = (base: Config, headers: Headers): Config => { + const map: Record = {}; + headers.forEach((value, key) => { + map[key] = value; + }); + + const auth = headers.get('authorization'); + const bearer = auth?.match(/^Bearer\s+(.+)$/i)?.[1]; + if (bearer) { + map['bearerToken'] = bearer; + } + + return base.replaceBy(map); +}; + +/** + * Whether the OAuth provider can be enabled for this config. OAuth is optional and only + * wired up for the HTTP transport when both the client ID and secret are configured. + */ +const isOAuthConfigured = (config: Config): boolean => + Boolean(config.oauthClientId && config.oauthClientSecret); + +/** + * Registers the OAuth routes on the Hono app, mirroring the Python `SimpleOAuthProvider.get_routes()` + * plus the GitHub-style `GET /oauth/callback` handler. Gated by {@link isOAuthConfigured}. + */ +const registerOAuthRoutes = (app: Hono<{ Bindings: Bindings }>, config: Config): void => { + const mcpServerUrl = config.mcpServerUrl!; + const provider = new SimpleOAuthProvider({ + storageApiUrl: config.storageApiUrl ?? '', + mcpServerUrl, + callbackEndpoint: OAUTH_CALLBACK_ENDPOINT, + clientId: config.oauthClientId!, + clientSecret: config.oauthClientSecret!, + serverUrl: config.oauthServerUrl ?? '', + scope: config.oauthScope ?? '', + jwtSecret: config.jwtSecret, + }); + + // .well-known metadata documents (discovery). + app.get('/.well-known/oauth-authorization-server', (c) => + c.json(buildAuthorizationServerMetadata(mcpServerUrl)), + ); + app.get('/.well-known/oauth-protected-resource', (c) => + c.json(buildProtectedResourceMetadata(mcpServerUrl)), + ); + + // Dynamic Client Registration: a no-op that echoes back the requested registration. + // Nothing is persisted, mirroring the Python provider. + app.post('/register', async (c) => { + const body = (await c.req.json().catch(() => ({}))) as Record; + const clientId = `client_${crypto.randomUUID()}`; + logger.debug(`Client registered: client_id=${clientId}`); + return c.json( + { + ...body, + client_id: clientId, + token_endpoint_auth_method: body['token_endpoint_auth_method'] ?? 'none', + }, + 201, + ); + }); + + // Authorization endpoint: validates the redirect URI, then redirects to the OAuth server. + app.get('/authorize', async (c) => { + const q = c.req.query(); + const redirectUri = q['redirect_uri']; + try { + validateRedirectUri(redirectUri); + } catch (err) { + if (err instanceof InvalidRedirectUriError) { + return c.json({ error: 'invalid_request', error_description: err.message }, 400); + } + throw err; + } + + const scopesParam = q['scope']; + const authUrl = await provider.authorize(q['client_id'] ?? '', { + redirectUri: redirectUri!, + redirectUriProvidedExplicitly: true, + codeChallenge: q['code_challenge'] ?? '', + state: q['state'] ?? null, + scopes: scopesParam ? scopesParam.split(' ') : [], + }); + return c.redirect(authUrl); + }); + + // GitHub-style callback: the OAuth server redirects here; we redirect back to the client. + app.get(OAUTH_CALLBACK_ENDPOINT, async (c) => { + const code = c.req.query('code'); + const state = c.req.query('state'); + if (!code || !state) { + return c.json({ error: 'invalid_request', error_description: 'Missing code or state' }, 400); + } + try { + const redirect = await provider.handleOAuthCallback(code, state); + return c.redirect(redirect); + } catch (err) { + if (err instanceof OAuthHttpError) { + return c.json( + { error: 'invalid_request', error_description: err.message }, + err.status as 400, + ); + } + throw err; + } + }); + + // Token endpoint: handles the authorization_code and refresh_token grants. + app.post('/token', async (c) => { + const form = await c.req.parseBody(); + const grantType = String(form['grant_type'] ?? ''); + const clientId = String(form['client_id'] ?? ''); + try { + if (grantType === 'authorization_code') { + const authCode = await provider.loadAuthorizationCode(String(form['code'] ?? '')); + if (!authCode) { + return c.json( + { error: 'invalid_grant', error_description: 'Invalid authorization code' }, + 400, + ); + } + return c.json(await provider.exchangeAuthorizationCode(clientId, authCode)); + } + if (grantType === 'refresh_token') { + const refreshToken = await provider.loadRefreshToken(String(form['refresh_token'] ?? '')); + if (!refreshToken) { + return c.json( + { error: 'invalid_grant', error_description: 'Invalid refresh token' }, + 400, + ); + } + const scopeParam = form['scope'] ? String(form['scope']).split(' ') : undefined; + return c.json(await provider.exchangeRefreshToken(clientId, refreshToken, scopeParam)); + } + return c.json({ error: 'unsupported_grant_type' }, 400); + } catch (err) { + if (err instanceof OAuthHttpError) { + return c.json( + { error: 'invalid_request', error_description: err.message }, + err.status as 400, + ); + } + throw err; + } + }); +}; + +export const createHttpApp = (baseConfig: Config): Hono<{ Bindings: Bindings }> => { + const app = new Hono<{ Bindings: Bindings }>(); + + app.get('/health-check', (c) => c.json({ status: 'ok' })); + app.get('/', (c) => c.json({ name: SERVER_NAME, version: SERVER_VERSION })); + + if (isOAuthConfigured(baseConfig)) { + if (baseConfig.mcpServerUrl) { + registerOAuthRoutes(app, baseConfig); + logger.info('OAuth provider enabled for the HTTP transport.'); + } else { + logger.warn( + 'OAuth client configured but mcpServerUrl is missing; OAuth routes are disabled.', + ); + } + } + + // Stateless streamable-HTTP: each POST gets a fresh server + transport, mirroring + // the Python `stateless_http=True` setup. No session is retained between requests. + app.post('/mcp', async (c) => { + const config = configFromHeaders(baseConfig, c.req.raw.headers); + const body: unknown = await c.req.json().catch(() => undefined); + + const server = createServer(config); + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); + await server.connect(transport); + + const { incoming, outgoing } = c.env; + outgoing.on('close', () => { + void transport.close(); + void server.close(); + }); + + await transport.handleRequest(incoming, outgoing, body); + return RESPONSE_ALREADY_SENT; + }); + + // Custom config-diff preview endpoint (port of the Python Starlette route). It is + // only registered for the HTTP transport — there is no stdio equivalent. It runs its + // own authorization (header + project/role/branch) and a read-only client, returning + // a config diff without performing any write. + app.post('/preview/configuration', async (c) => { + const config = configFromHeaders(baseConfig, c.req.raw.headers); + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ message: 'Invalid JSON in request body.' }, 400); + } + try { + const rq = parsePreviewRequest(body); + const resp = await runPreviewConfigDiff(config, rq); + return c.json(resp); + } catch (err) { + if (err instanceof PreviewHttpError) { + return c.json({ message: err.message }, err.status as 400); + } + throw err; + } + }); + + // Stateless mode has no sessions, so GET (SSE stream) and DELETE (session end) + // are not supported — match the SDK's stateless contract with 405. + const notAllowed = { + jsonrpc: '2.0', + error: { code: -32000, message: 'Method not allowed.' }, + id: null, + }; + app.get('/mcp', (c) => c.json(notAllowed, 405)); + app.delete('/mcp', (c) => c.json(notAllowed, 405)); + + return app; +}; + +export const startHttp = (config: Config, host: string, port: number): ReturnType => { + const app = createHttpApp(config); + const server = serve({ fetch: app.fetch, hostname: host, port }); + logger.info(`Starting MCP server with Streamable-HTTP transport on http://${host}:${port}/`); + return server; +}; diff --git a/src/transports/stdio.ts b/src/transports/stdio.ts new file mode 100644 index 000000000..30cc3955c --- /dev/null +++ b/src/transports/stdio.ts @@ -0,0 +1,8 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; + +/** Connects the server to stdio (the default transport for local MCP clients). */ +export const startStdio = async (server: McpServer): Promise => { + const transport = new StdioServerTransport(); + await server.connect(transport); +}; diff --git a/src/workspace.ts b/src/workspace.ts new file mode 100644 index 000000000..f0692fb2d --- /dev/null +++ b/src/workspace.ts @@ -0,0 +1,684 @@ +import { createQueryServiceClient } from '@keboola/api-client/queryService'; + +import type { RawClient } from '@/clients/raw'; +import { RawHttpError } from '@/clients/raw'; +import type { Config } from '@/config'; +import { logger } from '@/logger'; + +/** + * Workspace layer — port of `keboola_mcp_server.workspace`. + * + * Resolves (or creates) the read-only SQL workspace for the project/branch and runs + * `SELECT` queries over HTTP only: + * - Snowflake -> Query Service API (submit job -> poll -> paginate results) + * - BigQuery -> same Query Service API; differs only in identifier quoting, + * FQN construction, and error-message normalization. + * + * No DB drivers are used; all SQL flows through the Query Service. + */ + +const STORAGE_BRANCHES_FEATURE = 'storage-branches'; + +const QUERY_TIMEOUT_MS = 300_000; // 5 minutes +const CANCELLATION_TIMEOUT_MS = 30_000; // 30 seconds +const PAGE_SIZE = 1_000; +const SELECTED_ROWS_MSG = (rows: number, total: number | null) => + `Returning ${rows} of ${total} selected rows.`; + +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +// --------------------------------------------------------------------------- +// Result types (port of the pydantic dataclasses). +// --------------------------------------------------------------------------- + +export type SqlSelectDataRow = Record; + +export type SqlSelectData = { + columns: string[]; + rows: SqlSelectDataRow[]; +}; + +export type QueryResult = { + status: 'ok' | 'error'; + data?: SqlSelectData | null; + message?: string | null; +}; + +export type JobSubmittedInfo = { + job_id: string; + cancellation_url: string | null; + backend: string; +}; + +export type JobSubmittedCallback = (info: JobSubmittedInfo) => Promise; + +const isOk = (result: QueryResult): boolean => result.status === 'ok'; + +// --------------------------------------------------------------------------- +// Query Service client (built locally via the api-client factory). +// --------------------------------------------------------------------------- + +type QsClient = ReturnType; + +/** Status values Query Service reports for a terminal job. Snowflake/BQ both go through QS. */ +const TERMINAL_STATUSES = new Set(['completed', 'failed', 'canceled', 'cancelled']); +const CANCELLED_STATUSES = new Set(['canceled', 'cancelled']); + +// --------------------------------------------------------------------------- +// Workspace discovery (SAPI raw client) — port of `_WspInfo`. +// --------------------------------------------------------------------------- + +type WspInfo = { + id: number; + schema: string; + backend: string; + credentials: string | null; // serialized JSON for BigQuery + readonly: boolean | null; +}; + +const fromSapiInfo = (sapi: Record): WspInfo => { + const connection = (sapi.connection ?? {}) as Record; + return { + id: sapi.id as number, + schema: connection.schema as string, + backend: connection.backend as string, + credentials: (connection.user as string | undefined) ?? null, + readonly: (sapi.readOnlyStorageAccess as boolean | undefined) ?? null, + }; +}; + +// --------------------------------------------------------------------------- +// Per-backend workspace behavior — port of `_SnowflakeWorkspace` / `_BigQueryWorkspace`. +// --------------------------------------------------------------------------- + +abstract class Workspace { + protected qsClient: QsClient | null = null; + + constructor( + readonly id: number, + protected readonly deps: WorkspaceDeps, + ) {} + + abstract getSqlDialect(): string; + abstract getQuotedName(name: string): string; + protected formatErrorMessage(message: string | null): string | null { + return message; + } + + private async createQsClient(): Promise<{ client: QsClient; branchId: string }> { + let realBranchId = this.deps.branchId; + if (!realBranchId) { + const branches = await this.deps.rawStorage.get[]>('dev-branches'); + for (const branch of branches) { + if (branch.isDefault === true) { + realBranchId = String(branch.id); + break; + } + } + } + if (!realBranchId) { + throw new Error('Cannot determine the default branch ID'); + } + + const client = createQueryServiceClient({ + baseUrl: this.deps.queryServiceUrl, + token: this.deps.queryServiceToken, + middlewares: [], + }); + return { client, branchId: realBranchId }; + } + + private async ensureQsClient(): Promise<{ client: QsClient; branchId: string }> { + if (this.qsClient && this.cachedBranchId) { + return { client: this.qsClient, branchId: this.cachedBranchId }; + } + const created = await this.createQsClient(); + this.qsClient = created.client; + this.cachedBranchId = created.branchId; + return created; + } + + private cachedBranchId: string | null = null; + + async getBranchId(): Promise { + const { branchId } = await this.ensureQsClient(); + return branchId; + } + + buildCancelUrl(jobId: string): string { + return `${this.deps.queryServiceUrl}/api/v1/queries/${jobId}/cancel`; + } + + /** + * Cancel a query job and poll until cancellation is confirmed. + * Returns [cancellationConfirmed, queryCompleted]. + */ + private async cancelJobWithTimeout( + client: QsClient, + jobId: string, + reason: string, + ): Promise<[boolean, boolean]> { + try { + await client.cancelQueryJob(jobId); + logger.info(`Query cancellation requested: job_id=${jobId} reason=${reason}`); + + const cancelStart = Date.now(); + for (;;) { + const jobStatus = await client.getQueryJob(jobId); + const status = jobStatus.status as string | undefined; + if (!status) { + logger.warn(`Query status response missing "status" field: job_id=${jobId}`); + return [false, false]; + } + if (status === 'completed') { + logger.info(`Query completed successfully during cancellation attempt: job_id=${jobId}`); + return [true, true]; + } + if (status === 'failed' || CANCELLED_STATUSES.has(status)) { + logger.info(`Query job cancellation confirmed: job_id=${jobId}, status=${status}`); + return [true, false]; + } + if (Date.now() - cancelStart > CANCELLATION_TIMEOUT_MS) { + logger.info( + `Query cancellation polling timed out after ${CANCELLATION_TIMEOUT_MS / 1000}s: ` + + `job_id=${jobId}, status=${status}`, + ); + return [false, false]; + } + await wait(500); + } + } catch (error) { + logger.error({ err: error }, `Unexpected error during query cancellation: job_id=${jobId}`); + return [false, false]; + } + } + + async executeQuery( + sqlQuery: string, + opts: { + maxRows?: number | null; + maxChars?: number | null; + onJobSubmitted?: JobSubmittedCallback | null; + } = {}, + ): Promise { + const maxRows = opts.maxRows ?? null; + const maxChars = opts.maxChars ?? null; + if (maxRows !== null && maxRows <= 0) { + throw new Error('The "max_rows" must be a positive integer or None.'); + } + if (maxChars !== null && maxChars <= 0) { + throw new Error('The "max_chars" must be a positive integer or None.'); + } + + const { client, branchId } = await this.ensureQsClient(); + + const tsStart = Date.now(); + const submitResp = await client.createQueryJob(branchId, String(this.id), { + statements: [sqlQuery], + } as never); + const jobId = submitResp.queryJobId; + + if (opts.onJobSubmitted) { + const info: JobSubmittedInfo = { + job_id: jobId, + cancellation_url: this.buildCancelUrl(jobId), + backend: this.getSqlDialect().toLowerCase(), + }; + try { + await opts.onJobSubmitted(info); + } catch (exc) { + // Best-effort: a failed progress notification must not kill the running query. + logger.warn( + `on_job_submitted callback raised for job_id=${jobId}: ${String(exc)} — continuing`, + ); + } + } + + let jobStatus = await client.getQueryJob(jobId); + while (!TERMINAL_STATUSES.has(jobStatus.status as string)) { + await wait(1000); + const elapsed = Date.now() - tsStart; + if (elapsed > QUERY_TIMEOUT_MS) { + const [cancellationConfirmed, queryCompleted] = await this.cancelJobWithTimeout( + client, + jobId, + `Query timeout exceeded after ${(elapsed / 1000).toFixed(2)} seconds`, + ); + if (queryCompleted) { + logger.info( + `Query completed during cancellation polling, returning results: job_id=${jobId}`, + ); + jobStatus = await client.getQueryJob(jobId); + break; + } + if (cancellationConfirmed) { + throw new Error( + `Query execution timed out after ${(elapsed / 1000).toFixed(2)} seconds. ` + + `The query has been cancelled: job_id=${jobId}`, + ); + } + throw new Error( + `Query execution timed out after ${(elapsed / 1000).toFixed(2)} seconds. ` + + `Cancellation was attempted but could not be confirmed. ` + + `The query may still be running on the server: job_id=${jobId}`, + ); + } + jobStatus = await client.getQueryJob(jobId); + } + + // Short-circuit when the job reached a terminal CANCELLED state out-of-band. + const terminalStatus = jobStatus.status as string; + if (CANCELLED_STATUSES.has(terminalStatus)) { + logger.info(`Query was cancelled (terminal status=${terminalStatus}): job_id=${jobId}`); + return { status: 'error', data: null, message: 'Query was cancelled' }; + } + + const statements = jobStatus.statements as { id: string }[]; + const statementId = statements[0]!.id; + + // Fetch results with pagination. + const allRows: unknown[][] = []; + let allRowsChars = 0; + let columns: string[] = []; + let offset = 0; + let message: string | null = null; + let totalQueryRows: number | null = null; + + for (;;) { + let rowsToFetch: number; + if (maxRows !== null) { + const remaining = maxRows - allRows.length; + if (remaining <= 0) break; + rowsToFetch = Math.min(PAGE_SIZE, remaining); + } else { + rowsToFetch = PAGE_SIZE; + } + + const results = await client.getQueryResults(jobId, statementId, { + offset, + pageSize: Math.max(rowsToFetch, 100), // QueryService requires 100 - 10_000 + } as never); + + if (offset === 0) { + const status = results.status as string; + message = (results.message as string | undefined) ?? null; + totalQueryRows = (results.numberOfRows as number | undefined) ?? null; + + if (status === 'failed' || CANCELLED_STATUSES.has(status)) { + return { status: 'error', data: null, message: this.formatErrorMessage(message) }; + } + if (status !== 'completed') { + throw new Error(`Unexpected query status: ${status}`); + } + + columns = ((results.columns as { name: string }[] | undefined) ?? []).map((c) => c.name); + } + + const pageDataAll = (results.data as unknown[][] | undefined) ?? []; + if (pageDataAll.length === 0) break; + + const pageData = pageDataAll.slice(0, rowsToFetch); + let charLimitReached = false; + if (maxChars !== null) { + for (const row of pageData) { + const chars = row.reduce( + (sum, v) => (v !== null && v !== undefined ? sum + String(v).length : sum), + 0, + ); + if (allRowsChars + chars <= maxChars) { + allRows.push(row); + allRowsChars += chars; + } else { + // First row that does not fit ends pagination, keeping a contiguous prefix. + charLimitReached = true; + break; + } + } + } else { + allRows.push(...pageData); + } + + if (pageData.length < rowsToFetch) break; + if (maxRows !== null && allRows.length >= maxRows) break; + if (charLimitReached || (maxChars !== null && allRowsChars >= maxChars)) break; + + offset += pageData.length; + } + + const rows: SqlSelectDataRow[] = allRows.map((row) => { + const obj: SqlSelectDataRow = {}; + columns.forEach((colName, i) => { + obj[colName] = row[i]; + }); + return obj; + }); + + if (columns.length > 0) { + message = [message, SELECTED_ROWS_MSG(rows.length, totalQueryRows)].filter(Boolean).join(' '); + return { status: 'ok', data: { columns, rows }, message }; + } + return { status: 'ok', message }; + } +} + +class SnowflakeWorkspace extends Workspace { + getSqlDialect(): string { + return 'Snowflake'; + } + getQuotedName(name: string): string { + return `"${name}"`; + } +} + +class BigQueryWorkspace extends Workspace { + // Query Service surfaces BigQuery errors as a serialized object; extract the Message: "..." part. + private static readonly BQ_ERROR_MESSAGE_RE = /Message:\s*"((?:[^"\\]|\\.)*)"/; + + getSqlDialect(): string { + return 'BigQuery'; + } + getQuotedName(name: string): string { + return `\`${name}\``; + } + protected override formatErrorMessage(message: string | null): string | null { + if (message) { + const m = BigQueryWorkspace.BQ_ERROR_MESSAGE_RE.exec(message); + if (m) return m[1]!.replaceAll('\\"', '"'); + } + return message; + } +} + +// --------------------------------------------------------------------------- +// WorkspaceManager — port of `WorkspaceManager`. +// --------------------------------------------------------------------------- + +/** Everything a Workspace needs to talk to Storage + Query Service, resolved from Config. */ +type WorkspaceDeps = { + rawStorage: RawClient; + /** Effective branch id for SAPI branch-scoped endpoints ('default' on production). */ + storageBranchId: string; + /** The real branch id (config.branchId), or null on production. */ + branchId: string | null; + queryServiceUrl: string; + queryServiceToken: string; +}; + +export class WorkspaceManager { + static readonly MCP_META_KEY = 'KBC.McpServer.v2.workspaceId'; + static readonly MCP_WORKSPACE_COMPONENT_ID = 'keboola.mcp-server-tool'; + + private workspace: Workspace | null = null; + + private constructor( + private readonly deps: WorkspaceDeps, + private readonly workspaceSchema: string | undefined, + ) {} + + /** + * Builds a WorkspaceManager for the given config + raw storage client. + * + * On projects with the `storage-branches` feature (and a dev branch), the manager + * is bound to that branch's workspace. On legacy projects / the default branch it + * falls back to the production-branch workspace shared by the whole project. + */ + static async create( + config: Config, + deps: { + rawStorage: RawClient; + makeProdRawStorage: () => RawClient; + queryServiceUrl: string; + queryServiceToken: string; + }, + ): Promise { + const hasBranches = await WorkspaceManager.hasStorageBranches(config, deps.rawStorage); + if (hasBranches) { + return new WorkspaceManager( + { + rawStorage: deps.rawStorage, + storageBranchId: config.branchId ?? 'default', + branchId: config.branchId ?? null, + queryServiceUrl: deps.queryServiceUrl, + queryServiceToken: deps.queryServiceToken, + }, + config.workspaceSchema, + ); + } + // Fall back to the production-branch client. + return new WorkspaceManager( + { + rawStorage: deps.makeProdRawStorage(), + storageBranchId: 'default', + branchId: null, + queryServiceUrl: deps.queryServiceUrl, + queryServiceToken: deps.queryServiceToken, + }, + config.workspaceSchema, + ); + } + + private static async hasStorageBranches(config: Config, rawStorage: RawClient): Promise { + if (!config.branchId) return false; + const tokenInfo = await rawStorage.get>('tokens/verify'); + const owner = (tokenInfo.owner ?? {}) as Record; + const features = Array.isArray(owner.features) ? (owner.features as string[]) : []; + return features.includes(STORAGE_BRANCHES_FEATURE); + } + + private async findWsBySchema(schema: string): Promise { + const list = await this.deps.rawStorage.get[]>( + `branch/${this.deps.storageBranchId}/workspaces`, + ); + for (const sapi of list) { + const wi = fromSapiInfo(sapi); + if (wi.id && wi.backend && wi.schema && wi.schema === schema) { + return wi; + } + } + return null; + } + + private async findWsById(workspaceId: string | number): Promise { + try { + const sapi = await this.deps.rawStorage.get>( + `branch/${this.deps.storageBranchId}/workspaces/${workspaceId}`, + ); + const wi = fromSapiInfo(sapi); + if (wi.id && wi.backend && wi.schema) { + return wi; + } + throw new Error(`Invalid workspace info: ${JSON.stringify(sapi)}`); + } catch (error) { + if (error instanceof RawHttpError && error.status === 404) { + return null; + } + throw error; + } + } + + private async findWsInBranch(): Promise { + const metadata = await this.deps.rawStorage.get[]>( + `branch/${this.deps.storageBranchId}/metadata`, + ); + for (const m of metadata) { + if (m.key === WorkspaceManager.MCP_META_KEY && m.value) { + const info = await this.findWsById(m.value as string); + if (info && info.readonly) { + return info; + } + } + } + return null; + } + + private async createWs(timeoutSec = 300.0): Promise { + const tokenInfo = await this.deps.rawStorage.get>('tokens/verify'); + const owner = (tokenInfo.owner ?? {}) as Record; + const defaultBackend = owner.defaultBackend as string | undefined; + + let loginType: string; + if (defaultBackend === 'snowflake') { + loginType = 'snowflake-person-sso'; + } else if (defaultBackend === 'bigquery') { + loginType = 'default'; + } else { + throw new Error(`Unexpected default backend: ${defaultBackend}`); + } + + const componentId = WorkspaceManager.MCP_WORKSPACE_COMPONENT_ID; + const configName = `mcp-workspace-${Math.random().toString(16).slice(2, 10)}`; + const configResp = await this.deps.rawStorage.post>( + `branch/${this.deps.storageBranchId}/components/${componentId}/configs`, + { + body: { + name: configName, + description: 'Auto-created by MCP server for workspace billing.', + configuration: {}, + }, + }, + ); + const configId = String(configResp.id); + + let resp: Record; + try { + resp = await this.deps.rawStorage.post>( + `branch/${this.deps.storageBranchId}/components/${componentId}/configs/${configId}/workspaces`, + { + params: { async: true }, + body: { readOnlyStorageAccess: true, loginType, backend: defaultBackend }, + }, + ); + } catch (error) { + try { + await this.deps.rawStorage.delete( + `branch/${this.deps.storageBranchId}/components/${componentId}/configs/${configId}`, + ); + } catch (cleanupErr) { + logger.warn( + `Failed to clean up configuration ${componentId}/${configId} ` + + `after workspace creation failure: ${String(cleanupErr)}`, + ); + } + throw error; + } + + const jobId = resp.id as number; + const startTs = Date.now(); + logger.info( + `Requested new workspace: job_id=${jobId}, timeout=${timeoutSec.toFixed(2)} seconds`, + ); + + for (;;) { + const jobInfo = await this.deps.rawStorage.get>(`jobs/${jobId}`); + const jobStatusVal = jobInfo.status as string; + const duration = (Date.now() - startTs) / 1000; + logger.info( + `Job info: job_id=${jobId}, status=${jobStatusVal}, ` + + `duration=${duration.toFixed(2)} seconds, timeout=${timeoutSec.toFixed(2)} seconds`, + ); + + if (jobStatusVal === 'success') { + const jobResults = jobInfo.results as Record; + const workspaceId = jobResults.id as number; + logger.info(`Created workspace: ${workspaceId}`); + return this.findWsById(workspaceId); + } + if (duration > timeoutSec) { + logger.info(`Workspace creation timed out after ${duration.toFixed(2)} seconds.`); + return null; + } + const remaining = Math.max(0.0, timeoutSec - duration); + await wait(Math.min(5.0, remaining) * 1000); + } + } + + private initWorkspace(info: WspInfo): Workspace { + if (info.backend === 'snowflake') { + return new SnowflakeWorkspace(info.id, this.deps); + } + if (info.backend === 'bigquery') { + const credentials = JSON.parse(info.credentials || '{}') as Record; + const projectId = credentials.project_id as string | undefined; + if (projectId) { + return new BigQueryWorkspace(info.id, this.deps); + } + throw new Error(`No credentials or no project ID in workspace: ${info.schema}`); + } + throw new Error(`Unexpected backend type "${info.backend}" in workspace: ${info.schema}`); + } + + private async getWorkspace(): Promise { + if (this.workspace) return this.workspace; + + if (this.workspaceSchema) { + // Use the explicitly-requested workspace; never written to the default branch metadata. + logger.info(`Looking up workspace by schema: ${this.workspaceSchema}`); + const info = await this.findWsBySchema(this.workspaceSchema); + if (info) { + logger.info(`Found workspace: ${JSON.stringify(info)}`); + this.workspace = this.initWorkspace(info); + return this.workspace; + } + throw new Error( + `No Keboola workspace found or the workspace has no read-only storage access: ` + + `workspace_schema=${this.workspaceSchema}`, + ); + } + + logger.info('Looking up workspace in the default branch.'); + const existing = await this.findWsInBranch(); + if (existing) { + logger.info(`Found workspace: ${JSON.stringify(existing)}`); + this.workspace = this.initWorkspace(existing); + return this.workspace; + } + + logger.info('Creating workspace in the default branch.'); + const created = await this.createWs(); + if (created) { + // All tokens share the same read-only workspace; last-write-wins is acceptable. + await this.deps.rawStorage.post(`branch/${this.deps.storageBranchId}/metadata`, { + body: { + metadata: [{ key: WorkspaceManager.MCP_META_KEY, value: created.id }], + }, + }); + this.workspace = this.initWorkspace(created); + return this.workspace; + } + throw new Error('Failed to initialize Keboola Workspace.'); + } + + async executeQuery( + sqlQuery: string, + opts: { + maxRows?: number | null; + maxChars?: number | null; + onJobSubmitted?: JobSubmittedCallback | null; + } = {}, + ): Promise { + const workspace = await this.getWorkspace(); + return workspace.executeQuery(sqlQuery, opts); + } + + async getQuotedName(name: string): Promise { + const workspace = await this.getWorkspace(); + return workspace.getQuotedName(name); + } + + async getSqlDialect(): Promise { + const workspace = await this.getWorkspace(); + return workspace.getSqlDialect(); + } + + async getWorkspaceId(): Promise { + const workspace = await this.getWorkspace(); + return workspace.id; + } + + async getBranchId(): Promise { + const workspace = await this.getWorkspace(); + return workspace.getBranchId(); + } +} + +export { isOk }; diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index aed7735f6..000000000 --- a/tests/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -import os -import sys - -sys.path.append(os.path.dirname(os.path.realpath(__file__)) + '/../src') diff --git a/tests/clients/test_client.py b/tests/clients/test_client.py deleted file mode 100644 index 1d4e075a0..000000000 --- a/tests/clients/test_client.py +++ /dev/null @@ -1,761 +0,0 @@ -import importlib.metadata -import json -from typing import Any, Mapping -from unittest.mock import AsyncMock, Mock, PropertyMock, patch - -import httpx -import pytest -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.base import RawKeboolaClient -from keboola_mcp_server.clients.client import KeboolaClient, get_metadata_property -from keboola_mcp_server.clients.storage import AsyncStorageClient -from keboola_mcp_server.config import ServerRuntimeInfo -from keboola_mcp_server.mcp import SessionStateMiddleware - - -@pytest.fixture -def keboola_client() -> KeboolaClient: - return KeboolaClient(storage_api_url='https://connection.nowhere', storage_api_token='test-token') - - -@pytest.fixture -def mock_http_request() -> httpx.Request: - """Create a mock HTTP request.""" - request = Mock(spec=httpx.Request) - request.url = 'https://api.example.com/test' - request.method = 'GET' - return request - - -@pytest.fixture -def mock_http_response_500(mock_http_request: httpx.Request) -> httpx.Response: - """Create a mock HTTP response with 500 status.""" - response = Mock(spec=httpx.Response) - response.status_code = 500 - response.reason_phrase = 'Internal Server Error' - response.url = 'https://api.example.com/test' - response.request = mock_http_request - response.is_error = True - response.raise_for_status.side_effect = httpx.HTTPStatusError( - message=f"{response.reason_phrase} for url '{response.url}'", request=mock_http_request, response=response - ) - return response - - -@pytest.fixture -def mock_http_response_404(mock_http_request: httpx.Request) -> httpx.Response: - """Create a mock HTTP response with 404 status.""" - response = Mock(spec=httpx.Response) - response.status_code = 404 - response.reason_phrase = 'Not Found' - response.url = 'https://api.example.com/test' - response.request = mock_http_request - response.is_error = True - response.raise_for_status.side_effect = httpx.HTTPStatusError( - message=f"{response.reason_phrase} for url '{response.url}'", request=mock_http_request, response=response - ) - return response - - -class TestRawKeboolaClient: - """Test suite for enhanced HTTP client error handling.""" - - @pytest.fixture - def raw_client(self) -> RawKeboolaClient: - """Create a RawKeboolaClient instance for testing.""" - return RawKeboolaClient(base_api_url='https://api.example.com', api_token='test-token') - - def test_raise_for_status_500_with_exception_id( - self, raw_client: RawKeboolaClient, mock_http_response_500: httpx.Response - ): - """Test that HTTP 500 errors are enhanced with exception ID when available.""" - - # Mock response with valid JSON containing exception ID - mock_http_response_500.json.return_value = { - 'exceptionId': 'exc-123-456', - 'message': 'Application error', - 'errorCode': 'DB_ERROR', - 'requestId': 'req-789', - } - - match = ( - "Internal Server Error for url 'https://api.example.com/test'\n" - 'Exception ID: exc-123-456\n' - 'When contacting Keboola support please provide the exception ID.' - ) - with pytest.raises(httpx.HTTPStatusError, match=match): - raw_client._raise_for_status(mock_http_response_500) - - def test_raise_for_status_500_without_exception_id( - self, raw_client: RawKeboolaClient, mock_http_response_500: httpx.Response - ): - """Test that HTTP 500 errors without exception ID fall back gracefully.""" - - # Mock response with JSON but no exception ID - mock_http_response_500.json.return_value = {'message': 'Internal server error', 'errorCode': 'INTERNAL_ERROR'} - - with pytest.raises(httpx.HTTPStatusError, match="Internal Server Error for url 'https://api.example.com/test'"): - raw_client._raise_for_status(mock_http_response_500) - - def test_raise_for_status_500_with_malformed_json( - self, raw_client: RawKeboolaClient, mock_http_response_500: httpx.Response - ): - """Test that HTTP 500 errors with malformed JSON fall back to standard error handling.""" - - # Mock response with invalid JSON - type(mock_http_response_500).text = PropertyMock(return_value='Invalid JSON') - mock_http_response_500.json.side_effect = ValueError('Invalid JSON') - - match = "Internal Server Error for url 'https://api.example.com/test'\n" 'API error: Invalid JSON' - with pytest.raises(httpx.HTTPStatusError, match=match): - raw_client._raise_for_status(mock_http_response_500) - - def test_raise_for_status_404_uses_standard_exception( - self, raw_client: RawKeboolaClient, mock_http_response_404: httpx.Response - ): - """Test that HTTP 404 errors use standard HTTPStatusError.""" - - mock_http_response_404.json.return_value = { - 'exceptionId': 'exc-123-456', - 'error': 'The bucket "foo.bar.baz" was not found in the project "123"', - 'code': 'storage.buckets.notFound', - } - - match = ( - "Not Found for url 'https://api.example.com/test'\n" - 'API error: The bucket "foo.bar.baz" was not found in the project "123"\n' - 'Exception ID: exc-123-456\n' - 'When contacting Keboola support please provide the exception ID.' - ) - with pytest.raises(httpx.HTTPStatusError, match=match): - raw_client._raise_for_status(mock_http_response_404) - - @pytest.mark.asyncio - async def test_get_method_integration_with_enhanced_error_handling( - self, raw_client: RawKeboolaClient, mock_http_response_500: httpx.Response - ): - """Test that GET method integrates with enhanced error handling.""" - - # Mock the HTTP client to return a 500 error - with patch('httpx.AsyncClient') as mock_client_class: - mock_client_class.return_value.__aenter__.return_value = (mock_client := AsyncMock()) - mock_client.get.return_value = mock_http_response_500 - mock_http_response_500.json.return_value = {'exceptionId': 'test-exc-123', 'message': 'Test error message'} - - match = ( - "Internal Server Error for url 'https://api.example.com/test'\n" - 'Exception ID: test-exc-123\n' - 'When contacting Keboola support please provide the exception ID.' - ) - with pytest.raises(httpx.HTTPStatusError, match=match): - await raw_client.get('test-endpoint') - - @pytest.mark.asyncio - async def test_post_preserves_non_ascii_characters(self, raw_client: RawKeboolaClient): - """Test that POST requests preserve non-ASCII characters (e.g. Czech diacritics) in JSON payloads.""" - with patch('httpx.AsyncClient') as mock_client_class: - mock_client_class.return_value.__aenter__.return_value = (mock_client := AsyncMock()) - mock_client.post.return_value = (response := Mock(spec=httpx.Response)) - response.status_code = 200 - response.json.return_value = {} - - data = {'script': "SELECT * WHERE name = 'Česká republika'"} - await raw_client.post('test-endpoint', data=data) - - call_kwargs = mock_client.post.call_args - content_bytes = call_kwargs.kwargs['content'] - content_str = content_bytes.decode('utf-8') - - # Verify non-ASCII characters are preserved, not escaped to \uXXXX - assert 'Česká republika' in content_str - assert '\\u010c' not in content_str - - -class TestAsyncStorageClient: - @pytest.fixture - def storage_client(self, mocker: MockerFixture) -> AsyncStorageClient: - raw = mocker.AsyncMock(RawKeboolaClient) - return AsyncStorageClient(raw_client=raw, branch_id=None) - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('limit', 'offset', 'expected_params'), - [ - pytest.param(50, 0, {'runId': '456', 'limit': 50, 'offset': 0, 'forceUuid': 'true'}, id='basic'), - pytest.param(10, 100, {'runId': '456', 'limit': 10, 'offset': 100, 'forceUuid': 'true'}, id='with_offset'), - ], - ) - async def test_list_events( - self, - storage_client: AsyncStorageClient, - limit: int, - offset: int, - expected_params: dict[str, Any], - ): - """Tests list_events calls the correct endpoint with the right params.""" - storage_client.raw_client.get.return_value = [] - - await storage_client.list_events(job_id='456', limit=limit, offset=offset) - - storage_client.raw_client.get.assert_called_once_with( - endpoint='events', - params=expected_params, - ) - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('message', 'component_id', 'configuration_id', 'event_type', 'params', 'results', 'duration', 'run_id'), - [ - ('foo', 'bar', None, None, None, None, None, None), - ('foo', 'bar', 'baz', 'error', {'param1': 'value1'}, {'result1': 'value1'}, 123, '987654321'), - ], - ) - async def test_trigger_event( - self, - message: str, - component_id: str, - configuration_id: str | None, - event_type: str | None, - params: Mapping[str, Any] | None, - results: Mapping[str, Any], - duration: int | None, - run_id: str | None, - keboola_client: KeboolaClient, - ): - with patch('httpx.AsyncClient') as mock_client_class: - mock_client_class.return_value.__aenter__.return_value = (mock_client := AsyncMock()) - mock_client.post.return_value = (response := Mock(spec=httpx.Response)) - response.status_code = 200 - response.json.return_value = {'id': '13008826', 'uuid': '01958f48-b1fc-7f05-b9b9-8a4a7b385bc3'} - - result = await keboola_client.storage_client.trigger_event( - message=message, - component_id=component_id, - configuration_id=configuration_id, - event_type=event_type, - params=params, - results=results, - duration=duration, - run_id=run_id, - ) - - assert result == {'id': '13008826', 'uuid': '01958f48-b1fc-7f05-b9b9-8a4a7b385bc3'} - expected_payload = { - key: value - for key, value in [ - ('message', message), - ('component', component_id), - ('configurationId', configuration_id), - ('type', event_type), - ('params', params), - ('results', results), - ('duration', duration), - ('runId', run_id), - ] - if value - } - mock_client.post.assert_called_once_with( - 'https://connection.nowhere/v2/storage/events', - params=None, - headers={ - 'Content-Type': 'application/json', - 'Accept-Encoding': 'gzip', - 'X-StorageAPI-Token': 'test-token', - }, - content=json.dumps(expected_payload, ensure_ascii=False).encode('utf-8'), - ) - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('description', 'component_access', 'expires_in', 'expected_data'), - [ - # Basic token creation with just description - ('Test token', None, None, {'description': 'Test token'}), - # Token with component access - ( - 'OAuth token', - ['keboola.ex-google-analytics-v4'], - None, - {'description': 'OAuth token', 'componentAccess': ['keboola.ex-google-analytics-v4']}, - ), - # Token with expiration - ('Short-lived token', None, 3600, {'description': 'Short-lived token', 'expiresIn': 3600}), - # Token with all parameters - ( - 'Full token', - ['keboola.ex-gmail', 'keboola.ex-google-analytics-v4'], - 7200, - { - 'description': 'Full token', - 'componentAccess': ['keboola.ex-gmail', 'keboola.ex-google-analytics-v4'], - 'expiresIn': 7200, - }, - ), - ], - ) - async def test_token_create( - self, - description: str, - component_access: list[str] | None, - expires_in: int | None, - expected_data: dict[str, Any], - keboola_client: KeboolaClient, - ): - """Test token creation with various parameter combinations.""" - with patch('httpx.AsyncClient') as mock_client_class: - mock_client_class.return_value.__aenter__.return_value = (mock_client := AsyncMock()) - mock_client.post.return_value = (response := Mock(spec=httpx.Response)) - response.status_code = 201 - response.json.return_value = { - 'id': '12345', - 'token': 'KBC_TOKEN_TEST_12345', - 'description': description, - 'created': '2023-01-01T00:00:00+00:00', - 'expiresIn': expires_in, - 'componentAccess': component_access or [], - } - - result = await keboola_client.storage_client.token_create( - description=description, component_access=component_access, expires_in=expires_in - ) - - # Verify the response - assert result['token'] == 'KBC_TOKEN_TEST_12345' - assert result['description'] == description - - # Verify the API call was made with correct parameters - mock_client.post.assert_called_once_with( - 'https://connection.nowhere/v2/storage/tokens', - params=None, - headers={ - 'Content-Type': 'application/json', - 'Accept-Encoding': 'gzip', - 'X-StorageAPI-Token': 'test-token', - }, - content=json.dumps(expected_data, ensure_ascii=False).encode('utf-8'), - ) - - -class TestKeboolaClient: - - @pytest.fixture - def runtime_config(self) -> ServerRuntimeInfo: - return ServerRuntimeInfo(transport='stdio', server_id='test') - - @pytest.fixture - def keboola_client_with_headers(self, runtime_config: ServerRuntimeInfo) -> KeboolaClient: - headers = SessionStateMiddleware._get_headers(runtime_config) - return KeboolaClient( - storage_api_url='https://connection.nowhere', storage_api_token='test-token', headers=headers - ) - - @pytest.mark.asyncio - async def test_keboola_client_passing_headers(self, keboola_client_with_headers: KeboolaClient): - with patch('httpx.AsyncClient') as mock_client_class: - mock_client_class.return_value.__aenter__.return_value = (mock_client := AsyncMock()) - mock_client.get.return_value = (response := Mock(spec=httpx.Response)) - response.status_code = 201 - response.json.return_value = {'test': 'test'} - result = await keboola_client_with_headers.storage_client.verify_token() - assert result == {'test': 'test'} - kbc_version = importlib.metadata.version('keboola-mcp-server') - mcp_version = importlib.metadata.version('mcp') - fastmcp_version = importlib.metadata.version('fastmcp') - mock_client.get.assert_called_once_with( - 'https://connection.nowhere/v2/storage/tokens/verify', - params=None, - headers={ - 'Content-Type': 'application/json', - 'Accept-Encoding': 'gzip', - 'X-StorageAPI-Token': 'test-token', - 'User-Agent': f'Keboola MCP Server/{kbc_version} app_env=local transport=stdio', - 'MCP-Server-Transport': 'stdio', - 'MCP-Server-Versions': ( - f'keboola-mcp-server/{kbc_version} mcp/{mcp_version} fastmcp/{fastmcp_version}' - ), - }, - ) - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('status_code', 'expected_match'), - [ - (404, 'Branch "non-existent-branch" not found'), - (500, 'Internal Server Error'), - ], - ids=['not_found', 'server_error'], - ) - async def test_with_branch_id_http_error( - self, keboola_client: KeboolaClient, status_code: int, expected_match: str - ): - with patch('httpx.AsyncClient') as mock_client_class: - mock_client_class.return_value.__aenter__.return_value = (mock_client := AsyncMock()) - - response = Mock(spec=httpx.Response) - response.status_code = status_code - response.is_error = True - response.text = '{"error":"some error"}' - response.json.return_value = {'error': 'some error'} - response.request = Mock(spec=httpx.Request) - response.raise_for_status.side_effect = httpx.HTTPStatusError( - 'Internal Server Error', request=response.request, response=response - ) - mock_client.get.return_value = response - - with pytest.raises(httpx.HTTPStatusError, match=expected_match): - await keboola_client.with_branch_id('non-existent-branch') - mock_client.get.assert_called_once() - - @pytest.mark.parametrize( - ('bearer_token', 'storage_token', 'expected_scheduler_token'), - [ - ('oauth_bearer_123', 'sapi_token_456', 'Bearer oauth_bearer_123'), - (None, 'sapi_token_456', 'sapi_token_456'), - ('', 'sapi_token_456', 'sapi_token_456'), - ], - ids=['with_bearer_token', 'without_bearer_token', 'empty_bearer_token'], - ) - def test_scheduler_client_token_selection( - self, bearer_token: str | None, storage_token: str, expected_scheduler_token: str - ): - """Test SchedulerClient uses bearer token when available, falls back to storage token.""" - # Create KeboolaClient with different token configurations - client = KeboolaClient( - storage_api_url='https://connection.keboola.com', - storage_api_token=storage_token, - bearer_token=bearer_token, - ) - - # Verify scheduler client was initialized with correct token - # Check the headers of the underlying RawKeboolaClient - scheduler_headers = client.scheduler_client.raw_client.headers - - if expected_scheduler_token.startswith('Bearer '): - # Should use Authorization header for bearer token - assert 'Authorization' in scheduler_headers - assert scheduler_headers['Authorization'] == expected_scheduler_token - assert 'X-StorageAPI-Token' not in scheduler_headers - else: - # Should use X-StorageAPI-Token header for storage token - assert 'X-StorageAPI-Token' in scheduler_headers - assert scheduler_headers['X-StorageAPI-Token'] == expected_scheduler_token - assert 'Authorization' not in scheduler_headers - - def test_metastore_client_url_derivation(self) -> None: - client = KeboolaClient( - storage_api_url='https://connection.canary-orion.keboola.dev', - storage_api_token='sapi_token_456', - ) - - assert client.metastore_client.raw_client.base_api_url == 'https://metastore.canary-orion.keboola.dev' - assert client.metastore_client.raw_client.headers['X-StorageAPI-Token'] == 'sapi_token_456' - - @pytest.mark.parametrize( - ('bearer_token', 'storage_token', 'expected_metastore_token'), - [ - ('oauth_bearer_123', 'sapi_token_456', 'Bearer oauth_bearer_123'), - (None, 'sapi_token_456', 'sapi_token_456'), - ('', 'sapi_token_456', 'sapi_token_456'), - ], - ids=['with_bearer_token', 'without_bearer_token', 'empty_bearer_token'], - ) - def test_metastore_client_token_selection( - self, bearer_token: str | None, storage_token: str, expected_metastore_token: str - ): - """Test MetastoreClient uses bearer token when available, falls back to storage token.""" - client = KeboolaClient( - storage_api_url='https://connection.keboola.com', - storage_api_token=storage_token, - bearer_token=bearer_token, - ) - - metastore_headers = client.metastore_client.raw_client.headers - - if expected_metastore_token.startswith('Bearer '): - assert 'Authorization' in metastore_headers - assert metastore_headers['Authorization'] == expected_metastore_token - assert 'X-StorageAPI-Token' not in metastore_headers - else: - assert 'X-StorageAPI-Token' in metastore_headers - assert metastore_headers['X-StorageAPI-Token'] == expected_metastore_token - assert 'Authorization' not in metastore_headers - - @pytest.mark.parametrize( - ('bearer_token', 'storage_token', 'expected_data_science_token'), - [ - ('oauth_bearer_123', 'sapi_token_456', 'Bearer oauth_bearer_123'), - (None, 'sapi_token_456', 'sapi_token_456'), - ('', 'sapi_token_456', 'sapi_token_456'), - ], - ids=['with_bearer_token', 'without_bearer_token', 'empty_bearer_token'], - ) - def test_data_science_client_token_selection( - self, bearer_token: str | None, storage_token: str, expected_data_science_token: str - ): - """DataScienceClient uses the bearer token when available, falls back to the storage token. - - The sandboxes-service git-repo credential endpoints require an admin-context token - (CanManageAppRepoCredentials -> isAdminToken()); the OAuth bearer token carries it while the - minted SAPI token does not (AI-3398). - """ - client = KeboolaClient( - storage_api_url='https://connection.keboola.com', - storage_api_token=storage_token, - bearer_token=bearer_token, - ) - - data_science_headers = client.data_science_client.raw_client.headers - - if expected_data_science_token.startswith('Bearer '): - assert 'Authorization' in data_science_headers - assert data_science_headers['Authorization'] == expected_data_science_token - assert 'X-StorageAPI-Token' not in data_science_headers - else: - assert 'X-StorageAPI-Token' in data_science_headers - assert data_science_headers['X-StorageAPI-Token'] == expected_data_science_token - assert 'Authorization' not in data_science_headers - - -def test_flow_schema_cache_roundtrip(): - client = KeboolaClient( - storage_api_url='https://connection.keboola.com', - storage_api_token='dummy-token', - ) - assert client.get_cached_flow_schema('keboola.flow') is None - schema = {'type': 'object'} - client.cache_flow_schema('keboola.flow', schema) - assert client.get_cached_flow_schema('keboola.flow') is schema - # other flow types are independent - assert client.get_cached_flow_schema('keboola.orchestrator') is None - - -@pytest.mark.parametrize( - ('metadata', 'key', 'provider', 'preferred_providers', 'default', 'expected'), - [ - # Basic retrieval by key - ( - [{'key': 'description', 'value': 'Test description'}, {'key': 'owner', 'value': 'John Doe'}], - 'description', - None, - None, - None, - 'Test description', - ), - # Key not found returns None - ( - [{'key': 'description', 'value': 'Test description'}], - 'nonexistent', - None, - None, - None, - None, - ), - # Key not found returns default value - ( - [{'key': 'description', 'value': 'Test description'}], - 'nonexistent', - None, - None, - 'default_value', - 'default_value', - ), - # Filter by provider - ( - [ - {'key': 'description', 'value': 'Provider A description', 'provider': 'provider-a'}, - {'key': 'description', 'value': 'Provider B description', 'provider': 'provider-b'}, - ], - 'description', - 'provider-b', - None, - None, - 'Provider B description', - ), - # Most recent by timestamp - ( - [ - {'key': 'description', 'value': 'Old description', 'timestamp': '2024-01-01T00:00:00Z'}, - {'key': 'description', 'value': 'New description', 'timestamp': '2024-12-01T00:00:00Z'}, - {'key': 'description', 'value': 'Middle description', 'timestamp': '2024-06-01T00:00:00Z'}, - ], - 'description', - None, - None, - None, - 'New description', - ), - # Handles missing timestamps - ( - [ - {'key': 'description', 'value': 'No timestamp'}, - {'key': 'description', 'value': 'With timestamp', 'timestamp': '2024-01-01T00:00:00Z'}, - ], - 'description', - None, - None, - None, - 'With timestamp', - ), - # Preferred providers prioritized - ( - [ - { - 'key': 'description', - 'value': 'Provider A', - 'provider': 'provider-a', - 'timestamp': '2024-01-01T00:00:00Z', - }, - { - 'key': 'description', - 'value': 'Provider B', - 'provider': 'provider-b', - 'timestamp': '2024-01-02T00:00:00Z', - }, - { - 'key': 'description', - 'value': 'Provider C', - 'provider': 'provider-c', - 'timestamp': '2024-01-03T00:00:00Z', - }, - { - 'key': 'description', - 'value': 'Provider X', - 'provider': 'provider-X', # not in the preferred_providers list - 'timestamp': '2024-01-03T00:00:00Z', - }, - ], - 'description', - None, - ['provider-b', 'provider-c', 'provider-a'], - None, - 'Provider B', - ), - # Timestamp used when same provider preference - ( - [ - { - 'key': 'description', - 'value': 'Old preferred', - 'provider': 'provider-a', - 'timestamp': '2024-01-01T00:00:00Z', - }, - { - 'key': 'description', - 'value': 'New preferred', - 'provider': 'provider-a', - 'timestamp': '2024-12-01T00:00:00Z', - }, - ], - 'description', - None, - ['provider-a'], - None, - 'New preferred', - ), - # Empty metadata list returns None - ( - [], - 'description', - None, - None, - None, - None, - ), - # Empty metadata list returns default - ( - [], - 'description', - None, - None, - 'default_value', - 'default_value', - ), - # None value returns default - ( - [{'key': 'description', 'value': None}], - 'description', - None, - None, - 'default_value', - 'default_value', - ), - # Combined provider and timestamp filtering - ( - [ - { - 'key': 'description', - 'value': 'Provider A old', - 'provider': 'provider-a', - 'timestamp': '2024-01-01T00:00:00Z', - }, - { - 'key': 'description', - 'value': 'Provider A new', - 'provider': 'provider-a', - 'timestamp': '2024-12-01T00:00:00Z', - }, - { - 'key': 'description', - 'value': 'Provider B', - 'provider': 'provider-b', - 'timestamp': '2024-12-31T00:00:00Z', - }, - ], - 'description', - 'provider-a', - None, - None, - 'Provider A new', - ), - # Metadata entries without the provider field - ( - [ - {'key': 'description', 'value': 'No provider entry', 'timestamp': '2024-01-01T00:00:00Z'}, - { - 'key': 'description', - 'value': 'With provider', - 'provider': 'provider-a', - 'timestamp': '2024-01-02T00:00:00Z', - }, - ], - 'description', - None, - ['provider-a'], - None, - 'With provider', - ), - ], - ids=[ - 'basic_retrieval_by_key', - 'key_not_found_returns_none', - 'key_not_found_returns_default', - 'filter_by_provider', - 'most_recent_by_timestamp', - 'handles_missing_timestamps', - 'preferred_providers_prioritized', - 'timestamp_used_when_same_preference', - 'empty_metadata_list_returns_none', - 'empty_metadata_list_returns_default', - 'none_value_returns_default', - 'combined_provider_and_timestamp', - 'no_provider_in_metadata', - ], -) -def test_get_metadata_property( - metadata: list[Mapping[str, Any]], - key: str, - provider: str | None, - preferred_providers: list[str] | None, - default: Any, - expected: Any, -): - """Test get_metadata_property with various scenarios.""" - result = get_metadata_property( - metadata=metadata, - key=key, - provider=provider, - preferred_providers=preferred_providers, - default=default, - ) - assert result == expected diff --git a/tests/clients/test_data_science.py b/tests/clients/test_data_science.py deleted file mode 100644 index a3c4e0ced..000000000 --- a/tests/clients/test_data_science.py +++ /dev/null @@ -1,287 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta, timezone -from unittest.mock import AsyncMock - -import pytest - -from keboola_mcp_server.clients.data_science import ( - AppGitRepoResponse, - AppRunResponse, - CodeDataAppConfig, - CreatedGitCredentialResponse, - DataScienceClient, -) - - -@pytest.mark.asyncio -async def test_tail_app_logs_with_lines_calls_get_text_with_lines() -> None: - client = DataScienceClient.create('https://api.example.com', token=None) - client.get_text = AsyncMock(return_value='LOGS') # type: ignore[assignment] - - result = await client.tail_app_logs('app-123', since=None, lines=5) - - assert result == 'LOGS' - client.get_text.assert_awaited_once_with(endpoint='apps/app-123/logs/tail', params={'lines': 5}) - - -@pytest.mark.asyncio -async def test_tail_app_logs_with_lines_minimum_enforced() -> None: - client = DataScienceClient.create('https://api.example.com', token=None) - client.get_text = AsyncMock(return_value='LOGS') # type: ignore[assignment] - - _ = await client.tail_app_logs('app-123', since=None, lines=0) - - client.get_text.assert_awaited_once_with(endpoint='apps/app-123/logs/tail', params={'lines': 1}) - - -@pytest.mark.asyncio -async def test_tail_app_logs_with_since_calls_get_text_with_since_param() -> None: - client = DataScienceClient.create('https://api.example.com', token=None) - client.get_text = AsyncMock(return_value='LOGS') # type: ignore[assignment] - - since = datetime.now(timezone.utc) - timedelta(days=1) - result = await client.tail_app_logs('app-xyz', since=since, lines=None) - - assert result == 'LOGS' - client.get_text.assert_awaited_once_with( - endpoint='apps/app-xyz/logs/tail', params={'since': since.isoformat(timespec='microseconds')} - ) - - -@pytest.mark.asyncio -async def test_tail_app_logs_raises_when_both_since_and_lines_provided() -> None: - client = DataScienceClient.create('https://api.example.com', token=None) - - with pytest.raises(ValueError, match='You cannot use both "since" and "lines"'): - await client.tail_app_logs('app-123', since=datetime.now(timezone.utc), lines=10) - - -@pytest.mark.asyncio -async def test_tail_app_logs_raises_when_neither_param_provided() -> None: - client = DataScienceClient.create('https://api.example.com', token=None) - - with pytest.raises(ValueError, match='Either "since" or "lines" must be provided.'): - await client.tail_app_logs('app-123', since=None, lines=None) - - -# ---------- Tests for python-js / managed-git-repo support ---------- - - -def _code_app_config() -> CodeDataAppConfig: - return CodeDataAppConfig( - parameters=CodeDataAppConfig.Parameters( - auto_suspend_after_seconds=900, - data_app=CodeDataAppConfig.Parameters.DataApp(slug='my-app'), - ), - runtime=CodeDataAppConfig.Runtime( - image=CodeDataAppConfig.Runtime.Image(version='dev-1.0.0'), - ), - ) - - -def _create_app_response(app_id: str = 'app-123', config_id: str = 'cfg-456') -> dict: - return { - 'id': app_id, - 'projectId': 'p-1', - 'componentId': 'keboola.data-apps', - 'branchId': None, - 'configId': config_id, - 'configVersion': '1', - 'type': 'python-js', - 'state': 'created', - 'desiredState': 'created', - } - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('app_type', 'use_managed_git_repo', 'expected_extra_payload'), - [ - ('streamlit', False, {}), - ('python-js', True, {'useManagedGitRepo': True}), - # Dev twin path: python-js app with an external-git binding lives inside the configuration - # body (`parameters.dataApp.git`), not as a top-level request field — so no extra payload - # keys are expected. - ('python-js', False, {}), - ], -) -async def test_create_data_app_passes_type_and_managed_repo_flag( - app_type: str, - use_managed_git_repo: bool, - expected_extra_payload: dict, -) -> None: - client = DataScienceClient.create('https://api.example.com', token=None, branch_id='br-1') - client.post = AsyncMock(return_value=_create_app_response()) # type: ignore[assignment] - - config = _code_app_config() - _ = await client.create_data_app( - name='Demo', - description='desc', - configuration=config, - app_type=app_type, - use_managed_git_repo=use_managed_git_repo, - ) - - expected_payload = { - 'branchId': 'br-1', - 'name': 'Demo', - 'type': app_type, - 'description': 'desc', - 'config': config.model_dump(exclude_none=True, by_alias=True), - **expected_extra_payload, - } - client.post.assert_awaited_once_with(endpoint='apps', data=expected_payload) - - -@pytest.mark.asyncio -async def test_create_data_app_defaults_to_streamlit_without_managed_repo_flag() -> None: - """Backwards compatibility: a call with no app_type/use_managed_git_repo behaves like before.""" - client = DataScienceClient.create('https://api.example.com', token=None, branch_id='br-1') - client.post = AsyncMock(return_value=_create_app_response()) # type: ignore[assignment] - config = _code_app_config() - - _ = await client.create_data_app(name='Demo', description='desc', configuration=config) - - sent = client.post.await_args.kwargs['data'] - assert sent['type'] == 'streamlit' - assert 'useManagedGitRepo' not in sent - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('config_version', 'mode', 'expected_extra'), - [ - ('42', None, {'configVersion': '42'}), # Streamlit - (None, 'dev', {'mode': 'dev'}), # python-js dev preview - (None, 'production', {'mode': 'production'}), - (None, None, {}), # bare deploy (python-js without explicit mode) - ('5', 'dev', {'configVersion': '5', 'mode': 'dev'}), # both - ], -) -async def test_deploy_data_app_payload_with_mode_and_optional_config_version( - config_version: str | None, - mode: str | None, - expected_extra: dict, -) -> None: - client = DataScienceClient.create('https://api.example.com', token=None) - client.patch = AsyncMock(return_value=_create_app_response()) # type: ignore[assignment] - - _ = await client.deploy_data_app('app-123', config_version, mode=mode) - - expected_payload = { - 'desiredState': 'running', - 'restartIfRunning': True, - 'updateDependencies': False, - **expected_extra, - } - client.patch.assert_awaited_once_with(endpoint='apps/app-123', data=expected_payload) - - -@pytest.mark.parametrize('permissions', ['readWrite', 'readOnly']) -@pytest.mark.asyncio -async def test_create_app_git_credential_posts_to_expected_endpoint(permissions: str) -> None: - client = DataScienceClient.create('https://api.example.com', token=None) - client.post = AsyncMock( # type: ignore[assignment] - return_value={ - 'id': 'cred-1', - 'type': 'http_token', - 'name': '', - 'permissions': permissions, - 'ownerAdminId': 'admin-1', - 'createdAt': '2026-05-13T00:00:00Z', - 'secret': 'one-time-token-xyz', - } - ) - - result = await client.create_app_git_credential('app-123', permissions=permissions) - - assert isinstance(result, CreatedGitCredentialResponse) - assert result.id == 'cred-1' - assert result.type == 'http_token' - assert result.permissions == permissions - assert result.secret == 'one-time-token-xyz' - client.post.assert_awaited_once_with( - endpoint='apps/app-123/git-repo/credentials', - data={'type': 'http_token', 'permissions': permissions}, - ) - - -@pytest.mark.asyncio -async def test_get_app_git_repo_returns_urls() -> None: - client = DataScienceClient.create('https://api.example.com', token=None) - client.get = AsyncMock( # type: ignore[assignment] - return_value={ - 'sshUrl': 'git@managed.repo:org/app.git', - 'httpsUrl': 'https://managed.repo/org/app.git', - 'isManagedGitRepo': True, - } - ) - - result = await client.get_app_git_repo('app-123') - - assert isinstance(result, AppGitRepoResponse) - assert result.ssh_url == 'git@managed.repo:org/app.git' - assert result.https_url == 'https://managed.repo/org/app.git' - assert result.is_managed_git_repo is True - client.get.assert_awaited_once_with(endpoint='apps/app-123/git-repo') - - -def test_code_data_app_config_serializes_to_expected_shape() -> None: - """CodeDataAppConfig must match the data-science API payload exactly (aliased keys).""" - config = _code_app_config() - payload = config.model_dump(exclude_none=True, by_alias=True) - assert payload == { - 'parameters': { - 'autoSuspendAfterSeconds': 900, - 'dataApp': {'slug': 'my-app'}, - }, - 'runtime': {'image': {'version': 'dev-1.0.0'}}, - } - - -@pytest.mark.asyncio -async def test_list_app_runs_gets_expected_endpoint_and_parses_failure_reason() -> None: - client = DataScienceClient.create('https://api.example.com', token=None) - client.get = AsyncMock( # type: ignore[assignment] - return_value=[ - { - 'id': 'run-2', - 'appId': 'app-123', - 'state': 'failed', - 'createdAt': '2026-06-12T10:36:20+00:00', - 'startedAt': None, - 'stoppedAt': '2026-06-12T10:36:21+00:00', - 'startupLogs': None, - 'failureReason': { - 'reason': 'ConfigDecryptionFailed', - 'message': 'failed to decrypt key "#API_KEY"', - }, - 'mode': 'prod', - }, - { - 'id': 'run-1', - 'appId': 'app-123', - 'state': 'finished', - 'createdAt': '2026-06-12T09:00:00+00:00', - 'startedAt': '2026-06-12T09:00:05+00:00', - 'stoppedAt': '2026-06-12T09:30:00+00:00', - 'startupLogs': 'booting\nready', - 'failureReason': None, - 'mode': 'prod', - }, - ] - ) - - runs = await client.list_app_runs('app-123', limit=2) - - client.get.assert_awaited_once_with(endpoint='apps/app-123/runs', params={'limit': 2, 'offset': 0}) - assert len(runs) == 2 - assert all(isinstance(run, AppRunResponse) for run in runs) - assert runs[0].state == 'failed' - assert runs[0].started_at is None - assert runs[0].failure_reason is not None - assert runs[0].failure_reason.reason == 'ConfigDecryptionFailed' - assert runs[0].failure_reason.message == 'failed to decrypt key "#API_KEY"' - assert runs[1].failure_reason is None - assert runs[1].startup_logs == 'booting\nready' diff --git a/tests/clients/test_encryption.py b/tests/clients/test_encryption.py deleted file mode 100644 index 72cd2fe2e..000000000 --- a/tests/clients/test_encryption.py +++ /dev/null @@ -1,96 +0,0 @@ -from typing import Any - -import pytest - -from keboola_mcp_server.clients.encryption import ( - REDACTED_SECRET_VALUE, - contains_plaintext_secrets, - is_encrypted_value, - iter_secret_items, - redact_secrets, -) - - -@pytest.mark.parametrize( - ('value', 'expected'), - [ - ('KBC::ProjectSecure::abcd', True), - ('plaintext', False), - ('', False), - (None, False), - (123, False), - ({'KBC::': 'foo'}, False), - ], -) -def test_is_encrypted_value(value: Any, expected: bool) -> None: - assert is_encrypted_value(value) == expected - - -@pytest.mark.parametrize( - ('value', 'expected'), - [ - # no secrets at all - ({'host': 'db.example.com', 'port': 5432}, []), - # top-level secret - ({'#password': 'secret'}, [('#password', 'secret')]), - # nested in dicts and lists - ( - {'parameters': {'db': {'#password': 'secret'}, 'tables': [{'#api_key': 'key'}]}}, - [('#password', 'secret'), ('#api_key', 'key')], - ), - # non-dict, non-list values - ('just-a-string', []), - (None, []), - # already encrypted values are still yielded (filtering is up to the caller) - ({'#token': 'KBC::ProjectSecure::abcd'}, [('#token', 'KBC::ProjectSecure::abcd')]), - ], -) -def test_iter_secret_items(value: Any, expected: list[tuple[str, Any]]) -> None: - assert list(iter_secret_items(value)) == expected - - -@pytest.mark.parametrize( - ('value', 'expected'), - [ - ({'host': 'db.example.com'}, False), - ({'#password': 'secret'}, True), - ({'#password': 'KBC::ProjectSecure::abcd'}, False), - ({'parameters': {'#password': 'KBC::ProjectSecure::abcd', 'nested': [{'#key': 'plain'}]}}, True), - # a '#'-key holding a non-string value is treated as plaintext (fail-safe) - ({'#config': {'user': 'admin'}}, True), - ({}, False), - (None, False), - ], -) -def test_contains_plaintext_secrets(value: Any, expected: bool) -> None: - assert contains_plaintext_secrets(value) == expected - - -@pytest.mark.parametrize( - ('value', 'expected'), - [ - # plaintext secret is masked - ({'#password': 'secret'}, {'#password': REDACTED_SECRET_VALUE}), - # encrypted secret is kept as-is - ({'#password': 'KBC::ProjectSecure::abcd'}, {'#password': 'KBC::ProjectSecure::abcd'}), - # non-secret values are kept, nested structures are walked - ( - {'db': {'host': 'db.example.com', '#password': 'secret'}, 'list': [{'#api_key': 'key'}, 'foo']}, - { - 'db': {'host': 'db.example.com', '#password': REDACTED_SECRET_VALUE}, - 'list': [{'#api_key': REDACTED_SECRET_VALUE}, 'foo'], - }, - ), - # non-container values pass through - ('just-a-string', 'just-a-string'), - (None, None), - ], -) -def test_redact_secrets(value: Any, expected: Any) -> None: - assert redact_secrets(value) == expected - - -def test_redact_secrets_does_not_mutate_input() -> None: - original = {'db': {'#password': 'secret'}} - redact_secrets(original) - assert original == {'db': {'#password': 'secret'}} diff --git a/tests/clients/test_metastore.py b/tests/clients/test_metastore.py deleted file mode 100644 index db84d03fe..000000000 --- a/tests/clients/test_metastore.py +++ /dev/null @@ -1,298 +0,0 @@ -from __future__ import annotations - -from typing import Any -from unittest.mock import AsyncMock - -import pytest - -from keboola_mcp_server.clients.metastore import MetastoreClient - - -def _jsonapi_object( - name: str, - uuid: str, - object_type: str = 'semantic-model', - revision: int = 1, - deleted_at: str | None = None, - relationships: dict[str, Any] | None = None, - **extra_attrs: Any, -) -> dict: - """Build a single JSON:API resource object (inside the 'data' envelope).""" - return { - 'type': object_type, - 'id': uuid, - 'attributes': {'name': name, **extra_attrs}, - 'meta': { - 'branch': 'main', - 'name': name, - 'revision': revision, - 'schemaVersion': '1.0.0', - 'projectId': 123, - 'organizationId': '456', - 'createdAt': '2026-01-01T00:00:00Z', - 'lastUpdated': '2026-01-01T00:00:00Z', - 'deletedAt': deleted_at, - 'revisionCreatedAt': '2026-01-01T00:00:00Z', - }, - 'relationships': relationships, - } - - -def _list_response(*objects: dict) -> dict: - """Wrap objects in a JSON:API list envelope.""" - return {'data': list(objects)} - - -def _single_response(obj: dict) -> dict: - """Wrap a single object in a JSON:API envelope.""" - return {'data': obj} - - -@pytest.mark.asyncio -async def test_list_objects_returns_meta_objects() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.get = AsyncMock( # type: ignore[assignment] - return_value=_list_response(_jsonapi_object('finance-core', 'u1')), - ) - - result = await client.list_objects('semantic-model') - - assert len(result) == 1 - assert result[0].id == 'u1' - assert result[0].type == 'semantic-model' - assert result[0].attributes['name'] == 'finance-core' - assert result[0].meta is not None - assert result[0].meta.revision == 1 - assert result[0].meta.project_id == 123 - client.raw_client.get.assert_awaited_once_with( # type: ignore[attr-defined] - endpoint='api/v1/repository/semantic-model', - params=None, - ) - - -@pytest.mark.asyncio -async def test_list_objects_with_filter() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.get = AsyncMock( # type: ignore[assignment] - return_value=_list_response(_jsonapi_object('filtered', 'u2')), - ) - - result = await client.list_objects('semantic-model', filter_by='name=filtered') - - assert len(result) == 1 - client.raw_client.get.assert_awaited_once_with( # type: ignore[attr-defined] - endpoint='api/v1/repository/semantic-model', - params={'filter': 'name=filtered'}, - ) - - -@pytest.mark.asyncio -async def test_list_objects_with_limit_offset() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.get = AsyncMock( # type: ignore[assignment] - return_value=_list_response(_jsonapi_object('ds', 'd1', 'semantic-dataset')), - ) - - result = await client.list_objects('semantic-dataset', limit=10, offset=5) - - assert len(result) == 1 - client.raw_client.get.assert_awaited_once_with( # type: ignore[attr-defined] - endpoint='api/v1/repository/semantic-dataset', - params={'limit': 10, 'offset': 5}, - ) - - -@pytest.mark.asyncio -async def test_list_objects_organization_scope() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.get = AsyncMock( # type: ignore[assignment] - return_value=_list_response(_jsonapi_object('org-model', 'u3')), - ) - - result = await client.list_objects('semantic-model', organization_scope=True) - - assert len(result) == 1 - client.raw_client.get.assert_awaited_once_with( # type: ignore[attr-defined] - endpoint='api/v1/repository/semantic-model/organization', - params=None, - ) - - -@pytest.mark.asyncio -async def test_create_object_calls_post_with_branch() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token', branch_id='dev') - client.raw_client.post = AsyncMock( # type: ignore[assignment] - return_value=_single_response(_jsonapi_object('new-metric', 'm1', 'semantic-metric')), - ) - - created = await client.create_object( - 'semantic-metric', - name='new-metric', - data={'name': 'new-metric', 'modelUUID': 'u1', 'sql': 'SUM("amount")'}, - ) - - assert created.id == 'm1' - assert created.attributes['name'] == 'new-metric' - call_args = client.raw_client.post.call_args # type: ignore[attr-defined] - assert call_args.kwargs['data']['branch'] == 'dev' - - -@pytest.mark.asyncio -async def test_create_object_default_branch() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.post = AsyncMock( # type: ignore[assignment] - return_value=_single_response(_jsonapi_object('obj', 'o1')), - ) - - await client.create_object('semantic-model', data={'name': 'obj'}) - - call_args = client.raw_client.post.call_args # type: ignore[attr-defined] - assert 'branch' not in call_args.kwargs['data'] - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('version', 'response', 'expected_endpoint'), - [ - ( - None, - { - 'type': 'object', - 'title': 'semantic-model', - '$schema': 'https://json-schema.org/draft/2020-12/schema', - 'version': '1.0.0', - 'required': ['name', 'sql_dialect'], - 'properties': {'name': {'type': 'string'}}, - }, - 'api/v1/schema/semantic-model', - ), - ( - '1.0.0', - { - 'type': 'object', - 'title': 'semantic-model', - 'version': '1.0.0', - 'properties': {'name': {'type': 'string'}}, - }, - 'api/v1/schema/semantic-model/1.0.0', - ), - ], - ids=['latest', 'versioned'], -) -async def test_get_schema( - version: str | None, - response: dict, - expected_endpoint: str, -) -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.get = AsyncMock(return_value=response) # type: ignore[assignment] - - schema = await client.get_schema('semantic-model', version=version) - - assert isinstance(schema, dict) - assert schema['title'] == 'semantic-model' - client.raw_client.get.assert_awaited_once_with( # type: ignore[attr-defined] - endpoint=expected_endpoint, params=None - ) - - -@pytest.mark.asyncio -async def test_get_object() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - obj = _jsonapi_object('my-model', 'u1', sql_dialect='Snowflake') - client.raw_client.get = AsyncMock( # type: ignore[assignment] - return_value=_single_response(obj), - ) - - result = await client.get_object('semantic-model', 'u1') - - assert result.id == 'u1' - assert result.attributes['name'] == 'my-model' - assert result.attributes['sql_dialect'] == 'Snowflake' - - -@pytest.mark.asyncio -async def test_put_object() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.put = AsyncMock( # type: ignore[assignment] - return_value=_single_response(_jsonapi_object('updated', 'u1', revision=2)), - ) - - result = await client.put_object('semantic-model', 'u1', name='updated', data={'name': 'updated'}) - - assert result.id == 'u1' - assert result.meta is not None - assert result.meta.revision == 2 - - -@pytest.mark.asyncio -async def test_patch_object() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.patch = AsyncMock( # type: ignore[assignment] - return_value=_single_response(_jsonapi_object('patched', 'u1', revision=2)), - ) - - result = await client.patch_object('semantic-model', 'u1', name='patched') - - assert result.id == 'u1' - assert result.attributes['name'] == 'patched' - - -@pytest.mark.asyncio -async def test_list_revisions() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.get = AsyncMock( # type: ignore[assignment] - return_value=_list_response( - _jsonapi_object('m', 'u1', revision=1), - _jsonapi_object('m', 'u1', revision=2), - ), - ) - - result = await client.list_revisions('semantic-model', filter_by='id=u1') - - assert len(result) == 2 - assert result[0].meta is not None - assert result[1].meta is not None - assert result[0].meta.revision == 1 - assert result[1].meta.revision == 2 - - -@pytest.mark.asyncio -async def test_get_revision() -> None: - client = MetastoreClient.create('https://metastore.example.com', token='test-token') - client.raw_client.get = AsyncMock( # type: ignore[assignment] - return_value=_single_response(_jsonapi_object('m', 'u1', revision=3)), - ) - - result = await client.get_revision('semantic-model', 'u1', 3) - - assert result.id == 'u1' - assert result.meta is not None - assert result.meta.revision == 3 - - -def test_model_validate_allows_optional_fields_to_be_missing() -> None: - obj = MetastoreClient._parse_object({'data': {}}) - - assert obj.id is None - assert obj.type is None - assert obj.attributes is None - assert obj.relationships is None - assert obj.meta is None - - -def test_model_validate_maps_deleted_at_and_relationships() -> None: - result = MetastoreClient._parse_object( - _single_response( - _jsonapi_object( - 'my-model', - 'u1', - deleted_at='2026-01-02T00:00:00Z', - relationships={'dataset': {'data': {'type': 'semantic-dataset', 'id': 'd1'}}}, - ) - ) - ) - - assert result.relationships == {'dataset': {'data': {'type': 'semantic-dataset', 'id': 'd1'}}} - assert result.meta is not None - assert result.meta.deleted_at == '2026-01-02T00:00:00Z' diff --git a/tests/clients/test_storage.py b/tests/clients/test_storage.py deleted file mode 100644 index dc3b5be6f..000000000 --- a/tests/clients/test_storage.py +++ /dev/null @@ -1,199 +0,0 @@ -from typing import Any, Awaitable, Callable - -import pytest -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.base import JsonDict, RawKeboolaClient -from keboola_mcp_server.clients.encryption import REDACTED_SECRET_VALUE, EncryptionClient -from keboola_mcp_server.clients.storage import AsyncStorageClient - -WriteCall = Callable[[AsyncStorageClient, dict[str, Any]], Awaitable[JsonDict]] - - -def _create_config(client: AsyncStorageClient, configuration: dict[str, Any]) -> Awaitable[JsonDict]: - return client.configuration_create( - component_id='keboola.ex-test', name='test', description='test', configuration=configuration - ) - - -def _update_config(client: AsyncStorageClient, configuration: dict[str, Any]) -> Awaitable[JsonDict]: - return client.configuration_update( - component_id='keboola.ex-test', - configuration_id='config-1', - configuration=configuration, - change_description='change', - ) - - -def _create_row(client: AsyncStorageClient, configuration: dict[str, Any]) -> Awaitable[JsonDict]: - return client.configuration_row_create( - component_id='keboola.ex-test', config_id='config-1', name='row', description='row', configuration=configuration - ) - - -def _update_row(client: AsyncStorageClient, configuration: dict[str, Any]) -> Awaitable[JsonDict]: - return client.configuration_row_update( - component_id='keboola.ex-test', - config_id='config-1', - configuration_row_id='row-1', - configuration=configuration, - change_description='change', - ) - - -WRITE_CALLS: list[WriteCall] = [_create_config, _update_config, _create_row, _update_row] - - -@pytest.fixture -def raw_client(mocker: MockerFixture) -> RawKeboolaClient: - raw = mocker.AsyncMock(RawKeboolaClient) - raw.post.return_value = {'id': 'config-1', 'version': 1} - raw.put.return_value = {'id': 'config-1', 'version': 2} - # used by project_id() -> GET tokens/verify - raw.get.return_value = {'owner': {'id': 4214}} - return raw - - -@pytest.fixture -def encryption_client(mocker: MockerFixture) -> EncryptionClient: - return mocker.AsyncMock(EncryptionClient) - - -class TestConfigurationWriteEncryption: - """The storage client must encrypt plaintext '#'-prefixed secrets before writing configurations.""" - - @pytest.mark.parametrize('write_call', WRITE_CALLS) - @pytest.mark.asyncio - async def test_plaintext_secrets_are_encrypted_before_save( - self, raw_client: RawKeboolaClient, encryption_client: EncryptionClient, write_call: WriteCall - ) -> None: - plaintext_config = {'parameters': {'user': 'admin', '#password': 'plain-secret'}} - encrypted_config = {'parameters': {'user': 'admin', '#password': 'KBC::ProjectSecure::abcd'}} - encryption_client.encrypt.return_value = encrypted_config - - client = AsyncStorageClient(raw_client=raw_client, encryption_client=encryption_client) - await write_call(client, plaintext_config) - - encryption_client.encrypt.assert_called_once_with( - plaintext_config, component_id='keboola.ex-test', project_id='4214' - ) - # the payload sent to the Storage API must contain the encrypted configuration - http_call = raw_client.post if raw_client.post.called else raw_client.put - sent_payload = http_call.call_args.kwargs['data'] - assert sent_payload['configuration'] == encrypted_config - - @pytest.mark.parametrize( - 'configuration', - [ - {'parameters': {'user': 'admin'}}, # no secrets at all - {'parameters': {'#password': 'KBC::ProjectSecure::abcd'}}, # already encrypted - ], - ) - @pytest.mark.asyncio - async def test_no_plaintext_secrets_skips_encryption( - self, - raw_client: RawKeboolaClient, - encryption_client: EncryptionClient, - configuration: dict[str, Any], - ) -> None: - client = AsyncStorageClient(raw_client=raw_client, encryption_client=encryption_client) - await _create_config(client, configuration) - - encryption_client.encrypt.assert_not_called() - sent_payload = raw_client.post.call_args.kwargs['data'] - assert sent_payload['configuration'] == configuration - - @pytest.mark.asyncio - async def test_fails_closed_without_encryption_client(self, raw_client: RawKeboolaClient) -> None: - client = AsyncStorageClient(raw_client=raw_client, encryption_client=None) - - with pytest.raises(ValueError, match='plaintext secret values'): - await _create_config(client, {'parameters': {'#password': 'plain-secret'}}) - - raw_client.post.assert_not_called() - - @pytest.mark.asyncio - async def test_rejects_redacted_placeholder_values( - self, raw_client: RawKeboolaClient, encryption_client: EncryptionClient - ) -> None: - client = AsyncStorageClient(raw_client=raw_client, encryption_client=encryption_client) - - with pytest.raises(ValueError, match='redacted secret values'): - await _create_config(client, {'parameters': {'#password': REDACTED_SECRET_VALUE}}) - - encryption_client.encrypt.assert_not_called() - raw_client.post.assert_not_called() - - @pytest.mark.asyncio - async def test_encryption_failure_aborts_save( - self, raw_client: RawKeboolaClient, encryption_client: EncryptionClient - ) -> None: - encryption_client.encrypt.side_effect = RuntimeError('encryption service unavailable') - client = AsyncStorageClient(raw_client=raw_client, encryption_client=encryption_client) - - with pytest.raises(RuntimeError, match='encryption service unavailable'): - await _create_config(client, {'parameters': {'#password': 'plain-secret'}}) - - raw_client.post.assert_not_called() - - -class TestSearchEndpoints: - """The storage client must build correct query parameters for the SAPI search endpoints.""" - - @pytest.mark.parametrize( - ('branch_id', 'branch_scope', 'expected_branch_params'), - [ - (None, 'current', {'branchTypes[]': 'production'}), - ('123', 'current', {'branchTypes[]': 'development', 'branchIds[]': '123'}), - (None, 'all', {}), - ('123', 'all', {}), - ], - ids=['default_branch_current', 'dev_branch_current', 'default_branch_all', 'dev_branch_all'], - ) - @pytest.mark.asyncio - async def test_global_search_branch_scope( - self, - raw_client: RawKeboolaClient, - branch_id: str | None, - branch_scope: str, - expected_branch_params: dict[str, Any], - ) -> None: - async def get_side_effect(endpoint: str, params: dict[str, Any] | None = None, **kwargs: Any) -> JsonDict: - if endpoint == 'tokens/verify': - return {'owner': {'id': 4214}} - assert endpoint == 'global-search' - return {'all': 0, 'items': [], 'byType': {}, 'byProject': {}} - - raw_client.get.side_effect = get_side_effect - client = AsyncStorageClient(raw_client=raw_client, branch_id=branch_id) - - await client.global_search('foo', limit=10, offset=5, branch_scope=branch_scope) - - params = raw_client.get.call_args.kwargs['params'] - assert params == {'query': 'foo', 'projectIds[]': ['4214'], 'limit': 10, 'offset': 5, **expected_branch_params} - - @pytest.mark.asyncio - async def test_component_configurations_search_params(self, raw_client: RawKeboolaClient) -> None: - raw_client.get.return_value = [{'id': 'config-1', 'componentId': 'keboola.ex-test'}] - client = AsyncStorageClient(raw_client=raw_client, branch_id='123') - - result = await client.component_configurations_search( - component_id='keboola.ex-test', - metadata_keys=['KBC.configuration.folderName', 'KBC.other'], - ) - - raw_client.get.assert_called_once_with( - endpoint='branch/123/search/component-configurations', - params={ - 'componentId': 'keboola.ex-test', - 'metadataKeys[0]': 'KBC.configuration.folderName', - 'metadataKeys[1]': 'KBC.other', - }, - ) - assert result == [{'id': 'config-1', 'componentId': 'keboola.ex-test'}] - - @pytest.mark.asyncio - async def test_component_configurations_search_requires_filter(self, raw_client: RawKeboolaClient) -> None: - client = AsyncStorageClient(raw_client=raw_client) - assert await client.component_configurations_search() == [] - raw_client.get.assert_not_called() diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index d8084f6d1..000000000 --- a/tests/conftest.py +++ /dev/null @@ -1,85 +0,0 @@ -import pytest -from fastmcp import Context -from mcp.server.session import ServerSession -from mcp.shared.context import RequestContext - -from keboola_mcp_server.clients.ai_service import AIServiceClient -from keboola_mcp_server.clients.base import RawKeboolaClient -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.jobs_queue import JobsQueueClient -from keboola_mcp_server.clients.metastore import MetastoreClient -from keboola_mcp_server.clients.scheduler import SchedulerClient -from keboola_mcp_server.clients.storage import AsyncStorageClient -from keboola_mcp_server.clients.sync_actions import SyncActionsClient -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.mcp import CONVERSATION_ID, ServerState -from keboola_mcp_server.workspace import WorkspaceManager - - -@pytest.fixture -def keboola_client(mocker) -> KeboolaClient: - """Creates mocked `KeboolaClient` instance with mocked sub-clients.""" - client = mocker.AsyncMock(KeboolaClient) - client.storage_api_url = 'https://connection.test.keboola.com' - client.branch_id = None - client.token = 'test-token' - client.bearer_token = None # Default to no bearer token - client.hostname_suffix = 'test.keboola.com' - client.headers = {} - client.with_branch_id = mocker.AsyncMock(return_value=client) - # New per-session flow-schema cache: default to "empty cache" so resolve_flow_schema() - # always exercises the (patched) fetch_component in tests instead of returning a MagicMock. - client.get_cached_flow_schema = mocker.Mock(return_value=None) - client.cache_flow_schema = mocker.Mock() - - # Mock API clients - client.storage_client = mocker.AsyncMock(AsyncStorageClient) - client.storage_client.project_id.return_value = '69420' - client.jobs_queue_client = mocker.AsyncMock(JobsQueueClient) - client.ai_service_client = mocker.AsyncMock(AIServiceClient) - client.scheduler_client = mocker.AsyncMock(SchedulerClient) - client.sync_actions_client = mocker.AsyncMock(SyncActionsClient) - client.metastore_client = mocker.AsyncMock(MetastoreClient) - - # Mock the underlying api_client for async clients if needed for deeper testing - client.storage_client.api_client = mocker.AsyncMock(RawKeboolaClient) - client.jobs_queue_client.api_client = mocker.AsyncMock(RawKeboolaClient) - client.ai_service_client.api_client = mocker.AsyncMock(RawKeboolaClient) - - return client - - -@pytest.fixture -def workspace_manager(mocker) -> WorkspaceManager: - """Creates mocked `WorkspaceManager` instance.""" - return mocker.MagicMock(WorkspaceManager) - - -@pytest.fixture -def empty_context(mocker) -> Context: - """Creates the mocked `mcp.server.fastmcp.Context` instance with the `ServerSession` and empty state.""" - ctx = mocker.MagicMock(Context) - ctx.session = mocker.MagicMock(ServerSession) - ctx.session.state = {} - ctx.session.client_params = None - ctx.session_id = None - ctx.client_id = None - ctx.request_context = mocker.MagicMock(RequestContext) - ctx.request_context.lifespan_context = ServerState(Config(), ServerRuntimeInfo(transport='stdio')) - # `meta` is an instance attribute of RequestContext (set in __init__), not a class attribute, - # so MagicMock(spec=RequestContext) doesn't expose it. Default it to None so tools that read - # the progressToken don't trip AttributeError; individual tests can override. - ctx.request_context.meta = None - return ctx - - -@pytest.fixture -def mcp_context_client( - keboola_client: KeboolaClient, workspace_manager: WorkspaceManager, empty_context: Context -) -> Context: - """Fills the empty_context's state with the `KeboolaClient` and `WorkspaceManager` mocks.""" - client_context = empty_context - client_context.session.state[WorkspaceManager.STATE_KEY] = workspace_manager - client_context.session.state[KeboolaClient.STATE_KEY] = keboola_client - client_context.session.state[CONVERSATION_ID] = 'convo-1234' - return client_context diff --git a/tests/docker/ci.sh b/tests/docker/ci.sh deleted file mode 100755 index f27d0a934..000000000 --- a/tests/docker/ci.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -CONTAINER_NAME="keboola-mcp-server-test-docker" -IMAGE_NAME="keboola/mcp-server:ci" - -cleanup() { - docker stop "$CONTAINER_NAME" >/dev/null 2>&1 || true - docker rm "$CONTAINER_NAME" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -main() { - : "${STORAGE_API_TOKEN:?STORAGE_API_TOKEN is required}" - : "${STORAGE_API_URL:?STORAGE_API_URL is required}" - - # Start container. No --workspace-schema: the smoke test exercises get_buckets, which is a - # Storage-only tool and needs no workspace, so the server boots without one. - echo "Starting container..." - docker run -d \ - --name "$CONTAINER_NAME" \ - -p "8080:8000" \ - "$IMAGE_NAME" \ - --transport http-compat \ - --api-url "$STORAGE_API_URL" \ - --storage-token "$STORAGE_API_TOKEN" \ - --host "0.0.0.0" \ - --port 8000 >/dev/null - - # Give server time to start - sleep 5 - - # Wait and test MCP initialize - echo "Testing MCP initialize..." - for i in $(seq 1 30); do - response=$(curl -s -w "\n%{http_code}" -D "headers.txt" -X POST \ - -H "Content-Type: application/json" \ - -H "Accept: application/json, text/event-stream" \ - -d '{"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "ci-docker-test", "version": "1.0.0"}}}' \ - "http://localhost:8080/mcp" 2>/dev/null) || true - - http_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') - - if [ "$http_code" = "200" ] && [ -n "$body" ]; then - echo "✓ MCP server initialized successfully, session-less mode" - - response=$(curl -s -w "\n%{http_code}" -X POST \ - -H "Content-Type: application/json" \ - -H "Accept: application/json, text/event-stream" \ - -d '{"jsonrpc": "2.0", "method": "notifications/initialized"}' \ - "http://localhost:8080/mcp" 2>/dev/null) || true - - http_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') - - if [ "$http_code" = "202" ]; then - echo "✓ MCP initialization confirmed" - - response=$(curl -s -w "\n%{http_code}" -X POST \ - -H "Content-Type: application/json" \ - -H "Accept: application/json, text/event-stream" \ - -d '{"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "get_buckets", "arguments": {}}}' \ - "http://localhost:8080/mcp" 2>/dev/null) || true - - http_code=$(echo "$response" | tail -n1) - body=$(echo "$response" | sed '$d') - - if [ "$http_code" = "200" ] && [ -n "$body" ]; then - # A successful tool call returns a JSON-RPC result without an error and - # without the MCP tool-level isError flag. - status=$(echo "$body" | grep "^data: " | head -1 | cut -c7- \ - | jq -r 'if .error then "rpc_error" elif .result.isError == true then "tool_error" elif .result then "ok" else "unknown" end' 2>/dev/null || true) - - if [ "$status" = "ok" ]; then - echo "✓ get_buckets tool call succeeded" - exit 0 - else - echo "✗ get_buckets did not return a successful result ($status): $body" - fi - fi - fi - # If tool call didn't succeed, continue outer loop - fi - sleep 1 - done - - echo "✗ Server failed to respond" - docker logs "$CONTAINER_NAME" 2>&1 | tail -10 - exit 1 -} - -main "$@" diff --git a/tests/resources/flow/flow_invalid_1.json b/tests/resources/flow/flow_invalid_1.json deleted file mode 100644 index e752f20e1..000000000 --- a/tests/resources/flow/flow_invalid_1.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "phases": [ - { - "name": "Phase without ID" - } - ], - "tasks": [ - { - "id": "task1", - "name": "A valid task", - "phase": "1", - "task": { - "componentId": "keboola.db-ex-mysql" - } - } - ] - } - \ No newline at end of file diff --git a/tests/resources/flow/flow_invalid_2.json b/tests/resources/flow/flow_invalid_2.json deleted file mode 100644 index ba778caf6..000000000 --- a/tests/resources/flow/flow_invalid_2.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "phases": [ - { - "id": "phase1", - "name": "Valid Phase" - } - ], - "tasks": [ - { - "id": "task1", - "name": "Task without 'task' property", - "phase": "phase1" - } - ] - } - \ No newline at end of file diff --git a/tests/resources/flow/flow_invalid_3.json b/tests/resources/flow/flow_invalid_3.json deleted file mode 100644 index d840ea145..000000000 --- a/tests/resources/flow/flow_invalid_3.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "phases": [ - { - "id": null, - "name": "Phase with null id" - } - ], - "tasks": [ - { - "id": "task1", - "name": "Valid task", - "phase": null, - "task": { - "componentId": "keboola.db-ex-mysql" - } - } - ] - } - \ No newline at end of file diff --git a/tests/resources/flow/flow_invalid_4.json b/tests/resources/flow/flow_invalid_4.json deleted file mode 100644 index 8bfae2402..000000000 --- a/tests/resources/flow/flow_invalid_4.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "phases": [ - { - "id": "phase1", - "name": "Valid Phase" - } - ], - "tasks": [ - { - "id": "task1", - "name": "Task with invalid mode", - "phase": "phase1", - "task": { - "componentId": "keboola.db-ex-mysql", - "mode": "execute" - } - } - ] - } - \ No newline at end of file diff --git a/tests/resources/flow/flow_invalid_5.json b/tests/resources/flow/flow_invalid_5.json deleted file mode 100644 index 3dccab3e3..000000000 --- a/tests/resources/flow/flow_invalid_5.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "phases": [ - { - "id": "phase1", - "name": "Valid Phase" - } - ], - "tasks": [ - { - "id": "task1", - "name": "Task with empty componentId", - "phase": "phase1", - "task": { - "componentId": "" - } - } - ] - } - \ No newline at end of file diff --git a/tests/resources/flow/flow_invalid_6.json b/tests/resources/flow/flow_invalid_6.json deleted file mode 100644 index 75cf453fe..000000000 --- a/tests/resources/flow/flow_invalid_6.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "phases": [ - { - "id": "phase1", - "name": "Phase 1" - }, - { - "id": "phase2", - "name": "Phase 2", - "dependsOn": [42, true] - } - ], - "tasks": [] - } - \ No newline at end of file diff --git a/tests/resources/flow/flow_valid_1.json b/tests/resources/flow/flow_valid_1.json deleted file mode 100644 index 2aa975564..000000000 --- a/tests/resources/flow/flow_valid_1.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "phases": [ - { - "id": "phase-1", - "name": "Extract Data", - "description": "This phase extracts data from the source system.", - "dependsOn": [] - }, - { - "id": "phase-2", - "name": "Transform Data", - "description": "This phase transforms the data.", - "dependsOn": ["phase-1"] - } - ], - "tasks": [ - { - "id": "task-1", - "name": "Extract from MySQL", - "phase": "phase-1", - "enabled": true, - "continueOnFailure": false, - "task": { - "componentId": "keboola.db-ex-mysql", - "configId": "mysql-config-1", - "mode": "run", - "tag": "extract" - } - }, - { - "id": "task-2", - "name": "Transform in Python", - "phase": "phase-2", - "enabled": true, - "continueOnFailure": false, - "task": { - "componentId": "keboola.python-transformation-v2", - "configRowIds": ["row-1", "row-2"], - "mode": "debug", - "previousJobId": "123456" - } - } - ] - } - \ No newline at end of file diff --git a/tests/resources/flow/flow_valid_2.json b/tests/resources/flow/flow_valid_2.json deleted file mode 100644 index 23bc2cee5..000000000 --- a/tests/resources/flow/flow_valid_2.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "phases": [ - { - "id": 1, - "name": "Standalone Phase" - } - ], - "tasks": [ - { - "id": 101, - "name": "Minimal Task with Config Data", - "phase": 1, - "task": { - "componentId": "keboola.generic-runner", - "configData": { - "parameters": { - "url": "https://example.com/data", - "method": "GET" - } - }, - "mode": "run" - } - }, - { - "id": 102, - "name": "Debug Mode Task", - "phase": 1, - "task": { - "componentId": "keboola.test-component", - "mode": "debug" - } - } - ] - } - \ No newline at end of file diff --git a/tests/resources/flow/flow_valid_3.json b/tests/resources/flow/flow_valid_3.json deleted file mode 100644 index 9de89d172..000000000 --- a/tests/resources/flow/flow_valid_3.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "phases": [ - { - "id": "extract", - "name": "Extract Data" - }, - { - "id": "transform", - "name": "Transform Data", - "dependsOn": ["extract"] - }, - { - "id": "load", - "name": "Load Data", - "dependsOn": ["transform"] - } - ], - "tasks": [ - { - "id": "extract-snowflake", - "name": "Extract from Snowflake", - "phase": "extract", - "enabled": false, - "task": { - "componentId": "keboola.db-ex-snowflake", - "configId": "snowflake-config" - } - }, - { - "id": "transform-cleaning", - "name": "Data Cleaning", - "phase": "transform", - "continueOnFailure": true, - "task": { - "componentId": "keboola.python-transformation", - "configId": "cleaning-config", - "configRowIds": ["row-1", "row-2"], - "tag": "cleanup-tag" - } - }, - { - "id": "load-to-bigquery", - "name": "Load to BigQuery", - "phase": "load", - "task": { - "componentId": "keboola.db-wr-bigquery", - "configId": "bq-target", - "previousJobId": "job-1234567890" - } - } - ] - } - \ No newline at end of file diff --git a/tests/resources/parameters/root_parameters_invalid.json b/tests/resources/parameters/root_parameters_invalid.json deleted file mode 100644 index de58abb25..000000000 --- a/tests/resources/parameters/root_parameters_invalid.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "qdrant_settings": { - "url": "http://localhost:666", - "#api_key": "test-api-key" - } - } \ No newline at end of file diff --git a/tests/resources/parameters/root_parameters_schema.json b/tests/resources/parameters/root_parameters_schema.json deleted file mode 100644 index 9bfad1d94..000000000 --- a/tests/resources/parameters/root_parameters_schema.json +++ /dev/null @@ -1,337 +0,0 @@ -{ - "type": "object", - "title": "Embeddings Configuration", - "required": [ - "embedding_settings" - ], - "properties": { - "qdrant_settings": { - "type": "object", - "title": "Qdrant Settings", - "options": { - "dependencies": { - "db_type": "qdrant" - } - }, - "required": [ - "url", - "#api_key" - ], - "properties": { - "url": { - "type": "string", - "title": "URL", - "description": "Qdrant instance URL" - }, - "#api_key": { - "type": "string", - "title": "API Key", - "format": "password" - } - } - }, - "embedding_settings": { - "type": "object", - "title": "Embedding Service Settings", - "required": [ - "provider_type" - ], - "properties": { - "provider_type": { - "enum": [ - "openai", - "azure_openai", - "cohere", - "huggingface_hub", - "google_vertex", - "bedrock" - ], - "type": "string", - "title": "Embedding Provider", - "options": { - "tooltip": "Choose the AI service that will generate embeddings" - }, - "enumNames": [ - "OpenAI", - "Azure OpenAI", - "Cohere", - "HuggingFace Hub", - "Google Vertex AI", - "AWS Bedrock" - ], - "description": "Select the embedding service to use" - }, - "azure_settings": { - "type": "object", - "title": "Azure OpenAI Settings", - "options": { - "dependencies": { - "provider_type": "azure_openai" - } - }, - "required": [ - "deployment_name", - "#api_key", - "azure_endpoint" - ], - "properties": { - "#api_key": { - "type": "string", - "title": "API Key", - "format": "password" - }, - "api_version": { - "type": "string", - "title": "API Version", - "default": "2024-02-01" - }, - "azure_endpoint": { - "type": "string", - "title": "Azure Endpoint", - "options": { - "inputAttributes": { - "placeholder": "https://.openai.azure.com/" - } - }, - "description": "Your Azure OpenAI endpoint URL" - }, - "deployment_name": { - "type": "string", - "title": "Deployment Name", - "description": "Enter your Azure OpenAI deployment name" - } - } - }, - "cohere_settings": { - "type": "object", - "title": "Cohere Settings", - "options": { - "dependencies": { - "provider_type": "cohere" - } - }, - "required": [ - "model", - "#api_key" - ], - "properties": { - "model": { - "enum": [ - "embed-english-v3.0", - "embed-english-light-v3.0", - "embed-multilingual-v3.0", - "embed-multilingual-light-v3.0" - ], - "type": "string", - "title": "Model", - "default": "embed-english-v3.0", - "options": { - "tooltip": "Light models are faster but less accurate" - }, - "description": "Select the Cohere embedding model" - }, - "#api_key": { - "type": "string", - "title": "API Key", - "format": "password" - } - } - }, - "openai_settings": { - "type": "object", - "title": "OpenAI Settings", - "options": { - "dependencies": { - "provider_type": "openai" - } - }, - "required": [ - "model", - "#api_key" - ], - "properties": { - "model": { - "enum": [ - "text-embedding-3-small", - "text-embedding-3-large", - "text-embedding-ada-002" - ], - "type": "string", - "title": "Model", - "default": "text-embedding-3-small", - "options": { - "tooltip": "text-embedding-3-small is recommended for most use cases" - }, - "description": "Select the OpenAI embedding model" - }, - "#api_key": { - "type": "string", - "title": "API Key", - "format": "password" - } - } - }, - "bedrock_settings": { - "type": "object", - "title": "AWS Bedrock Settings", - "options": { - "dependencies": { - "provider_type": "bedrock" - } - }, - "required": [ - "#aws_access_key", - "#aws_secret_key", - "region", - "model_id" - ], - "properties": { - "region": { - "enum": [ - "us-east-1", - "us-west-2", - "ap-southeast-1", - "ap-northeast-1", - "eu-central-1" - ], - "type": "string", - "title": "AWS Region", - "description": "AWS region where Bedrock is available" - }, - "model_id": { - "enum": [ - "amazon.titan-embed-text-v1", - "amazon.titan-embed-g1-text-02", - "cohere.embed-english-v3", - "cohere.embed-multilingual-v3" - ], - "type": "string", - "title": "Model ID", - "default": "amazon.titan-embed-text-v1", - "description": "Bedrock model identifier" - }, - "#aws_access_key": { - "type": "string", - "title": "AWS Access Key", - "format": "password" - }, - "#aws_secret_key": { - "type": "string", - "title": "AWS Secret Key", - "format": "password" - } - } - }, - "huggingface_settings": { - "type": "object", - "title": "HuggingFace Hub Settings", - "options": { - "dependencies": { - "provider_type": "huggingface_hub" - } - }, - "required": [ - "model", - "#api_key" - ], - "properties": { - "model": { - "type": "string", - "title": "Model Name", - "default": "sentence-transformers/all-mpnet-base-v2", - "options": { - "tooltip": "Recommended models: all-mpnet-base-v2, all-MiniLM-L6-v2, bge-large-en-v1.5", - "inputAttributes": { - "placeholder": "sentence-transformers/all-mpnet-base-v2" - } - }, - "description": "Enter the HuggingFace model name" - }, - "#api_key": { - "type": "string", - "title": "API Key", - "format": "password" - }, - "show_progress": { - "type": "boolean", - "title": "Show Progress", - "default": false, - "description": "Whether to show a progress bar during embedding generation" - }, - "normalize_embeddings": { - "type": "boolean", - "title": "Normalize Embeddings", - "default": true, - "description": "Whether to normalize the computed embeddings to unit length" - } - } - }, - "google_vertex_settings": { - "type": "object", - "title": "Google Vertex AI Settings", - "options": { - "dependencies": { - "provider_type": "google_vertex" - } - }, - "required": [ - "#credentials", - "project" - ], - "properties": { - "project": { - "type": "string", - "title": "Project ID", - "description": "Google Cloud project ID" - }, - "location": { - "type": "string", - "title": "Location", - "default": "us-central1", - "description": "Google Cloud region" - }, - "model_name": { - "type": "string", - "title": "Model Name", - "default": "textembedding-gecko@latest", - "description": "Vertex AI model name" - }, - "#credentials": { - "type": "string", - "title": "Service Account JSON", - "format": "password", - "description": "Google Cloud service account credentials JSON" - } - } - } - }, - "propertyOrder": 200 - }, - "test_database_connection": { - "type": "button", - "format": "sync-action", - "options": { - "async": { - "cache": false, - "label": "Test Connection to Vector Store Database", - "action": "testVectorStoreConnection" - }, - "hidden": true - } - }, - "test_embedding_service_connection": { - "type": "button", - "format": "sync-action", - "options": { - "async": { - "cache": false, - "label": "Test Connection to Embedding Service", - "action": "testEmbeddingServiceConnection" - }, - "hidden": true - }, - "propertyOrder": 300 - } - } - } - \ No newline at end of file diff --git a/tests/resources/parameters/root_parameters_valid.json b/tests/resources/parameters/root_parameters_valid.json deleted file mode 100644 index 91ef9f764..000000000 --- a/tests/resources/parameters/root_parameters_valid.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "embedding_settings": { - "provider_type": "openai", - "openai_settings": { - "model": "text-embedding-3-small", - "#api_key": "test-api-key" - } - } - } \ No newline at end of file diff --git a/tests/resources/parameters/row_parameters_invalid.json b/tests/resources/parameters/row_parameters_invalid.json deleted file mode 100644 index 99aead468..000000000 --- a/tests/resources/parameters/row_parameters_invalid.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "_metadata_": { - "table": { - "id": "tbl_123", - "name": "customers", - "columns": ["name", "email", "notes"], - "primaryKey": ["email"] - } - }, - "destination": { - "collection_name": "customer_notes", - "load_type": "incremental_load" - }, - "advanced_options": { - "batch_size": 0, - "enable_chunking": true, - "chunking_settings": { - "chunk_size": 9000, - "chunk_overlap": -10, - "chunk_strategy": "sentence" - } - } - } - \ No newline at end of file diff --git a/tests/resources/parameters/row_parameters_schema.json b/tests/resources/parameters/row_parameters_schema.json deleted file mode 100644 index 7ed61137b..000000000 --- a/tests/resources/parameters/row_parameters_schema.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "type": "object", - "title": "Vector Store Configuration", - "required": [ - "text_column" - ], - "properties": { - "_metadata_": { - "type": "object", - "options": { - "hidden": true - }, - "properties": { - "table": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "columns": { - "type": "array" - }, - "primaryKey": { - "type": "array" - } - } - } - } - }, - "destination": { - "type": "object", - "title": "Destination", - "options": [], - "required": [ - "collection_name" - ], - "properties": { - "load_type": { - "enum": [ - "full_load", - "incremental_load" - ], - "type": "string", - "title": "Load Type", - "format": "checkbox", - "default": "full_load", - "options": { - "enum_titles": [ - "Full Load", - "Incremental Load" - ] - }, - "description": "If Full load is used, the destination table will be overwritten every run. If incremental load is used, data will be upserted into the destination table. Tables with a primary key will have rows updated, tables without a primary key will have rows appended.", - "propertyOrder": 30 - }, - "primary_key": { - "type": "string", - "title": "Primary Key Column", - "watch": { - "columns": "_metadata_.table.columns" - }, - "options": { - "dependencies": { - "load_type": "incremental_load" - } - }, - "required": false, - "enumSource": "columns", - "description": "Choose a column to use as unique identifier for upserts.", - "propertyOrder": 30 - }, - "collection_name": { - "type": "string", - "title": "Collection Name", - "default": "keboola_embeddings", - "propertyOrder": 10 - }, - "metadata_columns": { - "type": "array", - "items": { - "type": "string", - "title": "Column Name", - "watch": { - "columns": "_metadata_.table.columns" - }, - "enumSource": "columns" - }, - "title": "Metadata Columns", - "format": "select", - "options": { - "tags": true - }, - "required": false, - "description": "Choose columns to save to the vector store database as metadata.", - "uniqueItems": true, - "propertyOrder": 20 - } - }, - "propertyOrder": 300 - }, - "text_column": { - "type": "string", - "title": "Embed column name", - "watch": { - "columns": "_metadata_.table.columns" - }, - "required": true, - "enumSource": "columns", - "description": "Choose a column to embed data", - "propertyOrder": 1 - }, - "advanced_options": { - "type": "object", - "title": "Advanced Options", - "properties": { - "batch_size": { - "type": "integer", - "title": "Batch Size", - "default": 100, - "maximum": 1000, - "minimum": 1, - "description": "Number of texts to process in one batch", - "propertyOrder": 20 - }, - "enable_chunking": { - "type": "boolean", - "title": "Enable Text Chunking", - "format": "checkbox", - "default": false, - "description": "Split long texts into smaller chunks before embedding", - "propertyOrder": 30 - }, - "chunking_settings": { - "type": "object", - "title": "Chunking Settings", - "options": { - "dependencies": { - "enable_chunking": true - } - }, - "properties": { - "chunk_size": { - "type": "integer", - "title": "Chunk Size", - "default": 1000, - "maximum": 8000, - "minimum": 100, - "description": "Maximum number of characters in each chunk", - "propertyOrder": 50 - }, - "chunk_overlap": { - "type": "integer", - "title": "Chunk Overlap", - "default": 100, - "maximum": 1000, - "minimum": 0, - "description": "Number of characters to overlap between chunks", - "propertyOrder": 60 - }, - "chunk_strategy": { - "enum": [ - "character", - "sentence", - "word", - "paragraph" - ], - "type": "string", - "title": "Chunking Strategy", - "default": "paragraph", - "options": { - "tooltip": "Paragraph is recommended for most use cases" - }, - "description": "How to split the text into chunks", - "propertyOrder": 70 - } - }, - "propertyOrder": 40 - } - }, - "propertyOrder": 200 - } - } -} \ No newline at end of file diff --git a/tests/resources/parameters/row_parameters_valid.json b/tests/resources/parameters/row_parameters_valid.json deleted file mode 100644 index c8c126db3..000000000 --- a/tests/resources/parameters/row_parameters_valid.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "_metadata_": { - "table": { - "id": "tbl_123", - "name": "customers", - "columns": ["name", "email", "notes"], - "primaryKey": ["email"] - } - }, - "text_column": "notes", - "destination": { - "collection_name": "customer_notes", - "load_type": "incremental_load", - "primary_key": "email", - "metadata_columns": ["name"] - }, - "advanced_options": { - "batch_size": 100, - "enable_chunking": true, - "chunking_settings": { - "chunk_size": 1000, - "chunk_overlap": 100, - "chunk_strategy": "paragraph" - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_invalid_1.json b/tests/resources/storage/storage_invalid_1.json deleted file mode 100644 index 81e2dc23d..000000000 --- a/tests/resources/storage/storage_invalid_1.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "storage": { - "input": { - "tables": [ - { - "destination": "my_table" - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_invalid_2.json b/tests/resources/storage/storage_invalid_2.json deleted file mode 100644 index d3a1811ac..000000000 --- a/tests/resources/storage/storage_invalid_2.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "storage": { - "output": { - "table_files": [ - { - "source": "foo" - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_invalid_3.json b/tests/resources/storage/storage_invalid_3.json deleted file mode 100644 index 599cc5c94..000000000 --- a/tests/resources/storage/storage_invalid_3.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "storage": { - "input": { - "tables": [ - { - "destination": "my_table" - } - ] - }, - "output": { - "tables": [ - { - "source": "component_table" - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_invalid_4.json b/tests/resources/storage/storage_invalid_4.json deleted file mode 100644 index 4e754a858..000000000 --- a/tests/resources/storage/storage_invalid_4.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "storage": { - "output": { - "tables": [ - { - "destination": "out.c-main.data", - "source": "my_table", - "schema": [ - { "name": "id", "data_type": { "base": { "type": "INTEGER" } } } - ], - "columns": ["id", "name"] - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_invalid_5.json b/tests/resources/storage/storage_invalid_5.json deleted file mode 100644 index 58cb22a63..000000000 --- a/tests/resources/storage/storage_invalid_5.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "storage": { - "output": { - "tables": [ - { - "destination": "out.c-main.data" - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_invalid_6.json b/tests/resources/storage/storage_invalid_6.json deleted file mode 100644 index 69d92acae..000000000 --- a/tests/resources/storage/storage_invalid_6.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-main.data", - "where_column": "id", - "where_values": ["1"], - "where_operator": "gt" - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_invalid_7.json b/tests/resources/storage/storage_invalid_7.json deleted file mode 100644 index 6e4198203..000000000 --- a/tests/resources/storage/storage_invalid_7.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "storage": { - "output": { - "files": [ - { - "tags": ["foo"] - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_valid_1.json b/tests/resources/storage/storage_valid_1.json deleted file mode 100644 index 21dd7889a..000000000 --- a/tests/resources/storage/storage_valid_1.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "storage": { - "output": { - "tables": [ - { - "source": "local_table", - "destination": "out.c-main.data", - "delete_where": [ - { - "where_filters": [ - { - "column": "status", - "operator": "eq", - "values_from_set": ["inactive", "deleted"] - } - ] - } - ] - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_valid_2.json b/tests/resources/storage/storage_valid_2.json deleted file mode 100644 index 40f4d008a..000000000 --- a/tests/resources/storage/storage_valid_2.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-main.data", - "destination": "local_table" - } - ] - }, - "output": { - "tables": [ - { - "source": "local_table", - "destination": "out.c-main.data" - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_valid_3.json b/tests/resources/storage/storage_valid_3.json deleted file mode 100644 index 8b1636c9f..000000000 --- a/tests/resources/storage/storage_valid_3.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "storage": { - "input": { - "files": [ - { - "file_ids": ["file-123", "file-456"], - "overwrite": true - } - ] - }, - "output": { - "files": [ - { - "source": "local_file", - "tags": ["exported"] - } - ] - } - } - } \ No newline at end of file diff --git a/tests/resources/storage/storage_valid_4.json b/tests/resources/storage/storage_valid_4.json deleted file mode 100644 index 1b131a055..000000000 --- a/tests/resources/storage/storage_valid_4.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "storage": { - "input": { - "tables": [ - { - "source": "in.c-main.data", - "destination": "local_table", - "where_column": "id", - "where_values": ["1", "2"] - } - ] - }, - "output": { - "tables": [ - { - "source": "local_table", - "destination": "out.c-main.data", - "schema": [ - { - "name": "id", - "data_type": { - "base": { - "type": "INTEGER" - } - } - } - ] - } - ] - } - } -} \ No newline at end of file diff --git a/tests/resources/storage/storage_valid_5.json b/tests/resources/storage/storage_valid_5.json deleted file mode 100644 index f2de6f8fe..000000000 --- a/tests/resources/storage/storage_valid_5.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "storage": { - "output": { - "tables": [ - { - "destination": "in.c-ex-generic-v2.earthquake_events", - "unload_strategy": "direct-grant" - } - ] - } - } -} diff --git a/tests/test_authorization.py b/tests/test_authorization.py deleted file mode 100644 index 882766ad4..000000000 --- a/tests/test_authorization.py +++ /dev/null @@ -1,224 +0,0 @@ -"""Tests for the tool authorization middleware. - -Uses parameterized tests to reduce boilerplate while maintaining comprehensive coverage. -""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from fastmcp import Context -from fastmcp.exceptions import ToolError -from fastmcp.server.middleware import MiddlewareContext -from fastmcp.tools import Tool -from mcp.types import ToolAnnotations - -from keboola_mcp_server.authorization import ToolAuthorizationMiddleware - -# Sample tools: 3 read-only (get_configs, get_buckets, query_data), 2 write (create_config, update_descriptions) -ALL_TOOLS = {'get_configs', 'create_config', 'get_buckets', 'update_descriptions', 'query_data'} -READ_ONLY_TOOLS = {'get_configs', 'get_buckets', 'query_data'} - - -def create_mock_tool(name: str, read_only: bool = False) -> MagicMock: - """Create a mock Tool with the given name and read-only annotation.""" - tool = MagicMock(spec=Tool) - tool.name = name - tool.annotations = MagicMock(spec=ToolAnnotations) - tool.annotations.readOnlyHint = read_only - return tool - - -@pytest.fixture -def middleware(): - return ToolAuthorizationMiddleware() - - -@pytest.fixture -def mock_middleware_context(): - ctx = MagicMock(spec=Context) - middleware_ctx = MagicMock(spec=MiddlewareContext) - middleware_ctx.fastmcp_context = ctx - return middleware_ctx - - -@pytest.fixture -def sample_tools(): - """Create sample tools with proper read-only annotations.""" - return [ - create_mock_tool('get_configs', read_only=True), - create_mock_tool('create_config', read_only=False), - create_mock_tool('get_buckets', read_only=True), - create_mock_tool('update_descriptions', read_only=False), - create_mock_tool('query_data', read_only=True), - ] - - -# Parameterized test for on_list_tools with various header combinations -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('headers', 'expected_tools'), - [ - # No headers - all tools returned - (None, ALL_TOOLS), - ({}, ALL_TOOLS), - # X-Allowed-Tools only - ({'X-Allowed-Tools': 'get_configs, get_buckets'}, {'get_configs', 'get_buckets'}), - # X-Read-Only-Mode only - ({'X-Read-Only-Mode': 'true'}, READ_ONLY_TOOLS), - # X-Disallowed-Tools only - ({'X-Disallowed-Tools': 'create_config, update_descriptions'}, READ_ONLY_TOOLS), - # X-Allowed-Tools + X-Read-Only-Mode (intersection) - ( - {'X-Allowed-Tools': 'get_configs, create_config, get_buckets', 'X-Read-Only-Mode': 'true'}, - {'get_configs', 'get_buckets'}, - ), - # X-Allowed-Tools + X-Disallowed-Tools (disallowed takes precedence) - ( - {'X-Allowed-Tools': 'get_configs, create_config, get_buckets', 'X-Disallowed-Tools': 'create_config'}, - {'get_configs', 'get_buckets'}, - ), - # X-Read-Only-Mode + X-Disallowed-Tools - ({'X-Read-Only-Mode': 'true', 'X-Disallowed-Tools': 'get_configs'}, {'get_buckets', 'query_data'}), - # All three headers - ( - { - 'X-Allowed-Tools': 'get_configs, get_buckets, query_data, create_config', - 'X-Read-Only-Mode': 'true', - 'X-Disallowed-Tools': 'query_data', - }, - {'get_configs', 'get_buckets'}, - ), - # Empty/whitespace headers - treated as no restriction - ({'X-Allowed-Tools': ''}, ALL_TOOLS), - ({'X-Allowed-Tools': ' , , '}, ALL_TOOLS), - ({'X-Disallowed-Tools': ''}, ALL_TOOLS), - # Whitespace handling - ({'X-Allowed-Tools': ' get_configs , get_buckets , '}, {'get_configs', 'get_buckets'}), - ({'X-Disallowed-Tools': ' create_config , update_descriptions , '}, READ_ONLY_TOOLS), - ], - ids=[ - 'no_headers_none', - 'no_headers_empty_dict', - 'allowed_tools_only', - 'read_only_mode_only', - 'disallowed_tools_only', - 'allowed_and_read_only', - 'allowed_and_disallowed', - 'read_only_and_disallowed', - 'all_three_headers', - 'empty_allowed_tools', - 'whitespace_only_allowed_tools', - 'empty_disallowed_tools', - 'allowed_tools_with_whitespace', - 'disallowed_tools_with_whitespace', - ], -) -async def test_on_list_tools(middleware, mock_middleware_context, sample_tools, headers, expected_tools): - """Test on_list_tools with various header combinations.""" - call_next = AsyncMock(return_value=sample_tools) - mock_request = MagicMock() - mock_request.headers = headers if headers else {} - http_request = mock_request if headers is not None else None - - with patch('keboola_mcp_server.authorization.get_http_request_or_none', return_value=http_request): - result = await middleware.on_list_tools(mock_middleware_context, call_next) - - assert {t.name for t in result} == expected_tools - - -# Parameterized test for X-Read-Only-Mode truthy/falsy values -@pytest.mark.asyncio -@pytest.mark.parametrize('header_value', ['true', 'True', 'TRUE', '1', 'yes', 'Yes', 'YES']) -async def test_read_only_mode_truthy_values(middleware, mock_middleware_context, sample_tools, header_value): - """Test that various truthy values enable read-only mode.""" - call_next = AsyncMock(return_value=sample_tools) - mock_request = MagicMock() - mock_request.headers = {'X-Read-Only-Mode': header_value} - - with patch('keboola_mcp_server.authorization.get_http_request_or_none', return_value=mock_request): - result = await middleware.on_list_tools(mock_middleware_context, call_next) - - assert {t.name for t in result} == READ_ONLY_TOOLS - - -@pytest.mark.asyncio -@pytest.mark.parametrize('header_value', ['false', 'False', '0', 'no', '', 'random']) -async def test_read_only_mode_falsy_values(middleware, mock_middleware_context, sample_tools, header_value): - """Test that various falsy values do not enable read-only mode.""" - call_next = AsyncMock(return_value=sample_tools) - mock_request = MagicMock() - mock_request.headers = {'X-Read-Only-Mode': header_value} - - with patch('keboola_mcp_server.authorization.get_http_request_or_none', return_value=mock_request): - result = await middleware.on_list_tools(mock_middleware_context, call_next) - - assert result == sample_tools - - -# Parameterized test for on_call_tool with various header combinations -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('tool_name', 'tool_read_only', 'headers', 'should_allow'), - [ - # No headers - all tools allowed - ('create_config', False, None, True), - ('get_configs', True, None, True), - # X-Allowed-Tools - tool in list - ('get_configs', True, {'X-Allowed-Tools': 'get_configs, get_buckets'}, True), - # X-Allowed-Tools - tool not in list - ('create_config', False, {'X-Allowed-Tools': 'get_configs, get_buckets'}, False), - # X-Read-Only-Mode - read-only tool - ('get_configs', True, {'X-Read-Only-Mode': 'true'}, True), - # X-Read-Only-Mode - write tool - ('create_config', False, {'X-Read-Only-Mode': 'true'}, False), - # X-Disallowed-Tools - tool in list - ('create_config', False, {'X-Disallowed-Tools': 'create_config, update_descriptions'}, False), - # X-Disallowed-Tools - tool not in list - ('get_configs', True, {'X-Disallowed-Tools': 'create_config, update_descriptions'}, True), - # X-Allowed-Tools + X-Read-Only-Mode - tool in allowed but not read-only - ('create_config', False, {'X-Allowed-Tools': 'get_configs, create_config', 'X-Read-Only-Mode': 'true'}, False), - # X-Allowed-Tools + X-Disallowed-Tools - tool in both (disallowed wins) - ( - 'get_configs', - True, - {'X-Allowed-Tools': 'get_configs, get_buckets', 'X-Disallowed-Tools': 'get_configs'}, - False, - ), - ], - ids=[ - 'no_headers_write_tool', - 'no_headers_read_tool', - 'allowed_tool_in_list', - 'allowed_tool_not_in_list', - 'read_only_mode_read_tool', - 'read_only_mode_write_tool', - 'disallowed_tool_in_list', - 'disallowed_tool_not_in_list', - 'allowed_and_read_only_write_tool', - 'allowed_and_disallowed_same_tool', - ], -) -async def test_on_call_tool(middleware, mock_middleware_context, tool_name, tool_read_only, headers, should_allow): - """Test on_call_tool with various header combinations.""" - mock_middleware_context.message = MagicMock() - mock_middleware_context.message.name = tool_name - - mock_tool = create_mock_tool(tool_name, read_only=tool_read_only) - mock_middleware_context.fastmcp_context.fastmcp.get_tool = AsyncMock(return_value=mock_tool) - - mock_request = MagicMock() - mock_request.headers = headers if headers else {} - http_request = mock_request if headers is not None else None - - call_next = AsyncMock(return_value=MagicMock()) - - with patch('keboola_mcp_server.authorization.get_http_request_or_none', return_value=http_request): - if should_allow: - await middleware.on_call_tool(mock_middleware_context, call_next) - call_next.assert_called_once_with(mock_middleware_context) - else: - with pytest.raises(ToolError) as exc_info: - await middleware.on_call_tool(mock_middleware_context, call_next) - assert tool_name in str(exc_info.value) - assert 'not authorized' in str(exc_info.value) - call_next.assert_not_called() diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100644 index a04a68e22..000000000 --- a/tests/test_config.py +++ /dev/null @@ -1,110 +0,0 @@ -import dataclasses -from typing import Mapping - -import pytest - -from keboola_mcp_server.config import Config - - -class TestConfig: - @pytest.mark.parametrize( - ('d', 'expected'), - [ - ( - {'storage_token': 'foo', 'workspace_schema': 'bar'}, - Config(storage_token='foo', workspace_schema='bar'), - ), - ( - {'KBC_STORAGE_TOKEN': 'foo', 'KBC_WORKSPACE_SCHEMA': 'bar'}, - Config(storage_token='foo', workspace_schema='bar'), - ), - ( - {'X-Storage_Token': 'foo', 'KBC_WORKSPACE_SCHEMA': 'bar'}, - Config(storage_token='foo', workspace_schema='bar'), - ), - ( - {'X-StorageApi_Token': 'foo', 'KBC_WORKSPACE_SCHEMA': 'bar'}, - Config(storage_token='foo', workspace_schema='bar'), - ), - ( - {'foo': 'bar', 'storage_api_url': 'http://nowhere'}, - Config(storage_api_url='http://nowhere'), - ), - ( - {'X-Conversation-ID': '1234'}, - Config(conversation_id='1234'), - ), - ], - ) - def test_from_dict(self, d: Mapping[str, str], expected: Config) -> None: - assert Config.from_dict(d) == expected - - @pytest.mark.parametrize( - ('orig', 'd', 'expected'), - [ - ( - Config(), - {'storage_token': 'foo', 'workspace_schema': 'bar'}, - Config(storage_token='foo', workspace_schema='bar'), - ), - ( - Config(), - {'KBC_STORAGE_TOKEN': 'foo', 'KBC_WORKSPACE_SCHEMA': 'bar'}, - Config(storage_token='foo', workspace_schema='bar'), - ), - ( - Config(storage_token='bar'), - {'storage_token': 'foo', 'workspace_schema': 'bar'}, - Config(storage_token='foo', workspace_schema='bar'), - ), - ( - Config(storage_token='bar'), - {'storage_token': None, 'workspace_schema': 'bar'}, - Config(workspace_schema='bar'), - ), - (Config(branch_id='foo'), {'branch-id': ''}, Config()), - (Config(branch_id='foo'), {'branch-id': 'none'}, Config()), - (Config(branch_id='foo'), {'branch-id': 'Null'}, Config()), - (Config(branch_id='foo'), {'branch-id': 'Default'}, Config()), - (Config(branch_id='foo'), {'branch-id': 'pRoDuCtIoN'}, Config()), - ], - ) - def test_replace_by(self, orig: Config, d: Mapping[str, str], expected: Config) -> None: - assert orig.replace_by(d) == expected - - def test_defaults(self) -> None: - config = Config() - for f in dataclasses.fields(Config): - assert getattr(config, f.name) is None, f'Expected default value for {f.name} to be None' - - def test_no_token_password_in_repr(self) -> None: - config = Config(storage_token='foo') - assert str(config) == ( - "Config(storage_api_url=None, storage_token='****', branch_id=None, workspace_schema=None, " - 'oauth_client_id=None, oauth_client_secret=None, ' - 'oauth_server_url=None, oauth_scope=None, mcp_server_url=None, ' - 'jwt_secret=None, bearer_token=None, conversation_id=None)' - ) - - @pytest.mark.parametrize( - ('url', 'expected'), - [ - ('foo.bar', 'https://foo.bar'), - ('ftp://foo.bar', 'https://foo.bar'), - ('foo.bar/v2/storage', 'https://foo.bar'), - ('test:foo.bar/v2/storage', 'https://foo.bar'), - ('https://foo.bar/v2/storage', 'https://foo.bar'), - ('https://foo.bar', 'https://foo.bar'), - ('http://localhost:8000', 'http://localhost:8000'), - ('https://localhost:8000/foo/bar', 'https://localhost:8000'), - ], - ) - def test_url_field(self, url: str, expected: str) -> None: - config = Config( - storage_api_url=url, - oauth_server_url=url, - mcp_server_url=url, - ) - assert config.storage_api_url == expected - assert config.oauth_server_url == expected - assert config.mcp_server_url == expected diff --git a/tests/test_errors.py b/tests/test_errors.py deleted file mode 100644 index 6c7e0ad96..000000000 --- a/tests/test_errors.py +++ /dev/null @@ -1,424 +0,0 @@ -import json -import logging -import uuid -from importlib.metadata import distribution -from unittest.mock import ANY - -import httpx -import jsonschema -import pydantic -import pytest -import yaml -from fastmcp import Client, Context, FastMCP -from fastmcp.exceptions import ToolError -from fastmcp.tools import FunctionTool -from mcp.shared.context import RequestContext -from mcp.types import ClientCapabilities, Implementation, InitializeRequestParams - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.errors import MAX_ARG_VALUE_LEN, tool_errors -from keboola_mcp_server.mcp import ServerState -from keboola_mcp_server.server import create_server -from keboola_mcp_server.tools.storage.tools import TableColumnInfo -from keboola_mcp_server.tools.validation import RecoverableValidationError, ValidationContext - -PYDANTIC_DOCS_VERSION = '.'.join(pydantic.__version__.split('.')[:2]) - - -@pytest.fixture -def function_with_value_error(): - """A function that raises ValueError for testing general error handling.""" - - async def func(_ctx: Context): - raise ValueError('Simulated ValueError') - - return func - - -@pytest.fixture -def function_with_jsonschema_validation_error(): - """A function that raises jsonschema.ValidationError for testing validation wrapping.""" - - async def func(_ctx: Context): - raise jsonschema.ValidationError('Simulated jsonschema validation error') - - return func - - -@pytest.fixture -def function_with_recoverable_jsonschema_validation_error(): - """A function that raises RecoverableValidationError to test rich __str__ propagation.""" - - async def func(_ctx: Context): - try: - jsonschema.validate({'embedding_settings': {'provider_type': 'gpt-9000'}}, {'type': 'string'}) - except jsonschema.ValidationError as e: - raise RecoverableValidationError.create_from_values( - e, - initial_message='The "parameters" field is not valid.', - validation_context=ValidationContext( - component_id='keboola.wr-pinecone-embeddings', - configuration_id='cfg-1', - scope='parameters', - ), - ) - - return func - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('function_fixture', 'default_recovery', 'recovery_instructions', 'expected_recovery_message', 'exception_message'), - [ - # Case with both default_recovery and recovery_instructions specified - ( - 'function_with_value_error', - 'General recovery message.', - {ValueError: 'Check that data has valid types.'}, - 'Check that data has valid types.', - 'Simulated ValueError', - ), - # Case where only default_recovery is provided - ( - 'function_with_value_error', - 'General recovery message.', - {}, - 'General recovery message.', - 'Simulated ValueError', - ), - # Case with only recovery_instructions provided - ( - 'function_with_value_error', - None, - {ValueError: 'Check that data has valid types.'}, - 'Check that data has valid types.', - 'Simulated ValueError', - ), - # Case with no recovery instructions provided - ( - 'function_with_value_error', - None, - {}, - None, - 'Simulated ValueError', - ), - ], -) -async def test_tool_errors( - function_fixture, - default_recovery, - recovery_instructions, - expected_recovery_message, - exception_message, - request, - mcp_context_client: Context, -): - """ - Test that the appropriate recovery message is applied based on the exception type. - Verifies that the tool_errors decorator handles various combinations of recovery parameters. - """ - tool_func = request.getfixturevalue(function_fixture) - decorated_func = tool_errors(default_recovery=default_recovery, recovery_instructions=recovery_instructions)( - tool_func - ) - - if expected_recovery_message is None: - with pytest.raises(ValueError, match=exception_message) as excinfo: - await decorated_func(mcp_context_client) - else: - with pytest.raises(ToolError) as excinfo: - await decorated_func(mcp_context_client) - assert expected_recovery_message in str(excinfo.value) - assert exception_message in str(excinfo.value) - - -@pytest.mark.asyncio -async def test_logging_on_tool_exception(caplog, function_with_value_error, mcp_context_client: Context): - """Test that the tool_errors decorator logs exceptions properly.""" - decorated_func = tool_errors()(function_with_value_error) - - with pytest.raises(ValueError, match='Simulated ValueError'): - await decorated_func(mcp_context_client) - - assert len(caplog.records) == 1 - assert caplog.records[0].levelno == logging.ERROR - assert 'MCP tool "func" call failed. ValueError: Simulated ValueError' in caplog.records[0].message - assert 'Simulated ValueError' in caplog.records[0].message - - -@pytest.mark.asyncio -async def test_jsonschema_validation_error_wrapped( - function_with_jsonschema_validation_error, mcp_context_client: Context -): - decorated_func = tool_errors()(function_with_jsonschema_validation_error) - - with pytest.raises(ToolError) as excinfo: - await decorated_func(mcp_context_client) - - message = str(excinfo.value) - assert 'Simulated jsonschema validation error' in message - - -@pytest.mark.asyncio -async def test_recoverable_jsonschema_validation_error_uses_original_str( - function_with_recoverable_jsonschema_validation_error, mcp_context_client: Context -): - decorated_func = tool_errors()(function_with_recoverable_jsonschema_validation_error) - - with pytest.raises(ToolError) as excinfo: - await decorated_func(mcp_context_client) - - message = str(excinfo.value) - assert 'Failed validating' in message - assert 'The "parameters" field is not valid.' in message - assert 'Validation component context: component_id=keboola.wr-pinecone-embeddings' in message - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('transport', 'client_info', 'component_id'), - [ - ('http', None, 'keboola.mcp-server-tool'), - ('stdio', Implementation(name='read-only-chat', version='1.2.3'), 'keboola.ai-chat'), - ('stdio', Implementation(name='kai-assistant', version='x.y.z'), 'keboola.kai-assistant'), - ], -) -async def test_get_session_id( - transport: str, client_info: Implementation | None, component_id: str, mcp_context_client: Context, mocker -): - @tool_errors() - async def foo(_ctx: Context): - pass - - session_id = uuid.uuid4().hex - if transport == 'stdio': - mcp_context_client.session_id = None - mcp_context_client.request_context = mocker.MagicMock(RequestContext) - mcp_context_client.request_context.lifespan_context = ServerState( - config=Config(), runtime_info=ServerRuntimeInfo(transport='stdio', server_id=session_id) - ) - elif transport == 'http': - mcp_context_client.session_id = session_id - mcp_context_client.request_context.lifespan_context = ServerState( - config=Config(), runtime_info=ServerRuntimeInfo(transport='http', server_id=session_id) - ) - else: - pytest.fail(f'Unknown transport: {transport}') - - if client_info: - mcp_context_client.session.client_params = InitializeRequestParams( - protocolVersion='1.0', - clientInfo=client_info, - capabilities=ClientCapabilities(), - ) - - await foo(mcp_context_client) - client = KeboolaClient.from_state(mcp_context_client.session.state) - client.storage_client.trigger_event.assert_called_once_with( - message='MCP tool "foo" call succeeded.', - component_id=component_id, - event_type='success', - params={ - 'mcpServerContext': { - 'appEnv': 'DEV', - 'version': distribution('keboola_mcp_server').version, - 'userAgent': f'{client_info.name}/{client_info.version}' if client_info else '', - 'sessionId': session_id, - 'serverTransport': transport, - 'conversationId': 'convo-1234', - }, - 'tool': { - 'name': 'foo', - 'arguments': [], - }, - }, - duration=ANY, - ) - - -class TestPydanticValidationErrors: - @pytest.fixture - def mcp_server(self) -> FastMCP: - cfg_dict = { - 'storage_token': '123-test-storage-token', - 'storage_api_url': 'https://connection.keboola.com', - 'transport': 'stdio', - } - config = Config.from_dict(cfg_dict) - server = create_server(config, runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - return server - - @pytest.mark.asyncio - async def test_error_in_tool_call_params(self, mocker, mcp_server: FastMCP): - mocker.patch( - 'keboola_mcp_server.clients.base.KeboolaServiceClient.get', - return_value={'owner': {'id': '123'}}, - ) - - async with Client(mcp_server) as client: - with pytest.raises(ToolError) as excinfo: - await client.call_tool('query_data', arguments={'foo': 'bar'}) - - assert isinstance(excinfo.value, ToolError) - lines = str(excinfo.value).splitlines() - assert len(lines) > 0, 'Empty error message' - assert lines[0] == 'Found 3 validation error(s) for call[query_data]' - formatted = '\n'.join(lines[1:]) - error_details = yaml.safe_load(formatted) - assert error_details == { - 'errors': [ - { - 'field': 'sql_query', - 'message': 'Missing required argument', - 'extra': { - 'type': 'missing_argument', - 'input': "{'foo': 'bar'}", - 'url': f'https://errors.pydantic.dev/{PYDANTIC_DOCS_VERSION}/v/missing_argument', - }, - }, - { - 'field': 'query_name', - 'message': 'Missing required argument', - 'extra': { - 'type': 'missing_argument', - 'input': "{'foo': 'bar'}", - 'url': f'https://errors.pydantic.dev/{PYDANTIC_DOCS_VERSION}/v/missing_argument', - }, - }, - { - 'field': 'foo', - 'message': 'Unexpected keyword argument', - 'extra': { - 'type': 'unexpected_keyword_argument', - 'input': 'bar', - 'url': ( - f'https://errors.pydantic.dev/{PYDANTIC_DOCS_VERSION}/v/unexpected_keyword_argument' - ), - }, - }, - ] - } - - @staticmethod - @tool_errors() - async def foo(_ctx: Context): - # raises PydanticValidationError for missing quoted_name field - TableColumnInfo.model_validate({'name': 'bar', 'database_native_type': 'text', 'nullable': False}) - - @pytest.mark.asyncio - async def test_error_inside_tool_call(self, caplog, mocker, mcp_server: FastMCP): - mocker.patch( - 'keboola_mcp_server.clients.base.KeboolaServiceClient.get', - return_value={'owner': {'id': '123'}}, # response from GET /v2/storage/tokens/verify - ) - post_mock = mocker.patch( - 'keboola_mcp_server.clients.base.KeboolaServiceClient.post', - return_value={ # response from POST /v2/storage/events - 'id': '13008826', - 'uuid': '01958f48-b1fc-7f05-b9b9-8a4a7b385bc3', - }, - ) - - mcp_server.add_tool(FunctionTool.from_function(self.foo)) - - async with Client(mcp_server) as client: - with pytest.raises(ToolError) as excinfo: - await client.call_tool('foo') - - expected_error_details = { - 'errors': [ - { - 'field': 'quotedName', - 'message': 'Field required', - 'extra': { - 'type': 'missing', - 'input': "{'name': 'bar', 'database_native_type': 'text', 'nullable': False}", - 'url': f'https://errors.pydantic.dev/{PYDANTIC_DOCS_VERSION}/v/missing', - }, - }, - ] - } - - # check the message in the ToolError exception - assert isinstance(excinfo.value, ToolError) - lines = str(excinfo.value).splitlines() - assert len(lines) > 0, 'Empty error message' - assert lines[0] == 'Found 1 validation error(s) for TableColumnInfo' - assert expected_error_details == yaml.safe_load('\n'.join(lines[1:])) - - # check the message in the LOG from 'keboola_mcp_server.errors' logger - log_records = [r for r in caplog.records if r.name == 'keboola_mcp_server.errors'] - assert log_records, 'No log records from keboola_mcp_server.errors' - lines = log_records[0].message.splitlines() - assert len(lines) > 0, 'Empty log message' - assert lines[0] == 'MCP tool "foo" call failed. ToolError: Found 1 validation error(s) for TableColumnInfo' - assert expected_error_details == yaml.safe_load('\n'.join(lines[1:])) - - # check the message in the submitted SAPI event - post_mock.assert_called_once() - _, kwargs = post_mock.call_args - lines = str(kwargs.get('data', {}).get('message') or '').splitlines() - assert len(lines) > 0, 'Empty error message' - assert lines[0] == 'MCP tool "foo" call failed. ToolError: Found 1 validation error(s) for TableColumnInfo' - assert expected_error_details == yaml.safe_load('\n'.join(lines[1:])) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - 'event_error', - [ - httpx.HTTPStatusError( - '400 Request too large', - request=httpx.Request('POST', 'https://example.com/events'), - response=httpx.Response(400), - ), - httpx.HTTPStatusError( - '403 Forbidden', - request=httpx.Request('POST', 'https://example.com/events'), - response=httpx.Response(403), - ), - ConnectionError('Network failure'), - ], -) -async def test_event_logging_failure_does_not_fail_tool(caplog, event_error, mcp_context_client: Context): - """Event logging errors must never propagate as tool failures — the tool result is already determined.""" - - @tool_errors() - async def successful_tool(_ctx: Context) -> str: - return 'ok' - - client = KeboolaClient.from_state(mcp_context_client.session.state) - client.storage_client.trigger_event.side_effect = event_error - - # Tool must succeed despite event logging failure - result = await successful_tool(mcp_context_client) - assert result == 'ok' - - # Event failure must be logged as a warning, not re-raised - warning_records = [r for r in caplog.records if r.levelno == logging.WARNING] - assert any('Failed to trigger tool event' in r.message for r in warning_records) - - -@pytest.mark.asyncio -async def test_large_argument_value_is_truncated_in_event(mcp_context_client: Context): - """Argument values exceeding MAX_ARG_VALUE_LEN must be replaced with a truncation notice.""" - - large_value = 'x' * (MAX_ARG_VALUE_LEN + 1) - - @tool_errors() - async def tool_with_large_arg(_ctx: Context, big_param: str) -> str: - return 'done' - - client = KeboolaClient.from_state(mcp_context_client.session.state) - await tool_with_large_arg(mcp_context_client, big_param=large_value) - - client.storage_client.trigger_event.assert_called_once() - _, kwargs = client.storage_client.trigger_event.call_args - arguments = kwargs['params']['tool']['arguments'] - big_param_entry = next(a for a in arguments if a['key'] == 'big_param') - decoded = json.loads(big_param_entry['value']) - assert 'truncated' in decoded - expected_length = len(json.dumps(json.dumps(large_value, ensure_ascii=False), ensure_ascii=False).encode('utf-8')) - assert str(expected_length) in decoded diff --git a/tests/test_mcp.py b/tests/test_mcp.py deleted file mode 100644 index f16d6d57b..000000000 --- a/tests/test_mcp.py +++ /dev/null @@ -1,720 +0,0 @@ -import asyncio -from datetime import datetime, timedelta, timezone -from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from fastmcp import Context -from fastmcp.exceptions import ToolError -from pydantic import BaseModel, Field - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.mcp import ( - AggregateError, - ServerState, - SessionStateMiddleware, - ToolsFilteringMiddleware, - _exclude_none_serializer, - _filter_toon_nulls, - process_concurrently, - toon_serializer, - unwrap_results, -) - - -class SimpleModel(BaseModel): - field1: str | None = None - field2: int | None = Field(default=None, serialization_alias='field2_alias') - field3: datetime | None = None - - -class NestedModel(BaseModel): - field1: str | None = None - field2: list[str] | None = None - - -def _tool(name: str, read_only: bool = False, tags: set[str] | None = None) -> MagicMock: - tool = MagicMock() - tool.name = name - tool.tags = tags or set() - if read_only: - tool.annotations.readOnlyHint = True - else: - tool.annotations = None - return tool - - -async def _async_square(n: int) -> int: - """Simple async function that squares a number after a short delay.""" - await asyncio.sleep(0.01) # Simulate some async work - return n * n - - -async def _async_fail(n: int) -> None: - """Simple async function that always raises an exception.""" - await asyncio.sleep(0.01) - raise ValueError(f'Failed for {n}') - - -async def _async_square_or_fail(n: int) -> int: - """Async function that squares even numbers and fails for odd numbers.""" - await asyncio.sleep(0.01) - if n % 2 == 0: - return n * n - else: - raise ValueError(f'Failed for odd number {n}') - - -@pytest.mark.parametrize( - ('data', 'expected'), - [ - (None, ''), - # Exclude none values from a single model - (SimpleModel(field1='value1'), '{"field1":"value1"}'), - # Exclude none values from a list of models - ( - [SimpleModel(field1='value1', field2=None), SimpleModel(field2=123)], - '[{"field1":"value1"},{"field2":123}]', - ), - # Exclude none values from a dictionary with models - ( - {'key1': SimpleModel(field1='value1'), 'key2': None, 'key3': SimpleModel(field2=456)}, - '{"key1":{"field1":"value1"},"key3":{"field2":456}}', - ), - # Exclude none values from primitives - ({'key1': 123, 'key2': None, 'key3': 'value'}, '{"key1":123,"key3":"value"}'), - # Exclude none values with nested structures - ( - {'key1': [SimpleModel(field1='value1'), None], 'key2': {'nested_key': SimpleModel(field2=789)}}, - '{"key1":[{"field1":"value1"}],"key2":{"nested_key":{"field2":789}}}', - ), - ( - { - 'key1': [ - SimpleModel(field3=datetime(2025, 2, 3, 10, 11, 12, tzinfo=timezone(timedelta(hours=2)))), - None, - ], - 'key2': {'nested_key': SimpleModel(field2=789)}, - 'key3': datetime(2025, 1, 1, 1, 2, 3), - }, - '{"key1":[{"field3":"2025-02-03T10:11:12+02:00"}],' - '"key2":{"nested_key":{"field2":789}},' - '"key3":"2025-01-01T01:02:03"}', - ), - ], -) -def test_exclude_none_serializer(data, expected): - result = _exclude_none_serializer(data) - assert result == expected - - -@pytest.mark.parametrize( - ('data', 'expected'), - [ - # Top-level None - (None, 'null'), - # Empty dict - ({}, ''), - # Empty list - ([], '[0]:'), - # Empty tuple - ((), '[0]:'), - # Empty set - (set(), '[0]:'), - # Datetime - ( - datetime(2025, 1, 1), - '"2025-01-01T00:00:00"', - ), - # Simple dictionary - ( - {'key': 'value', 'none_key': None}, - 'key: value\nnone_key: null', - ), - # List - ( - ['item1', 'item2'], - '[2]: item1,item2', - ), - # Mixed types in a list - ( - ['a', 1, True, None], - '[4]: a,1,true,null', - ), - # Tuple - ( - (1, 2, 3), - '[3]: 1,2,3', - ), - # Nested dictionary - ( - {'a': {'b': 1}}, - 'a:\n b: 1', - ), - # Deeply nested None - ( - {'a': {'b': None}}, - 'a:\n b: null', - ), - # Model with some None values - toon_serializer includes None and does NOT use aliases - ( - SimpleModel(field1='value1', field2=123), - 'field1: value1\nfield2: 123\nfield3: null', - ), - # Simple model (only has primitive fields) in a list - ( - [SimpleModel(field1='value1', field2=123), SimpleModel(field1='value2', field2=456)], - '[2]{field1,field2,field3}:\n value1,123,null\n value2,456,null', - ), - # Nested model (has a list field) in a list - this disables the tabular view - ( - [ - NestedModel(field1='value1', field2=['item1', 'item2']), - NestedModel(field1='value2', field2=['item3', 'item4']), - ], - '[2]:\n' - ' - field1: value1\n' - ' field2[2]: item1,item2\n' - ' - field1: value2\n' - ' field2[2]: item3,item4', - ), - # Complex structure with models, lists, dicts, and None - ( - { - 'users': [ - {'name': 'Alice', 'active': True}, - {'name': 'Bob', 'active': None}, - ], - 'meta': SimpleModel(field1='test'), - }, - 'users[2]{name,active}:\n' - ' Alice,true\n' - ' Bob,null\n' - 'meta:\n' - ' field1: test\n' - ' field2: null\n' - ' field3: null', - ), - ], -) -def test_toon_serializer(data, expected): - result = toon_serializer(data) - assert result == expected - - -def test_filter_toon_nulls_single_item_list() -> None: - data = [{'a': 1, 'b': None, 'c': {'d': None, 'e': 2}}] - assert _filter_toon_nulls(data) == [{'a': 1, 'c': {'e': 2}}] - - -def test_filter_toon_nulls_multi_item_list_preserves_alignment() -> None: - data = [{'a': 1, 'b': None}, {'a': None, 'b': 2}] - assert _filter_toon_nulls(data) == [{'a': 1, 'b': None}, {'a': None, 'b': 2}] - - -def test_filter_toon_nulls_multi_item_list_preserves_key_order() -> None: - data = [ - { - 'b': 1, - 'd': None, - 'a': None, - }, - {'a': 2, 'b': None, 'c': 3, 'd': None, 'e': None}, - ] - result = _filter_toon_nulls(data) - assert result == [{'b': 1, 'a': None, 'c': None}, {'b': None, 'a': 2, 'c': 3}] - assert list(result[0].keys()) == ['b', 'a', 'c'] - - -@pytest.mark.parametrize( - ('data', 'expected'), - [ - ({}, {}), - ([], []), - (['a', None, 1], ['a', None, 1]), - ({'a': None, 'b': 2}, {'b': 2}), - ({'a': {'b': None, 'c': 3}}, {'a': {'c': 3}}), - ([{'a': None, 'b': None}, {'a': 1, 'b': None}], [{'a': None}, {'a': 1}]), - ([{'a': None}, {'b': None}], [{}, {}]), - ([{'a': {'b': None}, 'c': 1}], [{'a': {}, 'c': 1}]), - # Test that _filter_toon_nulls applies recursively to lists nested inside dicts - ( - [ - {'a': 1, 'b': [None, 2, 3]}, - {'a': None, 'b': [4, None]}, - ], - [ - {'a': 1, 'b': [None, 2, 3]}, - {'a': None, 'b': [4, None]}, - ], - ), - # Test with deeper nesting for key 'b' - ( - [ - {'a': 1, 'b': [{'c': None, 'd': 2}, {'c': None, 'd': None}]}, - {'a': 2, 'b': [{'c': None, 'd': None}, {'c': None, 'd': None}]}, - ], - [ - {'a': 1, 'b': [{'d': 2}, {'d': None}]}, - {'a': 2, 'b': [{}, {}]}, - ], - ), - ], -) -def test_filter_toon_nulls_edge_cases(data, expected) -> None: - assert _filter_toon_nulls(data) == expected - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('items', 'afunc', 'max_concurrency', 'expected_successes', 'expected_exceptions'), - [ - # All succeed - (list(range(5)), _async_square, 2, [0, 1, 4, 9, 16], []), - # Mixed success and failure (odd numbers fail) - (list(range(5)), _async_square_or_fail, 3, [0, 4, 16], ['Failed for odd number 1', 'Failed for odd number 3']), - # All fail - (list(range(3)), _async_fail, 2, [], ['Failed for 0', 'Failed for 1', 'Failed for 2']), - # Empty input - ([], _async_square, 5, [], []), - ], - ids=['all_succeed', 'mixed_success_failure', 'all_fail', 'empty_input'], -) -async def test_process_concurrently(items, afunc, max_concurrency, expected_successes, expected_exceptions): - """Test process_concurrently with various scenarios.""" - results = await process_concurrently(items, afunc, max_concurrency=max_concurrency) - - assert len(results) == len(items) - - successes = sorted([r for r in results if not isinstance(r, BaseException)]) - exceptions = [str(e) for e in results if isinstance(e, BaseException)] - - assert successes == expected_successes - assert exceptions == expected_exceptions - - -@pytest.mark.asyncio -async def test_process_concurrently_respects_max_concurrency(): - """Test that max_concurrency limits simultaneous executions.""" - max_concurrency = 3 - current_running = 0 - peak_running = 0 - lock = asyncio.Lock() - - async def track_concurrency(n: int) -> int: - nonlocal current_running, peak_running - async with lock: - current_running += 1 - peak_running = max(peak_running, current_running) - try: - await asyncio.sleep(0.01) - return n * n - finally: - async with lock: - current_running -= 1 - - results = await process_concurrently(list(range(10)), track_concurrency, max_concurrency=max_concurrency) - - assert sorted(results) == [i * i for i in range(10)] - assert peak_running <= max_concurrency - - -@pytest.mark.asyncio -@pytest.mark.parametrize('max_concurrency', [0, -1, -10]) -async def test_process_concurrently_invalid_max_concurrency(max_concurrency): - """Test that process_concurrently raises ValueError for invalid max_concurrency.""" - with pytest.raises(ValueError, match='max_concurrency must be a positive integer'): - await process_concurrently([1, 2, 3], _async_square, max_concurrency=max_concurrency) - - -@pytest.mark.parametrize( - ('results', 'expected'), - [ - # All successes - ([1, 2, 3], [1, 2, 3]), - # Empty list - ([], []), - # Single success - (['value'], ['value']), - ], - ids=['all_successes', 'empty', 'single_success'], -) -def test_unwrap_results_success(results, expected): - """Test unwrap_results returns successes when no exceptions present.""" - assert unwrap_results(results) == expected - - -def test_unwrap_results_raises_aggregate_error(): - """Test unwrap_results raises AggregateError when exceptions are present.""" - exc1 = ValueError('error 1') - exc2 = RuntimeError('error 2') - results: list[int | BaseException] = [1, exc1, 2, exc2, 3] - - with pytest.raises(AggregateError) as exc_info: - unwrap_results(results, 'Test errors') - - err = exc_info.value - assert err.message == 'Test errors' - assert err.exceptions == [exc1, exc2] - assert str(err) == 'Test errors (2 errors): ValueError: error 1; RuntimeError: error 2' - - -def test_unwrap_results_all_exceptions(): - """Test unwrap_results when all results are exceptions.""" - exc1 = ValueError('error 1') - exc2 = ValueError('error 2') - results: list[int | BaseException] = [exc1, exc2] - - with pytest.raises(AggregateError) as exc_info: - unwrap_results(results) - - err = exc_info.value - assert err.exceptions == [exc1, exc2] - assert str(err) == 'Multiple errors occurred (2 errors): ValueError: error 1; ValueError: error 2' - - -class TestToolsFilteringMiddleware: - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('branch_id', 'expect_filtered'), - [ - ('1234', True), - (None, False), - ], - ) - async def test_list_tools_filters_data_apps_by_branch( - self, - mcp_context_client, - branch_id: str | None, - expect_filtered: bool, - ) -> None: - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = branch_id - keboola_client.storage_client.verify_token = AsyncMock(return_value={'owner': {'features': []}, 'admin': {}}) - - data_app_tools = [ - 'modify_streamlit_data_app', - 'get_data_apps', - 'deploy_data_app', - 'delete_python_js_data_app_draft', - ] - tools = [_tool(name) for name in data_app_tools] + [_tool('other_tool')] - - async def call_next(_): - return tools - - middleware = ToolsFilteringMiddleware() - context = SimpleNamespace(fastmcp_context=mcp_context_client) - result = await middleware.on_list_tools(context, call_next) - - result_names = {t.name for t in result} - for name in data_app_tools: - if expect_filtered: - assert name not in result_names - else: - assert name in result_names - assert 'other_tool' in result_names - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('token_role', 'bearer_token', 'hidden_tools', 'visible_tools'), - [ - ('admin', None, {'update_flow'}, {'modify_flow', 'read_only_tool'}), - ('share', None, {'update_flow'}, {'modify_flow', 'read_only_tool'}), - ('', None, {'modify_flow'}, {'update_flow', 'read_only_tool'}), - ('readOnly', None, {'modify_flow', 'update_flow'}, {'read_only_tool'}), - ('guest', None, {'modify_flow'}, {'update_flow', 'read_only_tool'}), - # OAuth users: regular/guest users get modify_flow access (different from SAPI) - ('', 'oauth_token', {'update_flow'}, {'modify_flow', 'read_only_tool'}), - # Empty bearer token behaves the same as no bearer token (SAPI regular) - ('', '', {'modify_flow'}, {'update_flow', 'read_only_tool'}), - ], - ) - async def test_list_tools_filters_flow_tools_by_role( - self, - mcp_context_client, - keboola_client, - token_role: str, - bearer_token: str | None, - hidden_tools: set[str], - visible_tools: set[str], - ) -> None: - keboola_client.bearer_token = bearer_token - keboola_client.storage_client.verify_token = AsyncMock( - return_value={'owner': {'features': []}, 'admin': {'role': token_role}} - ) - - tools = [ - _tool('modify_flow'), - _tool('update_flow'), - _tool('other_tool'), - _tool('read_only_tool', read_only=True), - ] - - async def call_next(_): - return tools - - middleware = ToolsFilteringMiddleware() - context = SimpleNamespace(fastmcp_context=mcp_context_client) - result = await middleware.on_list_tools(context, call_next) - - result_names = {t.name for t in result} - for tool_name in hidden_tools: - assert tool_name not in result_names - for tool_name in visible_tools: - assert tool_name in result_names - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('token_role', 'bearer_token', 'called_tool', 'tool_read_only', 'expect_error'), - [ - ('admin', None, 'modify_flow', False, False), - ('admin', None, 'update_flow', False, True), - ('share', None, 'modify_flow', False, False), - ('share', None, 'update_flow', False, True), - ('', None, 'modify_flow', False, True), - ('', None, 'update_flow', False, False), - ('guest', None, 'write_tool', False, False), - ('guest', None, 'read_only_tool', True, False), - ('readOnly', None, 'write_tool', False, True), - ('readOnly', None, 'read_only_tool', True, False), - # OAuth users: regular users can call modify_flow (different from SAPI regular) - ('', 'oauth_token', 'modify_flow', False, False), - ('', 'oauth_token', 'update_flow', False, True), - # Empty bearer token behaves the same as no bearer token (SAPI regular) - ('', '', 'modify_flow', False, True), - ], - ) - async def test_call_tool_blocks_flow_tools_by_role( - self, - mcp_context_client, - keboola_client, - token_role: str, - bearer_token: str | None, - called_tool: str, - tool_read_only: bool, - expect_error: bool, - ) -> None: - keboola_client.bearer_token = bearer_token - keboola_client.storage_client.verify_token = AsyncMock( - return_value={'owner': {'features': []}, 'admin': {'role': token_role}} - ) - - tool = _tool(called_tool, read_only=tool_read_only) - mcp_context_client.fastmcp = SimpleNamespace(get_tool=AsyncMock(return_value=tool)) - context = SimpleNamespace(fastmcp_context=mcp_context_client, message=SimpleNamespace(name=called_tool)) - - expected = MagicMock() - - async def call_next(_): - return expected - - middleware = ToolsFilteringMiddleware() - if expect_error: - with pytest.raises(ToolError): - await middleware.on_call_tool(context, call_next) - else: - result = await middleware.on_call_tool(context, call_next) - assert result is expected - - @pytest.mark.asyncio - @pytest.mark.parametrize( - 'tool_name', - [ - 'modify_streamlit_data_app', - 'delete_python_js_data_app_draft', - ], - ) - @pytest.mark.parametrize( - ('branch_id', 'expect_error'), - [ - ('5678', True), - (None, False), - ], - ) - async def test_call_tool_blocks_data_apps_by_branch( - self, - mcp_context_client, - branch_id: str | None, - expect_error: bool, - tool_name: str, - ) -> None: - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = branch_id - keboola_client.storage_client.verify_token = AsyncMock(return_value={'owner': {'features': []}, 'admin': {}}) - - tool = _tool(tool_name) - mcp_context_client.fastmcp = SimpleNamespace(get_tool=AsyncMock(return_value=tool)) - context = SimpleNamespace(fastmcp_context=mcp_context_client, message=SimpleNamespace(name=tool_name)) - - expected = MagicMock() - - async def call_next(_): - return expected - - middleware = ToolsFilteringMiddleware() - if expect_error: - with pytest.raises(ToolError, match='Data apps are supported only in the main production branch'): - await middleware.on_call_tool(context, call_next) - else: - result = await middleware.on_call_tool(context, call_next) - assert result is expected - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('features', 'tool_name', 'expect_filtered'), - [ - ([], 'search_semantic_context', True), - ([], 'get_semantic_schema', True), - (['mcp-semantic-tooling'], 'search_semantic_context', False), - (['mcp-semantic-tooling'], 'get_semantic_schema', False), - (['other-feature'], 'search_semantic_context', True), - (['other-feature'], 'get_semantic_schema', True), - ], - ids=[ - 'no_feature_search', - 'no_feature_schema', - 'with_feature_search', - 'with_feature_schema', - 'unrelated_feature_search', - 'unrelated_feature_schema', - ], - ) - async def test_list_tools_filters_semantic_tools_by_feature( - self, - mcp_context_client, - features: list[str], - tool_name: str, - expect_filtered: bool, - ) -> None: - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.verify_token = AsyncMock( - return_value={'owner': {'features': features}, 'admin': {}} - ) - - tools = [ - _tool('search_semantic_context', tags={'semantic'}), - _tool('get_semantic_context', tags={'semantic'}), - _tool('get_semantic_schema', tags={'semantic'}), - _tool('validate_semantic_query', tags={'semantic'}), - _tool('other_tool'), - ] - - async def call_next(_): - return tools - - middleware = ToolsFilteringMiddleware() - context = SimpleNamespace(fastmcp_context=mcp_context_client) - result = await middleware.on_list_tools(context, call_next) - - result_names = {t.name for t in result} - if expect_filtered: - assert tool_name not in result_names - else: - assert tool_name in result_names - assert 'other_tool' in result_names - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('features', 'tool_name', 'tool_tags', 'expect_error'), - [ - ([], 'search_semantic_context', {'semantic'}, True), - ([], 'get_semantic_schema', {'semantic'}, True), - (['mcp-semantic-tooling'], 'search_semantic_context', {'semantic'}, False), - (['mcp-semantic-tooling'], 'get_semantic_schema', {'semantic'}, False), - ([], 'other_tool', set(), False), - ], - ids=[ - 'no_feature_search_tool', - 'no_feature_schema_tool', - 'with_feature_search_tool', - 'with_feature_schema_tool', - 'no_feature_non_semantic_tool', - ], - ) - async def test_call_tool_blocks_semantic_tools_by_feature( - self, - mcp_context_client, - features: list[str], - tool_name: str, - tool_tags: set[str], - expect_error: bool, - ) -> None: - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.verify_token = AsyncMock( - return_value={'owner': {'features': features}, 'admin': {}} - ) - - tool = _tool(tool_name, tags=tool_tags) - mcp_context_client.fastmcp = SimpleNamespace(get_tool=AsyncMock(return_value=tool)) - context = SimpleNamespace(fastmcp_context=mcp_context_client, message=SimpleNamespace(name=tool_name)) - - expected = MagicMock() - - async def call_next(_): - return expected - - middleware = ToolsFilteringMiddleware() - if expect_error: - with pytest.raises(ToolError, match='Semantic Layer Tooling'): - await middleware.on_call_tool(context, call_next) - else: - result = await middleware.on_call_tool(context, call_next) - assert result is expected - - -class TestSessionStateMiddleware: - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('method', 'expected_branch_id'), - [ - ('tools/list', None), - ('resources/list', None), - ('prompts/list', None), - ('tools/call', '999'), - ('resources/read', '999'), - ], - ids=['tools_list', 'resources_list', 'prompts_list', 'tools_call', 'resources_read'], - ) - async def test_on_request_branch_handling(self, method: str, expected_branch_id: str | None): - config = Config( - storage_api_url='https://connection.test.keboola.com', - storage_token='test-token', - branch_id='999', - ) - runtime_info = ServerRuntimeInfo(transport='stdio') - server_state = ServerState(config=config, runtime_info=runtime_info) - - # Use a non-MagicMock session so the middleware enters the branch-handling code path - session = SimpleNamespace(state={}) - - # ctx must pass isinstance(ctx, Context) check, so we use MagicMock(spec=Context). - # However ctx.session must NOT be a MagicMock (line 146 guard), so we override it. - ctx = MagicMock(spec=Context) - ctx.session = session - ctx.request_context.lifespan_context = server_state - - context = SimpleNamespace(method=method, fastmcp_context=ctx) - expected_result = object() - - async def call_next(_): - return expected_result - - captured_configs: list[Config] = [] - - async def fake_create_session_state(cfg, _runtime_info, readonly=None): - captured_configs.append(cfg) - return {'fake': 'state'} - - middleware = SessionStateMiddleware() - - with ( - patch.object(middleware, 'create_session_state', side_effect=fake_create_session_state), - patch('keboola_mcp_server.mcp.get_http_request_or_none', return_value=None), - ): - result = await middleware.on_request(context, call_next) - - assert result is expected_result - assert len(captured_configs) == 1 - assert captured_configs[0].branch_id == expected_branch_id diff --git a/tests/test_oauth.py b/tests/test_oauth.py deleted file mode 100644 index c42bc393e..000000000 --- a/tests/test_oauth.py +++ /dev/null @@ -1,219 +0,0 @@ -import time -from typing import Any, Mapping - -import pytest -from mcp.server.auth.provider import AccessToken, RefreshToken -from mcp.shared.auth import InvalidRedirectUriError, OAuthClientInformationFull -from pydantic import AnyHttpUrl, AnyUrl - -from keboola_mcp_server.oauth import SimpleOAuthProvider, _ExtendedAuthorizationCode, _OAuthClientInformationFull - -JWT_KEY = 'secret' - - -class TestSimpleOAuthProvider: - - @pytest.fixture - def oauth_provider(self) -> SimpleOAuthProvider: - return SimpleOAuthProvider( - storage_api_url='https://sapi', - mcp_server_url='https://mcp', - callback_endpoint='/callback', - client_id='mcp-server-id', - client_secret='mcp-server-secret', - server_url='https://oauth', - scope='scope', - jwt_secret=JWT_KEY, - ) - - @staticmethod - def authorization_code(*, scopes: list[str] | None = None, expires_at: float | None = None) -> Mapping[str, Any]: - auth_code = _ExtendedAuthorizationCode( - code='foo', - scopes=scopes or [], - expires_at=expires_at or time.time() + 5 * 60, # 5 minutes from now - client_id='foo-client-id', - code_challenge='foo-code-challenge', - redirect_uri=AnyUrl('foo://bar'), - redirect_uri_provided_explicitly=True, - oauth_access_token=AccessToken(token='oauth-access-token', client_id='mcp-server', scopes=['foo']), - oauth_refresh_token=RefreshToken(token='oauth-refresh-token', client_id='mcp-server', scopes=['foo']), - ) - auth_code_raw = auth_code.model_dump() - auth_code_raw['redirect_uri'] = str(auth_code_raw['redirect_uri']) # AnyUrl is not JSON serializable - return auth_code_raw - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('auth_code', 'key', 'expected'), - [ - # valid, no scopes - (code := authorization_code(), JWT_KEY, _ExtendedAuthorizationCode.model_validate(code)), - # valid, scopes - ( - code := authorization_code(scopes=['foo', 'bar']), - JWT_KEY, - _ExtendedAuthorizationCode.model_validate(code), - ), - # expired, no scopes - (code := authorization_code(expires_at=1), JWT_KEY, _ExtendedAuthorizationCode.model_validate(code)), - # wrong encryption key - (code := authorization_code(), '!@#$%^&', None), - ], - ) - async def test_load_authorization_code( - self, - auth_code: Mapping[str, Any], - key: str, - expected: _ExtendedAuthorizationCode, - oauth_provider: SimpleOAuthProvider, - ): - client_info = OAuthClientInformationFull(client_id='foo-client-id', redirect_uris=[AnyUrl('foo://bar')]) - auth_code_str = oauth_provider._encode(auth_code, key=key) - loaded_auth_code = await oauth_provider.load_authorization_code(client_info, auth_code_str) - assert loaded_auth_code == expected - - @pytest.mark.parametrize( - ('raw_at', 'raw_rt', 'scopes', 'at_expires_in', 'rt_expires_in'), - [ - ('foo', 'bar', ['email'], 3600, 168 * 3600), - ('foo', 'bar', ['user', 'email'], 3600, 168 * 3600), - ('foo', 'bar', [], 3600, 168 * 3600), - ('foo', 'bar', [], 1, 3600), # 168 * 1 second rounded up to the nearest hour -> 3600 - ('foo', 'bar', [], 7200, 168 * 3600), - ], - ) - def test_read_oauth_tokens( - self, - raw_at: str, - raw_rt: str, - scopes: list[str], - at_expires_in: int, - rt_expires_in: int, - oauth_provider: SimpleOAuthProvider, - ): - access_token, refresh_token = oauth_provider._read_oauth_tokens( - data={'access_token': raw_at, 'refresh_token': raw_rt, 'expires_in': at_expires_in}, scopes=scopes - ) - - assert access_token.token == raw_at - assert access_token.scopes == scopes - assert 0 <= at_expires_in - (access_token.expires_at - time.time()) < 1 - - assert refresh_token.token == raw_rt - assert refresh_token.scopes == scopes - assert 0 <= rt_expires_in - (refresh_token.expires_at - time.time()) < 1 - - @pytest.mark.parametrize( - ('uri', 'valid'), - [ - # === HTTP scheme - localhost only === - (AnyUrl('http://localhost:8080/foo'), True), - (AnyUrl('http://localhost:20388/oauth/callback'), True), - (AnyUrl('http://localhost/callback'), True), - (AnyUrl('http://127.0.0.1:1234/bar'), True), - (AnyUrl('http://127.0.0.1:54750/auth/callback'), True), - (AnyUrl('http://127.0.0.1/callback'), True), - # IPv6 localhost - (AnyUrl('http://[::1]:8080/callback'), True), - (AnyUrl('http://[::1]/callback'), True), - # HTTP to non-localhost should be rejected - (AnyUrl('http://example.com/callback'), False), - (AnyUrl('http://keboola.com/callback'), False), - (AnyUrl('http://192.168.1.1/callback'), False), - # === HTTPS scheme - whitelisted domains === - # Keboola domains (requires subdomain) - (AnyUrl('https://foo.keboola.com/bar/baz'), True), - (AnyUrl('https://bar.keboola.dev/baz'), True), - (AnyUrl('https://connection.keboola.com/oauth/callback'), True), - (AnyUrl('https://keboola.com/callback'), False), # requires subdomain - (AnyUrl('https://keboola.dev/callback'), False), # requires subdomain - # Data-app 'hub' subdomains are user-deployable and must be rejected (RISK-76) - (AnyUrl('https://my-app.hub.keboola.com/callback'), False), - (AnyUrl('https://my-app.hub.north-europe.azure.keboola.com/callback'), False), - (AnyUrl('https://my-app.hub.keboola.dev/callback'), False), - (AnyUrl('https://hub.keboola.com/callback'), False), # the hub root itself - (AnyUrl('https://my-app.hub.us-east4.gcp.keboola.com/callback'), False), - # ChatGPT (subdomain optional) - (AnyUrl('https://chatgpt.com'), True), - (AnyUrl('https://foo.chatgpt.com/bar'), True), - (AnyUrl('https://chatgpt.com/connector_platform_oauth_redirect'), True), - # Claude (subdomain optional) - (AnyUrl('https://claude.ai'), True), - (AnyUrl('https://foo.claude.ai/bar'), True), - (AnyUrl('https://claude.ai/api/mcp/auth_callback'), True), - # LibreChat (no subdomains allowed) - (AnyUrl('https://librechat.glami-ml.com'), True), - (AnyUrl('https://librechat.glami-ml.com/api/mcp/keboola/oauth/callback'), True), - (AnyUrl('https://foo.librechat.glami-ml.com/bar'), False), # no subdomains allowed - # Make.com (subdomain optional) - (AnyUrl('https://make.com'), True), - (AnyUrl('https://foo.make.com/bar'), True), - (AnyUrl('https://www.make.com/oauth/cb/mcp'), True), - # Devin (exact domain only) - (AnyUrl('https://api.devin.ai/callback'), True), - (AnyUrl('https://api.devin.ai'), True), - (AnyUrl('https://devin.ai/callback'), False), # must be api.devin.ai - (AnyUrl('https://foo.api.devin.ai/callback'), False), # no subdomains - # Onyx (no subdomains allowed) - (AnyUrl('https://cloud.onyx.app'), True), - (AnyUrl('https://cloud.onyx.app/mcp/oauth/callback'), True), - (AnyUrl('https://foo.cloud.onyx.app/bar'), False), # no subdomains allowed - (AnyUrl('https://onyx.app/callback'), False), # must be cloud.onyx.app - # Azure APIM (no subdomains allowed) - (AnyUrl('https://global.consent.azure-apim.net'), True), - (AnyUrl('https://global.consent.azure-apim.net/oauth/callback'), True), - (AnyUrl('https://foo.global.consent.azure-apim.net/bar'), False), # no subdomains allowed - # n8n at Groupon (no subdomains allowed) - (AnyUrl('https://n8n.groupondev.com'), True), - (AnyUrl('https://n8n.groupondev.com/rest/oauth2-credential/callback'), True), - (AnyUrl('https://n8n-business.groupondev.com'), True), - (AnyUrl('https://n8n-business.groupondev.com/rest/oauth2-credential/callback'), True), - (AnyUrl('https://n8n-merchant.groupondev.com'), True), - (AnyUrl('https://n8n-merchant.groupondev.com/rest/oauth2-credential/callback'), True), - (AnyUrl('https://n8n-llm-traffic.groupondev.com'), True), - (AnyUrl('https://n8n-llm-traffic.groupondev.com/rest/oauth2-credential/callback'), True), - (AnyUrl('https://n8n-finance.groupondev.com'), True), - (AnyUrl('https://n8n-finance.groupondev.com/rest/oauth2-credential/callback'), True), - (AnyUrl('https://n8n-playground.groupondev.com'), True), - (AnyUrl('https://n8n-playground.groupondev.com/rest/oauth2-credential/callback'), True), - (AnyUrl('https://n8n-staging.groupondev.com'), True), - (AnyUrl('https://n8n-staging.groupondev.com/rest/oauth2-credential/callback'), True), - (AnyUrl('https://foo.n8n-playground.groupondev.com/bar'), False), # no subdomains allowed - (AnyUrl('https://n8n-unknown.groupondev.com'), False), # not whitelisted - # Unknown HTTPS domains should be rejected - (AnyUrl('https://foo.bar.com/callback'), False), - (AnyUrl('https://evil.com/callback'), False), - (AnyUrl('https://fakechatgpt.com/callback'), False), - (AnyUrl('https://evilclaude.ai/callback'), False), - # === Cursor scheme - specific hosts only === - (AnyUrl('cursor://anysphere.cursor-retrieval/oauth/user-keboola-Data_warehouse/callback'), True), - (AnyUrl('cursor://anysphere.cursor-mcp/oauth/callback'), True), - (AnyUrl('cursor://anysphere.cursor-mcp/some/path'), True), - # Cursor with unknown hosts should be rejected - (AnyUrl('cursor://evil.com/callback'), False), - (AnyUrl('cursor://localhost/callback'), False), - (AnyUrl('cursor://anysphere.cursor-other/callback'), False), - # === Unknown/forbidden schemes should be rejected === - (AnyUrl('ftp://foo.bar.com'), False), - (AnyUrl('file:///etc/passwd'), False), - (AnyUrl('javascript://alert(1)'), False), - (AnyUrl('data://text/html,'), False), - # Custom schemes that are NOT whitelisted should be rejected - (AnyUrl('vscode://localhost/callback'), False), - (AnyUrl('jetbrains://localhost/callback'), False), - (AnyUrl('zed://localhost/callback'), False), - (AnyUrl('myapp://localhost/callback'), False), - (AnyUrl('evil://localhost/callback'), False), - # === Edge cases === - (None, False), # no redirect_uri - ], - ) - def test_validate_redirect_uri(self, uri: AnyUrl | None, valid: bool): - info = _OAuthClientInformationFull(redirect_uris=[AnyHttpUrl('http://foo')], client_id='foo') - if valid: - actual = info.validate_redirect_uri(uri) - assert actual == uri - else: - with pytest.raises(InvalidRedirectUriError): - info.validate_redirect_uri(uri) diff --git a/tests/test_preview.py b/tests/test_preview.py deleted file mode 100644 index 391bc7580..000000000 --- a/tests/test_preview.py +++ /dev/null @@ -1,1149 +0,0 @@ -import copy -from typing import Generator, cast - -import pytest -import pytest_asyncio -from fastmcp import FastMCP -from starlette.applications import Starlette -from starlette.testclient import TestClient - -from keboola_mcp_server import cli -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.mcp import ServerState, is_read_only_tool, is_semantic_tool -from keboola_mcp_server.preview import preview_config_diff -from keboola_mcp_server.server import create_server -from keboola_mcp_server.tools.components.utils import get_nested, set_nested_value - - -@pytest_asyncio.fixture -async def starlette_app() -> Starlette: - """Create a Starlette app with the preview endpoint.""" - config = Config( - storage_token='test-token', - storage_api_url='https://connection.test.keboola.com', - workspace_schema='test-workspace', - ) - server_state = ServerState(config=config, runtime_info=ServerRuntimeInfo(transport='stdio')) - mcp_server = create_server(config, runtime_info=server_state.runtime_info) - assert isinstance(mcp_server, FastMCP) - - app = Starlette(exception_handlers=cli._exception_handlers) - app.state.server_state = server_state - _tools = await mcp_server.list_tools(run_middleware=False) - app.state.mcp_tools_input_schema = {tool.name: tool.parameters for tool in _tools} - app.state.mcp_read_only_tools = {tool.name for tool in _tools if is_read_only_tool(tool)} - app.state.mcp_semantic_tools = {tool.name for tool in _tools if is_semantic_tool(tool)} - - app.add_route('/preview/configuration', preview_config_diff, methods=['POST']) - - return app - - -def _configure_preview_auth(mock_client, mocker, *, role='admin', features=(), bearer_token=None, branch_id=None): - """Configure the auth signals the preview endpoint reads for ToolsFilteringMiddleware-parity gating. - - Defaults to an admin SAPI token on the main/production branch, which clears the project-feature / - token-role / branch gates for ordinary write tools. - """ - mock_client.bearer_token = bearer_token - mock_client.branch_id = branch_id - mock_client.storage_client.verify_token = mocker.AsyncMock( - return_value={'owner': {'features': list(features)}, 'admin': {'role': role}} - ) - - -@pytest.fixture -def test_client(starlette_app: Starlette) -> Generator[TestClient, None, None]: - """Create a test client for the Starlette app.""" - with TestClient(starlette_app) as client: - yield client - - -class TestPreviewConfigDiff: - """Tests for the POST /preview/configuration endpoint.""" - - @pytest.mark.parametrize( - ('headers', 'expected_status'), - [ - # No authorization headers -> allowed (reaches the mutator path). - ({}, 200), - # Tool explicitly allowed. - ({'X-Allowed-Tools': 'update_config,get_tables'}, 200), - # Tool not in the allow-list -> denied. - ({'X-Allowed-Tools': 'get_tables,get_buckets'}, 403), - # Tool explicitly disallowed -> denied. - ({'X-Disallowed-Tools': 'update_config'}, 403), - # Read-only mode denies a mutator preview. - ({'X-Read-Only-Mode': 'true'}, 403), - ], - ) - def test_preview_tool_authorization(self, test_client: TestClient, mocker, headers, expected_status): - """The preview endpoint enforces the same tool-authorization headers as the MCP middleware.""" - from keboola_mcp_server.clients.storage import ComponentAPIResponse - - async def mock_fetch_component(**kwargs): - return ComponentAPIResponse.model_validate( - { - 'id': 'keboola.ex-test', - 'name': 'Test Extractor', - 'type': 'extractor', - 'configurationSchema': {}, - 'component_flags': [], - } - ) - - mocker.patch( - 'keboola_mcp_server.tools.components.tools.fetch_component', - side_effect=mock_fetch_component, - ) - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - - async def mock_config_detail(**kwargs): - return {'id': 'config-123', 'name': 'C', 'configuration': {'parameters': {}}} - - mock_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_config_detail) - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - request_payload = { - 'toolName': 'update_config', - 'toolParams': { - 'component_id': 'keboola.ex-test', - 'configuration_id': 'config-123', - 'change_description': 'Test change', - }, - } - - response = test_client.post('/preview/configuration', json=request_payload, headers=headers) - assert response.status_code == expected_status - if expected_status == 403: - assert 'not authorized' in response.json()['message'] - - @pytest.mark.parametrize( - ('tool_name', 'role', 'branch_id', 'expected_fragment'), - [ - # Read-only token role cannot drive a write tool's preview. - ('update_config', 'readOnly', None, 'read-only operations'), - # Data app tools are gated to the main/production branch. - ('modify_streamlit_data_app', 'admin', 'dev-123', 'main production branch'), - # update_flow is not available to admin/OAuth tokens (they use modify_flow). - ('update_flow', 'admin', None, 'admin/OAuth'), - ], - ) - def test_preview_project_role_branch_authorization( - self, test_client: TestClient, mocker, tool_name, role, branch_id, expected_fragment - ): - """The preview endpoint mirrors ToolsFilteringMiddleware: token-role, feature and branch gating. - - The denial happens before the mutator path runs, so no component/config mocks are needed. - """ - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - _configure_preview_auth(mock_client, mocker, role=role, branch_id=branch_id) - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - response = test_client.post( - '/preview/configuration', - json={'toolName': tool_name, 'toolParams': {'configuration_id': 'cfg-1'}}, - ) - assert response.status_code == 403 - assert expected_fragment in response.json()['message'] - - def test_preview_update_config_success(self, test_client: TestClient, mocker): - """Test successful preview of update_config tool.""" - # Mock the storage client's configuration_detail method - original_config_data = { - 'id': 'config-123', - 'name': 'Original Config Name', - 'description': 'Original description', - 'configuration': { - 'parameters': { - 'foo': 'bar', - 'baz': 42, - } - }, - } - - # Mock fetch_component to return a ComponentAPIResponse - from keboola_mcp_server.clients.storage import ComponentAPIResponse - - async def mock_fetch_component(**kwargs): - return ComponentAPIResponse.model_validate( - { - 'id': 'keboola.ex-test', - 'name': 'Test Extractor', - 'type': 'extractor', - 'configurationSchema': {}, - 'component_flags': [], - } - ) - - mocker.patch( - 'keboola_mcp_server.tools.components.tools.fetch_component', - side_effect=mock_fetch_component, - ) - - # Mock the KeboolaClient.from_state to return a mocked client - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - - # Properly mock async method - async def mock_config_detail(**kwargs): - return copy.deepcopy(original_config_data) - - mock_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_config_detail) - - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - # Request payload - request_payload = { - 'toolName': 'update_config', - 'toolParams': { - 'component_id': 'keboola.ex-test', - 'configuration_id': 'config-123', - 'change_description': 'Test change', - 'name': 'Updated Config Name', - 'description': 'Updated description', - 'parameter_updates': [ - {'op': 'set', 'path': 'foo', 'value': 'updated_bar'}, - {'op': 'set', 'path': 'new_param', 'value': 'new_value'}, - ], - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - # Assertions - assert response.status_code == 200 - result = response.json() - - # Check response structure - assert 'coordinates' in result - assert 'originalConfig' in result - assert 'updatedConfig' in result - assert 'isValid' in result - - # Check coordinates - assert result['coordinates']['componentId'] == 'keboola.ex-test' - assert result['coordinates']['configurationId'] == 'config-123' - assert 'configurationRowId' not in result['coordinates'] - - # Check that isValid is True - assert result['isValid'] is True - assert 'validationErrors' not in result - - # Check original config - assert result['originalConfig']['id'] == 'config-123' - assert result['originalConfig']['name'] == 'Original Config Name' - assert result['originalConfig']['description'] == 'Original description' - assert result['originalConfig']['configuration']['parameters']['foo'] == 'bar' - assert result['originalConfig']['configuration']['parameters']['baz'] == 42 - - # Check updated config - assert result['updatedConfig']['id'] == 'config-123' - assert result['updatedConfig']['name'] == 'Updated Config Name' - assert result['updatedConfig']['description'] == 'Updated description' - assert result['updatedConfig']['configuration']['parameters']['foo'] == 'updated_bar' - assert result['updatedConfig']['configuration']['parameters']['new_param'] == 'new_value' - assert result['updatedConfig']['configuration']['parameters']['baz'] == 42 - assert result['updatedConfig']['changeDescription'] == 'Test change' - - def test_preview_update_config_validation_error(self, test_client: TestClient, mocker): - """Test preview with validation error.""" - # Mock the storage client to raise a validation error - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - - # Properly mock async method that raises an error - async def mock_config_detail(**kwargs): - raise ValueError('Invalid configuration ID') - - mock_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_config_detail) - - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - # Request payload - request_payload = { - 'toolName': 'update_config', - 'toolParams': { - 'component_id': 'keboola.ex-test', - 'configuration_id': 'invalid-config', - 'change_description': 'Test change', - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - # Assertions - assert response.status_code == 200 - result = response.json() - - # Check that isValid is False - assert result['isValid'] is False - assert 'validationErrors' in result - assert len(result['validationErrors']) > 0 - assert 'Invalid configuration ID' in result['validationErrors'][0] - - # Check that empty configs are in the response (required by KAI backend) - assert result['originalConfig'] == {} - assert result['updatedConfig'] == {} - - def test_preview_invalid_tool_name(self, test_client: TestClient, mocker): - """Test preview with invalid tool name.""" - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - # Request payload with invalid tool name - request_payload = { - 'toolName': 'invalid_tool', - 'toolParams': { - 'component_id': 'keboola.ex-test', - 'configuration_id': 'config-123', - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - assert response.status_code == 400 - - def test_preview_update_config_only_required_params(self, test_client: TestClient, mocker): - """Test preview with only required parameters.""" - from keboola_mcp_server.clients.storage import ComponentAPIResponse - - original_config_data = { - 'id': 'config-123', - 'name': 'Original Config', - 'description': 'Original description', - 'configuration': {'parameters': {'foo': 'bar'}}, - } - - # Mock fetch_component - async def mock_fetch_component(**kwargs): - return ComponentAPIResponse.model_validate( - { - 'id': 'keboola.ex-test', - 'name': 'Test Extractor', - 'type': 'extractor', - 'configurationSchema': {}, - 'component_flags': [], - } - ) - - mocker.patch( - 'keboola_mcp_server.tools.components.tools.fetch_component', - side_effect=mock_fetch_component, - ) - - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - - # Properly mock async method - async def mock_config_detail(**kwargs): - return copy.deepcopy(original_config_data) - - mock_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_config_detail) - - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - # Request payload with minimal params - request_payload = { - 'toolName': 'update_config', - 'toolParams': { - 'component_id': 'keboola.ex-test', - 'configuration_id': 'config-123', - 'change_description': 'Test change', - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - # Assertions - assert response.status_code == 200 - result = response.json() - - assert result['isValid'] is True - # Name and description should remain unchanged - assert result['updatedConfig']['name'] == 'Original Config' - assert result['updatedConfig']['description'] == 'Original description' - # Configuration should remain the same - assert result['updatedConfig']['configuration']['parameters']['foo'] == 'bar' - - def test_preview_update_config_row_success(self, test_client: TestClient, mocker): - """Test successful preview of update_config_row tool.""" - from keboola_mcp_server.clients.storage import ComponentAPIResponse - - # Mock the configuration row data - original_row_data = { - 'id': 'row-456', - 'name': 'Original Row Name', - 'description': 'Original row description', - 'configuration': { - 'parameters': { - 'foo': 'bar', - 'baz': 42, - } - }, - } - - # Mock fetch_component - async def mock_fetch_component(**kwargs): - return ComponentAPIResponse.model_validate( - { - 'id': 'keboola.ex-test', - 'name': 'Test Extractor', - 'type': 'extractor', - 'configurationSchema': {}, - 'component_flags': [], - } - ) - - mocker.patch( - 'keboola_mcp_server.tools.components.tools.fetch_component', - side_effect=mock_fetch_component, - ) - - # Mock the KeboolaClient - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - - # Mock async method for configuration row detail - async def mock_row_detail(**kwargs): - return copy.deepcopy(original_row_data) - - mock_client.storage_client.configuration_row_detail = mocker.AsyncMock(side_effect=mock_row_detail) - - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - # Request payload - request_payload = { - 'toolName': 'update_config_row', - 'toolParams': { - 'component_id': 'keboola.ex-test', - 'configuration_id': 'config-123', - 'configuration_row_id': 'row-456', - 'change_description': 'Test row change', - 'name': 'Updated Row Name', - 'description': 'Updated row description', - 'parameter_updates': [ - {'op': 'set', 'path': 'foo', 'value': 'updated_bar'}, - ], - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - # Assertions - assert response.status_code == 200 - result = response.json() - - # Check response structure - assert result['isValid'] is True - assert result['coordinates']['componentId'] == 'keboola.ex-test' - assert result['coordinates']['configurationId'] == 'config-123' - assert result['coordinates']['configurationRowId'] == 'row-456' - - # Check updated config - assert result['updatedConfig']['name'] == 'Updated Row Name' - assert result['updatedConfig']['description'] == 'Updated row description' - assert result['updatedConfig']['configuration']['parameters']['foo'] == 'updated_bar' - - def test_preview_update_sql_transformation_success(self, test_client: TestClient, mocker): - """Test successful preview of update_sql_transformation tool.""" - from keboola_mcp_server.clients.storage import ComponentAPIResponse - - # Mock the transformation configuration data - original_config_data = { - 'id': 'config-123', - 'name': 'Original Transformation', - 'description': 'Original transformation description', - 'configuration': { - 'parameters': { - 'blocks': [ - { - 'name': 'Block 1', - 'codes': [{'name': 'Code 1', 'script': ['SELECT * FROM table1;']}], - } - ], - }, - 'storage': { - 'input': {'tables': []}, - 'output': {'tables': []}, - }, - }, - } - - # Mock fetch_component for transformation - async def mock_fetch_component(**kwargs): - return ComponentAPIResponse.model_validate( - { - 'id': 'keboola.snowflake-transformation', - 'name': 'Snowflake Transformation', - 'type': 'transformation', - 'configurationSchema': {}, - 'component_flags': [], - } - ) - - mocker.patch( - 'keboola_mcp_server.tools.components.tools.fetch_component', - side_effect=mock_fetch_component, - ) - - # Mock the KeboolaClient - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - - async def mock_config_detail(**kwargs): - return copy.deepcopy(original_config_data) - - mock_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_config_detail) - mock_client.storage_client.configuration_metadata_get = mocker.AsyncMock( - return_value=[{'key': 'KBC.configuration.folderName', 'value': 'Old Folder'}] - ) - - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - # Mock WorkspaceManager - mock_workspace_manager = mocker.AsyncMock() - mock_workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='snowflake') - - mocker.patch( - 'keboola_mcp_server.preview.WorkspaceManager.from_state', - return_value=mock_workspace_manager, - ) - - # Request payload - request_payload = { - 'toolName': 'update_sql_transformation', - 'toolParams': { - 'configuration_id': 'config-123', - 'change_description': 'Update transformation', - 'folder': 'My Folder', - 'parameter_updates': [ - { - 'op': 'add_block', - 'block': {'name': 'Updated Block', 'codes': [{'name': 'Updated Code', 'script': 'SELECT 1'}]}, - 'position': 'end', - }, - ], - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - # Assertions - assert response.status_code == 200 - result = response.json() - - # Check response structure - assert result['isValid'] is True - assert result['coordinates']['componentId'] == 'keboola.snowflake-transformation' - assert result['coordinates']['configurationId'] == 'config-123' - - # Check that configuration was updated - assert result['updatedConfig']['configuration']['parameters']['blocks'][-1]['name'] == 'Updated Block' - - # Check that folder diff is injected into originalConfig/updatedConfig - assert result['originalConfig']['folder'] == 'Old Folder' - assert result['updatedConfig']['folder'] == 'My Folder' - - def test_preview_update_flow_success(self, test_client: TestClient, mocker): - """Test successful preview of update_flow tool.""" - # Mock the flow configuration data - original_config_data = { - 'id': 'flow-123', - 'name': 'Original Flow', - 'description': 'Original flow description', - 'configuration': { - 'phases': [ - { - 'id': 'phase1', - 'name': 'Simple Phase', - 'description': 'A simple conditional flow phase', - 'next': [{'id': 'transition1', 'name': 'Simple Transition', 'goto': None}], - }, - ], - 'tasks': [ - { - 'id': 'task1', - 'name': 'Simple Task', - 'phase': 'phase1', - 'enabled': True, - 'task': { - 'componentId': 'keboola.ex-test', - 'type': 'notification', - 'recipients': [{'channel': 'email', 'address': 'admin@company.com'}], - 'title': 'Simple Notification', - 'message': 'This is a simple notification task', - }, - }, - ], - }, - } - - # Mock the KeboolaClient - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - - async def mock_config_detail(**kwargs): - return copy.deepcopy(original_config_data) - - mock_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_config_detail) - # update_flow is the flow tool exposed to non-admin/non-OAuth tokens, so preview it as one. - _configure_preview_auth(mock_client, mocker, role='') - - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - # Request payload - request_payload = { - 'toolName': 'update_flow', - 'toolParams': { - 'flow_type': 'keboola.orchestrator', - 'configuration_id': 'flow-123', - 'change_description': 'Update flow', - 'name': 'Updated Flow', - 'description': 'Updated flow description', - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - # Assertions - assert response.status_code == 200 - result = response.json() - - # Check response structure - assert result['isValid'] is True - assert result['coordinates']['componentId'] == 'keboola.orchestrator' - assert result['coordinates']['configurationId'] == 'flow-123' - - # Check updated config - assert result['updatedConfig']['name'] == 'Updated Flow' - assert result['updatedConfig']['description'] == 'Updated flow description' - - @pytest.mark.parametrize( - ('existing_schedule_dicts', 'schedules_request', 'expected_original', 'expected_updated'), - [ - pytest.param( - # add: no existing schedulers → new one appears only in updated - [], - [{'action': 'add', 'cronTab': '0 8 * * 1,2,3,4,5', 'timezone': 'Europe/Prague', 'state': 'enabled'}], - [], - [{'scheduleId': None, 'cronTab': '0 8 * * 1,2,3,4,5', 'timezone': 'Europe/Prague', 'state': 'enabled'}], - id='add', - ), - pytest.param( - # update: existing scheduler's cron changes; original must keep old value - [{'schedule_id': 'sched-1', 'cron_tab': '0 8 * * 1,2,3,4,5', 'timezone': 'UTC', 'state': 'enabled'}], - [{'action': 'update', 'scheduleId': 'sched-1', 'cronTab': '0 10 * * 1,2,3,4,5'}], - [{'scheduleId': 'sched-1', 'cronTab': '0 8 * * 1,2,3,4,5', 'timezone': 'UTC', 'state': 'enabled'}], - [{'scheduleId': 'sched-1', 'cronTab': '0 10 * * 1,2,3,4,5', 'timezone': 'UTC', 'state': 'enabled'}], - id='update', - ), - pytest.param( - # remove: existing scheduler disappears in updated; original must still have it - [{'schedule_id': 'sched-1', 'cron_tab': '0 8 * * 1,2,3,4,5', 'timezone': 'UTC', 'state': 'enabled'}], - [{'action': 'remove', 'scheduleId': 'sched-1'}], - [{'scheduleId': 'sched-1', 'cronTab': '0 8 * * 1,2,3,4,5', 'timezone': 'UTC', 'state': 'enabled'}], - [], - id='remove', - ), - pytest.param( - # multiple actions in one request: update one, remove another, add a new one - [ - {'schedule_id': 'sched-1', 'cron_tab': '0 8 * * 1,2,3,4,5', 'timezone': 'UTC', 'state': 'enabled'}, - { - 'schedule_id': 'sched-2', - 'cron_tab': '0 20 * * 1,2,3,4,5', - 'timezone': 'UTC', - 'state': 'disabled', - }, - ], - [ - {'action': 'update', 'scheduleId': 'sched-1', 'state': 'disabled'}, - {'action': 'remove', 'scheduleId': 'sched-2'}, - {'action': 'add', 'cronTab': '0 6 * * 1', 'timezone': 'US/Eastern', 'state': 'enabled'}, - ], - [ - {'scheduleId': 'sched-1', 'cronTab': '0 8 * * 1,2,3,4,5', 'timezone': 'UTC', 'state': 'enabled'}, - {'scheduleId': 'sched-2', 'cronTab': '0 20 * * 1,2,3,4,5', 'timezone': 'UTC', 'state': 'disabled'}, - ], - [ - {'scheduleId': 'sched-1', 'cronTab': '0 8 * * 1,2,3,4,5', 'timezone': 'UTC', 'state': 'disabled'}, - {'scheduleId': None, 'cronTab': '0 6 * * 1', 'timezone': 'US/Eastern', 'state': 'enabled'}, - ], - id='multiple_actions', - ), - ], - ) - def test_preview_modify_flow_with_schedules_success( - self, - test_client: TestClient, - mocker, - existing_schedule_dicts: list[dict], - schedules_request: list[dict], - expected_original: list[dict], - expected_updated: list[dict], - ): - """Test that preview returns the correct scheduler diff for add/update/remove actions.""" - from keboola_mcp_server.tools.flow.scheduler_model import ScheduleDetail - - original_config_data = { - 'id': 'flow-123', - 'name': 'My Flow', - 'description': 'A test flow', - 'configuration': { - 'phases': [ - { - 'id': 'phase1', - 'name': 'Phase 1', - 'next': [{'id': 't1', 'name': 'End', 'goto': None}], - } - ], - 'tasks': [ - { - 'id': 'task1', - 'name': 'Task 1', - 'phase': 'phase1', - 'enabled': True, - 'task': {'type': 'job', 'componentId': 'keboola.ex-test', 'configId': 'cfg-1', 'mode': 'run'}, - } - ], - }, - } - - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - mock_client.storage_client.configuration_detail = mocker.AsyncMock( - side_effect=lambda **_: copy.deepcopy(original_config_data) - ) - # modify_flow is the flow tool exposed to admin/OAuth tokens, so preview it as an admin. - _configure_preview_auth(mock_client, mocker, role='admin') - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - existing_schedules = [ - ScheduleDetail.model_construct( - schedule_id=s['schedule_id'], - cron_tab=s['cron_tab'], - timezone=s['timezone'], - state=s['state'], - target_executions=[], - ) - for s in existing_schedule_dicts - ] - mocker.patch( - 'keboola_mcp_server.tools.flow.scheduler.list_schedules_for_config', - new=mocker.AsyncMock(return_value=existing_schedules), - ) - - response = test_client.post( - '/preview/configuration', - json={ - 'toolName': 'modify_flow', - 'toolParams': { - 'flow_type': 'keboola.flow', - 'configuration_id': 'flow-123', - 'change_description': 'Update schedules', - 'schedules': schedules_request, - }, - }, - ) - - assert response.status_code == 200 - result = response.json() - assert result['isValid'] is True - assert result['coordinates']['componentId'] == 'keboola.flow' - assert result['coordinates']['configurationId'] == 'flow-123' - # Scheduler diff: original must reflect the state before, updated must reflect the state after - assert result['originalConfig']['schedulers'] == expected_original - assert result['updatedConfig']['schedulers'] == expected_updated - - @pytest.mark.parametrize( - ('existing_schedule_dicts', 'schedules_request', 'expected_error_fragment'), - [ - pytest.param( - [], - [{'action': 'remove', 'scheduleId': 'sched-missing'}], - 'sched-missing', - id='remove_nonexistent', - ), - pytest.param( - [], - [{'action': 'update', 'scheduleId': 'sched-missing', 'cronTab': '0 8 * * 1,2,3,4,5'}], - 'sched-missing', - id='update_nonexistent', - ), - pytest.param( - [], - [{'action': 'add', 'cronTab': 'not-a-cron', 'state': 'enabled'}], - 'Invalid cron tab', - id='add_invalid_cron', - ), - ], - ) - def test_preview_modify_flow_with_schedules_errors( - self, - test_client: TestClient, - mocker, - existing_schedule_dicts: list[dict], - schedules_request: list[dict], - expected_error_fragment: str, - ): - """Test that scheduler validation errors are surfaced as isValid=False responses.""" - from keboola_mcp_server.tools.flow.scheduler_model import ScheduleDetail - - original_config_data = { - 'id': 'flow-123', - 'name': 'My Flow', - 'description': 'A test flow', - 'configuration': { - 'phases': [ - { - 'id': 'phase1', - 'name': 'Phase 1', - 'next': [{'id': 't1', 'name': 'End', 'goto': None}], - } - ], - 'tasks': [ - { - 'id': 'task1', - 'name': 'Task 1', - 'phase': 'phase1', - 'enabled': True, - 'task': {'type': 'job', 'componentId': 'keboola.ex-test', 'configId': 'cfg-1', 'mode': 'run'}, - } - ], - }, - } - - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - mock_client.storage_client.configuration_detail = mocker.AsyncMock( - side_effect=lambda **_: copy.deepcopy(original_config_data) - ) - # modify_flow is the flow tool exposed to admin/OAuth tokens, so preview it as an admin. - _configure_preview_auth(mock_client, mocker, role='admin') - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - existing_schedules = [ - ScheduleDetail.model_construct( - schedule_id=s['schedule_id'], - cron_tab=s['cron_tab'], - timezone=s['timezone'], - state=s['state'], - target_executions=[], - ) - for s in existing_schedule_dicts - ] - mocker.patch( - 'keboola_mcp_server.tools.flow.scheduler.list_schedules_for_config', - new=mocker.AsyncMock(return_value=existing_schedules), - ) - - response = test_client.post( - '/preview/configuration', - json={ - 'toolName': 'modify_flow', - 'toolParams': { - 'flow_type': 'keboola.flow', - 'configuration_id': 'flow-123', - 'change_description': 'Update schedules', - 'schedules': schedules_request, - }, - }, - ) - - assert response.status_code == 200 - result = response.json() - assert result['isValid'] is False - assert expected_error_fragment in str(result.get('validationErrors', '')) - - def test_preview_modify_streamlit_data_app_success(self, test_client: TestClient, mocker): - """Test successful preview of modify_streamlit_data_app tool.""" - from keboola_mcp_server.clients.client import DATA_APP_COMPONENT_ID - - # Mock the data app configuration data - original_config_data = { - 'id': 'app-123', - 'name': 'Original Data App', - 'description': 'Original data app description', - 'configuration': { - 'parameters': { - 'dataApp': { - 'slug': 'old-slug', - 'secrets': {'FOO': 'old', 'KEEP': 'x'}, - }, - 'script': ['old'], - 'packages': ['numpy'], - }, - 'authorization': {}, - }, - } - - # Mock the KeboolaClient - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - mock_client.token = 'test-token' - mock_client.storage_api_url = 'https://connection.test.keboola.com' - - async def mock_config_detail(**kwargs): - return copy.deepcopy(original_config_data) - - async def mock_encrypt(*args, **kwargs): - return args[0] - - mock_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_config_detail) - mock_client.storage_client.project_id = mocker.AsyncMock(return_value='test-project') - mock_client.encryption_client.encrypt = mocker.AsyncMock(side_effect=mock_encrypt) - # Data app tools are allowed only on the main/production branch (branch_id=None). - _configure_preview_auth(mock_client, mocker, role='admin', branch_id=None) - - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - # Mock WorkspaceManager - mock_workspace_manager = mocker.AsyncMock() - mock_workspace_manager.get_workspace_id = mocker.AsyncMock(return_value=123) - mock_workspace_manager.get_branch_id = mocker.AsyncMock(return_value=456) - mock_workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='snowflake') - - mocker.patch( - 'keboola_mcp_server.preview.WorkspaceManager.from_state', - return_value=mock_workspace_manager, - ) - - # Mock _fetch_data_app function - from keboola_mcp_server.tools.data_apps import DataApp - - mock_data_app = DataApp( - name='Original Data App', - description='Original data app description', - component_id='keboola.data-apps', - configuration_id='app-123', - data_app_id='data-app-123', - project_id='project-123', - branch_id='456', - config_version='1', - state='running', - type='streamlit', - configuration={ - 'parameters': { - 'dataApp': { - 'slug': 'old-slug', - 'secrets': {'FOO': 'old', 'KEEP': 'x'}, - }, - 'script': ['old'], - 'packages': ['numpy'], - }, - 'authorization': {}, - 'storage': {}, - }, - ) - - async def mock_fetch_data_app(client, **kwargs): - return mock_data_app - - mocker.patch( - 'keboola_mcp_server.tools.data_apps._fetch_data_app', - side_effect=mock_fetch_data_app, - ) - - # Request payload - request_payload = { - 'toolName': 'modify_streamlit_data_app', - 'toolParams': { - 'configuration_id': 'app-123', - 'change_description': 'Update data app', - 'name': 'Updated Data App', - 'description': 'Updated data app description', - 'source_code': 'print("Hello World")', - 'packages': ['streamlit', 'pandas'], - 'authentication_type': 'default', - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - # Assertions - assert response.status_code == 200 - result = response.json() - - # Check response structure - assert result['isValid'] is True - assert result['coordinates']['componentId'] == DATA_APP_COMPONENT_ID - assert result['coordinates']['configurationId'] == 'app-123' - - # Check original config - assert result['originalConfig'] == mock_data_app.model_dump() - - # Check updated config - updated_config_no_script = copy.deepcopy(result['updatedConfig']) - set_nested_value(updated_config_no_script, 'configuration.parameters.script', []) - assert updated_config_no_script == { - **mock_data_app.model_dump(), - 'changeDescription': 'Update data app', - 'name': 'Updated Data App', - 'description': 'Updated data app description', - 'configuration': { - 'parameters': { - 'dataApp': { - 'slug': 'updated-data-app', - 'secrets': { - 'FOO': 'old', - 'KEEP': 'x', - 'BRANCH_ID': '456', - 'WORKSPACE_ID': '123', - }, - }, - 'script': [], - 'packages': ['httpx', 'pandas', 'streamlit'], - }, - 'authorization': {}, - # The empty `storage: {}` from the original config is pruned on re-save so it is - # never persisted as `[]` (which breaks the Writable Tables editor, AI-3135). - }, - } - # check the script - actual_script = get_nested(result['updatedConfig'], 'configuration.parameters.script') - assert isinstance(actual_script, list) - assert len(actual_script) == 1 - assert cast(str, actual_script[0]).endswith('print("Hello World")') - - def test_preview_validation_missing_required_param(self, test_client: TestClient, mocker): - """Test validation error for missing required parameter.""" - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - # Request missing required 'configuration_id' - request_payload = { - 'toolName': 'update_config', - 'toolParams': { - 'component_id': 'keboola.ex-test', - # 'configuration_id': missing! - 'change_description': 'Test', - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - # Assertions - assert response.status_code == 200 - result = response.json() - print(result) - assert result['isValid'] is False - assert 'validationErrors' in result - assert 'configuration_id' in str(result['validationErrors']) - - # Check that empty configs are in the response (required by KAI backend) - assert result['originalConfig'] == {} - assert result['updatedConfig'] == {} - - def test_preview_validation_invalid_param_type(self, test_client: TestClient, mocker): - """Test validation error for invalid parameter type. - - Note: Type validation errors at the API level return 400 Bad Request - rather than 200 with validation errors, as they represent malformed requests. - """ - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - request_payload = { - 'toolName': 'update_config', - 'toolParams': { - 'component_id': 'keboola.ex-test', - 'configuration_id': 'config-123', - 'change_description': 'Test change', - 'name': 'Updated Config Name', - 'description': 'Updated description', - 'parameter_updates': [ - {'op': 'set', 'path': 'foo', 'value': 'updated_bar'}, - {'op': 'foo', 'path': 'new_param', 'value': 'new_value'}, # Invalid op - ], - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - print(response.json()) - - assert response.status_code == 200 - result = response.json() - assert result['isValid'] is False - assert 'validationErrors' in result - assert 'parameter_updates.1' in str(result['validationErrors']) - - # Check that empty configs are in the response (required by KAI backend) - assert result['originalConfig'] == {} - assert result['updatedConfig'] == {} - - def test_preview_validation_passes_for_valid_params(self, test_client: TestClient, mocker): - """Test that validation passes for valid parameters and processing continues.""" - from keboola_mcp_server.clients.storage import ComponentAPIResponse - - original_config_data = { - 'id': 'config-123', - 'name': 'Original Config', - 'description': 'Original description', - 'configuration': {'parameters': {'foo': 'bar'}}, - } - - # Mock fetch_component - async def mock_fetch_component(**kwargs): - return ComponentAPIResponse.model_validate( - { - 'id': 'keboola.ex-test', - 'name': 'Test Extractor', - 'type': 'extractor', - 'configurationSchema': {}, - 'component_flags': [], - } - ) - - mocker.patch( - 'keboola_mcp_server.tools.components.tools.fetch_component', - side_effect=mock_fetch_component, - ) - - # Mock the KeboolaClient - mock_client = mocker.AsyncMock(KeboolaClient) - _configure_preview_auth(mock_client, mocker) # admin/main default; overridden below where needed - - async def mock_config_detail(**kwargs): - return copy.deepcopy(original_config_data) - - mock_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_config_detail) - - mocker.patch('keboola_mcp_server.preview.KeboolaClient.from_state', return_value=mock_client) - - # Request payload with valid params (should pass validation) - request_payload = { - 'toolName': 'update_config', - 'toolParams': { - 'component_id': 'keboola.ex-test', - 'configuration_id': 'config-123', - 'change_description': 'Test change', - }, - } - - # Make the request - response = test_client.post('/preview/configuration', json=request_payload) - - # Assertions - assert response.status_code == 200 - result = response.json() - # Validation passed, so processing continued and isValid should be True - assert result['isValid'] is True - assert 'validationErrors' not in result - # Config should be returned - assert 'originalConfig' in result - assert 'updatedConfig' in result diff --git a/tests/test_server.py b/tests/test_server.py deleted file mode 100644 index 31c25a1df..000000000 --- a/tests/test_server.py +++ /dev/null @@ -1,544 +0,0 @@ -import asyncio -import json -import subprocess -import tempfile -from dataclasses import asdict -from pathlib import Path -from typing import Annotated, Any - -import httpx -import pytest -from fastmcp import Client, Context, FastMCP -from fastmcp.client import StreamableHttpTransport -from fastmcp.tools import FunctionTool -from mcp.types import TextContent -from pydantic import Field - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import Config, ServerRuntimeInfo -from keboola_mcp_server.mcp import ServerState, _exclude_none_serializer, toon_serializer, toon_serializer_compact -from keboola_mcp_server.server import create_server -from keboola_mcp_server.tools.components.tools import COMPONENT_TOOLS_TAG -from keboola_mcp_server.tools.constants import CONFIG_DIFF_PREVIEW_TAG -from keboola_mcp_server.tools.data_apps import DATA_APP_TOOLS_TAG -from keboola_mcp_server.tools.doc import DOC_TOOLS_TAG -from keboola_mcp_server.tools.flow.tools import FLOW_TOOLS_TAG -from keboola_mcp_server.tools.jobs import JOB_TOOLS_TAG -from keboola_mcp_server.tools.oauth import OAUTH_TOOLS_TAG -from keboola_mcp_server.tools.project import PROJECT_TOOLS_TAG -from keboola_mcp_server.tools.search import SEARCH_TOOLS_TAG -from keboola_mcp_server.tools.semantic import SEMANTIC_TOOLS_TAG -from keboola_mcp_server.tools.sql import SQL_TOOLS_TAG -from keboola_mcp_server.tools.storage.tools import STORAGE_TOOLS_TAG -from keboola_mcp_server.workspace import WorkspaceManager - - -class TestServer: - @pytest.mark.asyncio - async def test_list_tools(self): - server = create_server(Config(), runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - tools = await server.list_tools(run_middleware=False) - assert sorted(tool.name for tool in tools) == [ - 'add_config_row', - 'create_conditional_flow', - 'create_config', - 'create_flow', - 'create_oauth_url', - 'create_python_js_data_app_git_credential', - 'create_sql_transformation', - 'delete_python_js_data_app_draft', - 'deploy_data_app', - 'docs_query', - 'find_component_id', - 'get_buckets', - 'get_components', - 'get_config_examples', - 'get_configs', - 'get_data_apps', - 'get_flow_examples', - 'get_flow_schema', - 'get_flows', - 'get_jobs', - 'get_project_info', - 'get_semantic_context', - 'get_semantic_schema', - 'get_tables', - 'modify_flow', - 'modify_python_js_data_app', - 'modify_streamlit_data_app', - 'query_data', - 'run_job', - 'run_sync_action', - 'search', - 'search_semantic_context', - 'update_config', - 'update_config_row', - 'update_descriptions', - 'update_flow', - 'update_project_description', - 'update_sql_transformation', - 'validate_semantic_query', - ] - - @pytest.mark.asyncio - async def test_tools_have_descriptions(self): - server = create_server(Config(), runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - tools = await server.list_tools(run_middleware=False) - - missing_descriptions: list[str] = [] - for tool in tools: - if not tool.description: - missing_descriptions.append(tool.name) - - missing_descriptions.sort() - assert not missing_descriptions, f'These tools have no description: {missing_descriptions}' - - @pytest.mark.asyncio - async def test_tools_have_serializer(self): - server = create_server(Config(), runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - tools = await server.list_tools(run_middleware=False) - - missing_serializer: list[str] = [] - for tool in tools: - if not tool.serializer: - missing_serializer.append(tool.name) - if tool.serializer not in (_exclude_none_serializer, toon_serializer, toon_serializer_compact): - missing_serializer.append(tool.name) - - missing_serializer.sort() - assert not missing_serializer, f'These tools have no serializer: {missing_serializer}' - - @pytest.mark.asyncio - async def test_tools_input_schema(self): - server = create_server(Config(), runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - tools = await server.list_tools(run_middleware=False) - - missing_properties: list[str] = [] - missing_type: list[str] = [] - missing_default: list[str] = [] - for tool in tools: - properties = tool.parameters['properties'] - if not properties: - missing_properties.append(tool.name) - continue - - required = tool.parameters.get('required') or [] - for prop_name, prop_def in properties.items(): - if all( - [ - 'type' not in prop_def, - ('anyOf' not in prop_def or any('type' not in t for t in prop_def['anyOf'])), - ] - ): - missing_type.append(f'{tool.name}.{prop_name}') - if prop_name not in required and 'default' not in prop_def: - missing_default.append(f'{tool.name}.{prop_name}') - - missing_properties.sort() - assert missing_properties == ['get_project_info'] - missing_type.sort() - assert not missing_type, f'These tool params have no "type" info: {missing_type}' - missing_default.sort() - assert not missing_default, f'These tool params are optional, but have no default value: {missing_default}' - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('config', 'envs'), - [ - ( # config params in Config class - Config( - storage_token='SAPI_1234', storage_api_url='http://connection.sapi', workspace_schema='WORKSPACE_1234' - ), - {}, - ), - ( # config params in the OS environment - Config(), - { - 'KBC_STORAGE_TOKEN': 'SAPI_1234', - 'KBC_STORAGE_API_URL': 'http://connection.sapi', - 'KBC_WORKSPACE_SCHEMA': 'WORKSPACE_1234', - }, - ), - ( # config params mixed up in both the Config class and the OS environment - Config(storage_api_url='http://connection.sapi'), - {'KBC_STORAGE_TOKEN': 'SAPI_1234', 'KBC_WORKSPACE_SCHEMA': 'WORKSPACE_1234'}, - ), - ( # the OS environment overrides the initial Config class - Config(storage_token='foo-bar', storage_api_url='http://connection.sapi', workspace_schema='xyz_123'), - {'KBC_STORAGE_TOKEN': 'SAPI_1234', 'KBC_WORKSPACE_SCHEMA': 'WORKSPACE_1234'}, - ), - # TODO: Also test values obtained from an HTTP request. - ], -) -async def test_with_session_state(config: Config, envs: dict[str, Any], mocker): - expected_param_description = 'Parameter 1 description' - - async def assessed_function( - ctx: Context, param: Annotated[str, Field(description=expected_param_description)] - ) -> str: - """custom text""" - assert hasattr(ctx.session, 'state') - - keboola_client = KeboolaClient.from_state(ctx.session.state) - assert keboola_client is not None - assert keboola_client.token == 'SAPI_1234' - - workspace_manager = WorkspaceManager.from_state(ctx.session.state) - assert workspace_manager is not None - assert workspace_manager._workspace_schema == 'WORKSPACE_1234' - - return param - - # mock the environment variables - os_mock = mocker.patch('keboola_mcp_server.server.os') - os_mock.environ = envs - - mocker.patch( - 'keboola_mcp_server.clients.client.AsyncStorageClient.verify_token', - return_value={ - 'owner': {'features': ['global-search', 'waii-integration', 'hide-conditional-flows']}, - 'admin': {'role': 'admin'}, - }, - ) - - # create MCP server with the initial Config - mcp = create_server(config, runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(mcp, FastMCP) - tools_count = len(await mcp.list_tools(run_middleware=False)) - mcp.add_tool(FunctionTool.from_function(assessed_function, name='assessed-function')) - - # running the server as stdio transport through client - async with Client(mcp) as client: - tools = await client.list_tools() - # plus the one we've added in this test minus two filtered tools - # create_flow() and update_flow(), and four semantic tools (feature not enabled in mock) - assert len(tools) == tools_count + 1 - 2 - 4 - assert tools[-1].name == 'assessed-function' - assert tools[-1].description == 'custom text' - # check if the inputSchema contains the expected param description - assert expected_param_description in str(tools[-1].inputSchema) - result = await client.call_tool('assessed-function', {'param': 'value'}) - assert isinstance(result.content[0], TextContent) - assert result.content[0].text == 'value' - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('admin_info', 'expected_included', 'expected_excluded'), - [ - ({'role': 'admin'}, 'modify_flow', 'update_flow'), - ({'role': None}, 'update_flow', 'modify_flow'), - ({}, 'update_flow', 'modify_flow'), - ], -) -async def test_with_session_state_admin_role_tools(mocker, admin_info, expected_included, expected_excluded): - - os_mock = mocker.patch('keboola_mcp_server.server.os') - os_mock.environ = { - 'KBC_STORAGE_TOKEN': 'SAPI_1234', - 'KBC_STORAGE_API_URL': 'http://connection.sapi', - 'KBC_WORKSPACE_SCHEMA': 'WORKSPACE_1234', - } - - mocker.patch( - 'keboola_mcp_server.clients.client.AsyncStorageClient.verify_token', - return_value={ - 'owner': {'features': ['global-search', 'waii-integration', 'hide-conditional-flows']}, - 'admin': admin_info, - }, - ) - - mcp = create_server(Config(), runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(mcp, FastMCP) - - async with Client(mcp) as client: - tools = await client.list_tools() - tool_names = {tool.name for tool in tools} - assert expected_included in tool_names - assert expected_excluded not in tool_names - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('os_environ_params', 'expected_params'), - [ - # no params in os.environ, tokens as in the config - ({}, {'storage_token': 'test-storage-token', 'workspace_schema': 'test-workspace-schema'}), - # params in os.environ, tokens configured from os.environ, missing from the config - ( - {'storage_token': 'test-storage-token-2'}, - {'storage_token': 'test-storage-token-2', 'workspace_schema': 'test-workspace-schema'}, - ), - ], -) -async def test_keboola_injection_and_lifespan( - mocker, os_environ_params: dict[str, str], expected_params: dict[str, str] -): - """ - Test that the KeboolaClient and WorkspaceManager are injected into the context and that the lifespan of the client - is managed by the server. - Test that the ServerState is properly initialized and that the client and workspace are properly disposed of. - """ - cfg_dict = { - 'storage_token': 'test-storage-token', - 'workspace_schema': 'test-workspace-schema', - 'storage_api_url': 'https://connection.keboola.com', - 'transport': 'stdio', - } - config = Config.from_dict(cfg_dict) - - mocker.patch('keboola_mcp_server.server.os.environ', os_environ_params) - mocker.patch( - 'keboola_mcp_server.clients.client.AsyncStorageClient.verify_token', - return_value={'owner': {'features': ['global-search', 'waii-integration', 'conditional-flows']}}, - ) - - server = create_server(config, runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - - async def assessed_function(ctx: Context, param: str) -> str: - assert hasattr(ctx.session, 'state') - client = KeboolaClient.from_state(ctx.session.state) - assert isinstance(client, KeboolaClient) - workspace = WorkspaceManager.from_state(ctx.session.state) - assert isinstance(workspace, WorkspaceManager) - - # check that the server state config contains the initial params + the environment params - server_state = ServerState.from_context(ctx) - assert asdict(server_state.config) == asdict(config) | os_environ_params - - assert client.token == expected_params['storage_token'] - assert workspace._workspace_schema == expected_params['workspace_schema'] - - return param - - server.add_tool(FunctionTool.from_function(assessed_function, name='assessed_function')) - - async with Client(server) as client: - result = await client.call_tool('assessed_function', {'param': 'value'}) - assert isinstance(result.content[0], TextContent) - assert result.content[0].text == 'value' - - -@pytest.mark.asyncio -async def test_tool_annotations_and_tags(): - """ - Test that the tool annotations are properly set. - """ - server = create_server(Config(), runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - tools = await server.list_tools(run_middleware=False) - for tool in tools: - assert tool.tags is not None, f'{tool.name} has no tags' - if tool.annotations is not None: - if tool.annotations.readOnlyHint: - assert tool.annotations.destructiveHint is None, f'{tool.name} has destructiveHint' - assert tool.annotations.idempotentHint is None, f'{tool.name} has idempotentHint' - elif tool.annotations.destructiveHint: - assert tool.annotations.readOnlyHint is None, f'{tool.name} has readOnlyHint' - elif tool.annotations.destructiveHint is False: - assert tool.annotations.idempotentHint is None, f'{tool.name} has idempotentHint' - if tool.annotations.idempotentHint: - assert tool.annotations.readOnlyHint is None, f'{tool.name} has readOnlyHint' - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('tool_name', 'expected_readonly', 'expected_destructive', 'expected_idempotent', 'tags'), - [ - # components - ('get_components', True, None, None, {COMPONENT_TOOLS_TAG}), - ('get_configs', True, None, None, {COMPONENT_TOOLS_TAG}), - ('get_config_examples', True, None, None, {COMPONENT_TOOLS_TAG}), - ('create_config', None, False, None, {COMPONENT_TOOLS_TAG}), - ('update_config', None, True, None, {COMPONENT_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}), - ('add_config_row', None, False, None, {COMPONENT_TOOLS_TAG}), - ('update_config_row', None, True, None, {COMPONENT_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}), - ('run_sync_action', True, None, None, {COMPONENT_TOOLS_TAG}), - ('create_sql_transformation', None, False, None, {COMPONENT_TOOLS_TAG}), - ('update_sql_transformation', None, True, None, {COMPONENT_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}), - # storage - ('get_buckets', True, None, None, {STORAGE_TOOLS_TAG}), - ('get_tables', True, None, None, {STORAGE_TOOLS_TAG}), - ('update_descriptions', None, True, None, {STORAGE_TOOLS_TAG}), - # flows - ('create_flow', None, False, None, {FLOW_TOOLS_TAG}), - ('create_conditional_flow', None, False, None, {FLOW_TOOLS_TAG}), - ('get_flows', True, None, None, {FLOW_TOOLS_TAG}), - ('update_flow', None, True, None, {FLOW_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}), - ('modify_flow', None, True, None, {FLOW_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}), - ('get_flow_examples', True, None, None, {FLOW_TOOLS_TAG}), - ('get_flow_schema', True, None, None, {FLOW_TOOLS_TAG}), - # sql - ('query_data', True, None, None, {SQL_TOOLS_TAG}), - # jobs - ('get_jobs', True, None, None, {JOB_TOOLS_TAG}), - ('run_job', None, True, None, {JOB_TOOLS_TAG}), - # project/doc/search - ('get_project_info', True, None, None, {PROJECT_TOOLS_TAG}), - ('update_project_description', None, True, None, {PROJECT_TOOLS_TAG}), - ('docs_query', True, None, None, {DOC_TOOLS_TAG}), - ('find_component_id', True, None, None, {SEARCH_TOOLS_TAG}), - # semantic - ('search_semantic_context', True, None, None, {SEMANTIC_TOOLS_TAG}), - ('get_semantic_context', True, None, None, {SEMANTIC_TOOLS_TAG}), - ('get_semantic_schema', True, None, None, {SEMANTIC_TOOLS_TAG}), - ('validate_semantic_query', True, None, None, {SEMANTIC_TOOLS_TAG}), - # oauth - ('create_oauth_url', None, True, None, {OAUTH_TOOLS_TAG}), - # data apps - ('modify_streamlit_data_app', None, True, None, {DATA_APP_TOOLS_TAG, CONFIG_DIFF_PREVIEW_TAG}), - ('modify_python_js_data_app', None, True, None, {DATA_APP_TOOLS_TAG}), - ('create_python_js_data_app_git_credential', None, False, None, {DATA_APP_TOOLS_TAG}), - ('get_data_apps', True, None, None, {DATA_APP_TOOLS_TAG}), - ('deploy_data_app', None, False, None, {DATA_APP_TOOLS_TAG}), - ], -) -async def test_tool_annotations_tags_values( - tool_name: str, - expected_readonly: bool | None, - expected_destructive: bool | None, - expected_idempotent: bool | None, - tags: set[str], -) -> None: - """ - Test that the tool annotations are having the expected values. - """ - server = create_server(Config(), runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - tools = {t.name: t for t in await server.list_tools(run_middleware=False)} - - # check tool registration - assert tool_name in tools, f'Missing tool registered: {tool_name}' - - # check annotations - tool = tools[tool_name] - if all(exp_val is None for exp_val in (expected_readonly, expected_destructive, expected_idempotent)): - assert tool.annotations is None, f'{tool_name} has annotations' - else: - assert tool.annotations is not None, f'{tool_name} has no annotations' - assert tool.annotations.readOnlyHint is expected_readonly, f'{tool_name}.readOnlyHint mismatch' - assert tool.annotations.destructiveHint is expected_destructive, f'{tool_name}.destructiveHint mismatch' - assert tool.annotations.idempotentHint is expected_idempotent, f'{tool_name}.idempotentHint mismatch' - - # check tags - assert tool.tags == tags, f'{tool_name} tags mismatch' - - -@pytest.mark.asyncio -async def test_json_logging(): - with tempfile.TemporaryDirectory() as tmp_dir: - log_config_file = Path(__file__).parent.parent / 'logging-json.conf' - assert log_config_file.is_file(), f'No logging config file found at {log_config_file.absolute()}' - - tmp_log_config_file = Path(tmp_dir) / 'logging-json.conf' - tmp_log_config_file.write_text(log_config_file.read_text().replace('level=INFO', 'level=DEBUG')) - - # start the MCP server process with json logging - p = subprocess.Popen( - [ - 'python', - '-m', - 'keboola_mcp_server', - '--transport', - 'streamable-http', - '--api-url', - 'http://connection.nowhere', - '--storage-token', - 'foo', - '--log-config', - tmp_log_config_file.absolute(), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - - # Read output streams in background to prevent buffer blocking - stdout_lines = [] - stderr_lines = [] - - async def read_stream(stream, lines_list): - """Read from stream in a non-blocking way""" - loop = asyncio.get_event_loop() - while True: - line = await loop.run_in_executor(None, stream.readline) - if not line: - break - lines_list.append(line) - - stdout_task = asyncio.create_task(read_stream(p.stdout, stdout_lines)) - stderr_task = asyncio.create_task(read_stream(p.stderr, stderr_lines)) - - try: - # Poll until the server is ready (up to 30s) instead of a fixed sleep. - # A fixed sleep is fragile: slow CI runners may need more than 5s to start. - for attempt in range(60): - await asyncio.sleep(0.5) - if p.poll() is not None: - raise RuntimeError(f'MCP server process exited early (rc={p.returncode})') - try: - async with httpx.AsyncClient() as hc: - await hc.get('http://localhost:8000/mcp', timeout=1.0) - break - except (httpx.ConnectError, httpx.TimeoutException): - if attempt == 59: - raise RuntimeError('MCP server did not become ready within 30s') - - # connect to the server and list prompts to force 'fastmcp' logger to get used - # the listing of the prompts does not require SAPI connection - async with Client(StreamableHttpTransport('http://localhost:8000/mcp')) as client: - prompts = await client.list_prompts() - assert len(prompts) > 1 - - finally: - # kill the server and wait for output tasks - p.terminate() - p.wait() - - # Cancel background tasks and collect remaining output - stdout_task.cancel() - stderr_task.cancel() - - stdout = ''.join(stdout_lines) - stderr = ''.join(stderr_lines) - - # Filter out known deprecation warnings (these bypass logging config) - # These warnings come from uvicorn's dependencies or fastmcp and are not actual logging errors - stderr_lines = [ - line - for line in stderr.splitlines() - if not any( - pattern in line - for pattern in [ - 'websockets/legacy/__init__.py', - 'websockets.legacy is deprecated', - 'websockets_impl.py', - 'WebSocketServerProtocol is deprecated', - 'warnings.warn', - 'from websockets.server import WebSocketServerProtocol', - 'FastMCPDeprecationWarning', - 'serializer` parameter is deprecated', - 'FunctionTool.from_function(', - ] - ) - ] - filtered_stderr = '\n'.join(stderr_lines) - - # there is only one handler (the root one) in logging-json.conf which sends messages to stdout - assert filtered_stderr == '', f'Unexpected stderr: {filtered_stderr}' - - # all messages should be JSON-formatted, including those logged by FastMCP loggers - top_names: set[str] = set() - for line in stdout.splitlines(): - message = json.loads(line) - name = message['name'] - top_names.add(name.split('.')[0]) - - missing_top_names = {'fastmcp', 'keboola_mcp_server', 'uvicorn'} - top_names - assert not missing_top_names, f'Missing logger names: {missing_top_names}' diff --git a/tests/test_workspace.py b/tests/test_workspace.py deleted file mode 100644 index 86b3540bf..000000000 --- a/tests/test_workspace.py +++ /dev/null @@ -1,448 +0,0 @@ -import asyncio -from unittest.mock import AsyncMock, Mock, patch -from urllib.parse import urlparse - -import pytest -from httpx import HTTPStatusError, Request, Response - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.query import QueryServiceClient -from keboola_mcp_server.workspace import JobSubmittedInfo, WorkspaceManager, _SnowflakeWorkspace - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('bearer_token', 'storage_token', 'expected_token'), - [ - ('oauth_bearer_123', 'sapi_token_456', 'Bearer oauth_bearer_123'), - (None, 'sapi_token_456', 'sapi_token_456'), - ('', 'sapi_token_456', 'sapi_token_456'), - ], - ids=['with_bearer_token', 'without_bearer_token', 'empty_bearer_token'], -) -async def test_query_client_token_selection(bearer_token: str | None, storage_token: str, expected_token: str): - """Test QueryServiceClient uses bearer token when available, falls back to storage token.""" - # Create mock KeboolaClient with different token configurations - mock_client = Mock(spec=KeboolaClient) - mock_client.token = storage_token - mock_client.bearer_token = bearer_token - mock_client.hostname_suffix = 'keboola.com' - mock_client.branch_id = '12345' - mock_client.headers = {} - - # Create a mock storage client to avoid real API calls - mock_storage_client = Mock() - mock_client.storage_client = mock_storage_client - - # Create workspace instance - workspace = _SnowflakeWorkspace(workspace_id=1, schema='test_schema', client=mock_client) - - # Mock QueryServiceClient.create to capture the token parameter - with patch.object(QueryServiceClient, 'create') as mock_qs_create: - mock_qs_instance = AsyncMock(spec=QueryServiceClient) - mock_qs_instance.branch_id = '12345' - mock_qs_create.return_value = mock_qs_instance - - # Call the method that creates the QueryServiceClient - result = await workspace._create_qs_client() - - # Verify QueryServiceClient.create was called with the expected token - mock_qs_create.assert_called_once() - call_kwargs = mock_qs_create.call_args.kwargs - assert call_kwargs['token'] == expected_token - # Use proper URL parsing instead of substring check to avoid security alerts - parsed_url = urlparse(call_kwargs['root_url']) - assert parsed_url.scheme == 'https' - assert parsed_url.netloc == 'query.keboola.com' - assert call_kwargs['branch_id'] == '12345' - assert result == mock_qs_instance - - -@pytest.mark.asyncio -async def test_query_client_token_selection_with_branch_lookup(): - """Test QueryServiceClient token selection when branch_id needs to be looked up.""" - # Create mock KeboolaClient with bearer token but no branch_id - mock_client = Mock(spec=KeboolaClient) - mock_client.token = 'sapi_token_456' - mock_client.bearer_token = 'oauth_bearer_123' - mock_client.hostname_suffix = 'keboola.com' - mock_client.branch_id = None # No branch_id, will trigger lookup - mock_client.headers = {} - - # Mock storage client with branches_list that returns default branch - mock_storage_client = AsyncMock() - mock_storage_client.branches_list.return_value = [ - {'id': '999', 'isDefault': False}, - {'id': '888', 'isDefault': True}, # Default branch - {'id': '777', 'isDefault': False}, - ] - mock_client.storage_client = mock_storage_client - - # Create workspace instance - workspace = _SnowflakeWorkspace(workspace_id=1, schema='test_schema', client=mock_client) - - # Mock QueryServiceClient.create - with patch.object(QueryServiceClient, 'create') as mock_qs_create: - mock_qs_instance = AsyncMock(spec=QueryServiceClient) - mock_qs_instance.branch_id = '888' - mock_qs_create.return_value = mock_qs_instance - - # Call the method that creates the QueryServiceClient - await workspace._create_qs_client() - - # Verify branch lookup was performed - mock_storage_client.branches_list.assert_called_once() - - # Verify QueryServiceClient.create was called with bearer token and correct branch - mock_qs_create.assert_called_once() - call_kwargs = mock_qs_create.call_args.kwargs - assert call_kwargs['token'] == 'Bearer oauth_bearer_123' - assert call_kwargs['branch_id'] == '888' # Found default branch - - -@pytest.mark.asyncio -@pytest.mark.parametrize('terminal_status', ['canceled', 'cancelled']) -async def test_execute_query_returns_clear_message_when_job_cancelled(terminal_status: str): - """When the QS poll loop exits because the job reached a CANCELLED terminal state - (typically because kai-agent POSTed to Query Service's cancel endpoint after the user - clicked STOP), we must short-circuit the results-fetch and surface a precise - "Query was cancelled" message — not the generic "Job is still running or not completed - yet" that QS returns from get_job_results for non-completed jobs. - - Both 'canceled' (US spelling, the QS canonical form) and 'cancelled' (UK spelling, - seen in the wild) are accepted as terminal-cancel statuses by the poll loop, so both - must take this fast path. - """ - workspace, qs_mock = _make_snowflake_workspace_with_mocked_qs(job_id='job-cancel') - # Override the polling status so the loop exits via the cancelled branch. - qs_mock.get_job_status.return_value = { - 'status': terminal_status, - 'statements': [{'id': 'stmt-1'}], - } - - result = await workspace.execute_query('SELECT SYSTEM$WAIT(300)') - - assert result.is_error - assert result.data is None - assert result.message == 'Query was cancelled' - # The fast path must skip the results fetch entirely; no need to ask QS for rows - # we know don't exist. - qs_mock.get_job_results.assert_not_called() - - -@pytest.mark.asyncio -async def test_workspace_creation_cleans_up_config_on_failure(): - """Test that WorkspaceManager._create_ws cleans up config when workspace creation fails.""" - mock_client = Mock(spec=KeboolaClient) - mock_client.branch_id = None - mock_storage_client = AsyncMock() - mock_client.storage_client = mock_storage_client - - mock_storage_client.verify_token.return_value = {'owner': {'defaultBackend': 'snowflake'}} - mock_storage_client.configuration_create.return_value = {'id': 'test-config-123', 'name': 'test'} - - mock_response = Mock(spec=Response) - mock_response.status_code = 500 - mock_response.text = 'Workspace creation failed' - mock_request = Mock(spec=Request) - mock_request.url = 'https://connection.keboola.com/v2/storage' - mock_storage_client.workspace_create_for_config = AsyncMock( - side_effect=HTTPStatusError('Workspace creation failed', request=mock_request, response=mock_response) - ) - mock_storage_client.configuration_delete = AsyncMock() - - manager = WorkspaceManager(mock_client) - - with pytest.raises(HTTPStatusError): - await manager._create_ws() - - mock_storage_client.configuration_create.assert_called_once() - mock_storage_client.configuration_delete.assert_called_once_with( - WorkspaceManager.MCP_WORKSPACE_COMPONENT_ID, 'test-config-123' - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('input_branch_id', 'has_sb_feature', 'workspace_schema', 'expected_bound_branch_id'), - [ - # default branch: always production, regardless of feature - (None, True, None, None), - (None, False, None, None), - # dev branch + storage-branches feature on: keep dev branch - ('456', True, None, '456'), - # dev branch without storage-branches (legacy): fall back to production - ('456', False, None, None), - # dev branch + storage-branches + explicit workspace_schema (KBC_WORKSPACE_SCHEMA): - # stay branch-aware. The user is responsible for ensuring the named workspace - # exists in the explicitly-bound branch — there is no carve-out for explicit schemas. - ('456', True, 'WORKSPACE_XYZ', '456'), - # dev branch + legacy + explicit workspace_schema: still rebinds to production, - # since branched workspaces don't exist on legacy projects. - ('456', False, 'WORKSPACE_XYZ', None), - ], - ids=[ - 'default_branch_with_sb', - 'default_branch_without_sb', - 'dev_branch_with_sb', - 'dev_branch_legacy', - 'dev_branch_with_sb_explicit_schema', - 'dev_branch_legacy_explicit_schema', - ], -) -async def test_workspace_manager_create_is_branch_aware( - input_branch_id: str | None, - has_sb_feature: bool, - workspace_schema: str | None, - expected_bound_branch_id: str | None, -): - """ - WorkspaceManager.create() must keep the client on the dev branch only when the project - has the `storage-branches` feature; otherwise it must rebind to the production branch. - The rule applies uniformly whether the workspace is auto-managed or pinned via an - explicit `workspace_schema` (KBC_WORKSPACE_SCHEMA) — branch context is governed solely - by KBC_BRANCH_ID and the project's `storage-branches` feature. - """ - input_client = Mock(spec=KeboolaClient) - input_client.branch_id = input_branch_id - input_client.has_feature = AsyncMock(return_value=has_sb_feature) - - # Mirror the real `with_branch_id` semantics: same branch → return self; - # different branch → return a fresh client bound to the requested branch. - def _rebind(target_branch_id: str | None) -> Mock: - if target_branch_id == input_client.branch_id: - return input_client - rebound = Mock(spec=KeboolaClient) - rebound.branch_id = target_branch_id - return rebound - - input_client.with_branch_id = AsyncMock(side_effect=_rebind) - - manager = await WorkspaceManager.create(input_client, workspace_schema=workspace_schema) - - # noinspection PyProtectedMember - bound_client = manager._client - assert bound_client.branch_id == expected_bound_branch_id - # noinspection PyProtectedMember - assert manager._workspace_schema == workspace_schema - - # has_feature is only meaningful when the client is on a dev branch — the helper - # short-circuits otherwise, so on the default branch we should not even ask. - if input_branch_id is None: - input_client.has_feature.assert_not_called() - else: - input_client.has_feature.assert_awaited_once() - - -def _make_snowflake_workspace_with_mocked_qs(job_id: str = 'job-abc-123') -> tuple[_SnowflakeWorkspace, AsyncMock]: - """Builds a _SnowflakeWorkspace whose QueryServiceClient is fully mocked to run a one-row query end to end. - - Returns (workspace, qs_mock) so tests can assert on the mock and access build_cancel_url's return value. - """ - qs_mock = AsyncMock(spec=QueryServiceClient) - qs_mock.submit_job.return_value = job_id - qs_mock.get_job_status.return_value = { - 'status': 'completed', - 'statements': [{'id': 'stmt-1'}], - } - # `data` (not `rows`) matches the QS response shape that - # `_SnowflakeWorkspace.execute_query()` reads via `results.get('data', [])`. - # Keeping the mock aligned with production prevents regressions where the - # results-fetch path silently misses a renamed/missing field. - qs_mock.get_job_results.return_value = { - 'status': 'completed', - 'message': 'ok', - 'numberOfRows': 1, - 'columns': [{'name': 'col'}], - 'data': [['v']], - } - qs_mock.build_cancel_url = Mock(return_value=f'https://query.keboola.com/api/v1/queries/{job_id}/cancel') - - workspace = _SnowflakeWorkspace(workspace_id=1, schema='S', client=Mock(spec=KeboolaClient)) - workspace._qsclient = qs_mock - return workspace, qs_mock - - -@pytest.mark.asyncio -async def test_execute_query_invokes_on_job_submitted_with_full_info(): - """The callback fires exactly once, immediately after submit_job, carrying the cancel URL.""" - workspace, qs_mock = _make_snowflake_workspace_with_mocked_qs(job_id='job-xyz') - - received: list[JobSubmittedInfo] = [] - - async def callback(info: JobSubmittedInfo) -> None: - received.append(info) - - await workspace.execute_query('SELECT 1', on_job_submitted=callback) - - assert len(received) == 1 - assert received[0] == JobSubmittedInfo( - job_id='job-xyz', - cancellation_url='https://query.keboola.com/api/v1/queries/job-xyz/cancel', - backend='snowflake', - ) - qs_mock.build_cancel_url.assert_called_once_with('job-xyz') - - -@pytest.mark.asyncio -async def test_execute_query_without_callback_does_not_call_build_cancel_url(): - """When no callback is supplied, the workspace must not waste a call to build_cancel_url.""" - workspace, qs_mock = _make_snowflake_workspace_with_mocked_qs() - - await workspace.execute_query('SELECT 1') - - qs_mock.build_cancel_url.assert_not_called() - - -@pytest.mark.asyncio -async def test_execute_query_swallows_callback_exception_and_completes_query(): - """A misbehaving callback (network failure when sending the notification, anything) must - never abort the underlying query. The query still completes and returns its result.""" - workspace, qs_mock = _make_snowflake_workspace_with_mocked_qs() - - async def boom(info: JobSubmittedInfo) -> None: - raise RuntimeError('progress send failed') - - result = await workspace.execute_query('SELECT 1', on_job_submitted=boom) - - # The query completed despite the callback error. - assert result.is_ok - qs_mock.get_job_results.assert_awaited() - - -@pytest.mark.asyncio -async def test_execute_query_cancels_backend_when_cancelled_during_callback(): - """If the in-flight task is cancelled while awaiting `on_job_submitted` (the job is already - submitted by then), the backend job must still be cancelled rather than left running.""" - workspace, qs_mock = _make_snowflake_workspace_with_mocked_qs(job_id='job-cb-cancel') - # Make QS report the job as cancelled so `_cancel_job_with_timeout` confirms immediately. - qs_mock.get_job_status.return_value = {'status': 'cancelled', 'statements': [{'id': 'stmt-1'}]} - - async def cancel_during_callback(info: JobSubmittedInfo) -> None: - raise asyncio.CancelledError() - - with pytest.raises(asyncio.CancelledError): - await workspace.execute_query('SELECT 1', on_job_submitted=cancel_during_callback) - - # The backend cancel must have been issued for the already-submitted job. - qs_mock.cancel_job.assert_awaited_once() - assert qs_mock.cancel_job.await_args.args[0] == 'job-cb-cancel' - # The cancellation short-circuits before any results fetch. - qs_mock.get_job_results.assert_not_awaited() - - -def test_build_cancel_url_uses_raw_client_base_api_url(): - """build_cancel_url must produce an absolute URL clients can POST to without further assembly.""" - qs = QueryServiceClient.create( - root_url='https://query.keboola.com', - branch_id='42', - token='Bearer t', - ) - url = qs.build_cancel_url('job-1') - assert url == 'https://query.keboola.com/api/v1/queries/job-1/cancel' - - -@pytest.mark.asyncio -async def test_workspace_manager_execute_query_forwards_callback(): - """WorkspaceManager.execute_query must plumb on_job_submitted through to the active workspace. - - Without this, the tool-layer notification path is silently disabled for any consumer that goes - through the manager (which is everyone, in practice). - """ - workspace, qs_mock = _make_snowflake_workspace_with_mocked_qs(job_id='job-fwd') - - manager = WorkspaceManager(Mock(spec=KeboolaClient)) - manager._workspace = workspace - - received: list[JobSubmittedInfo] = [] - - async def callback(info: JobSubmittedInfo) -> None: - received.append(info) - - await manager.execute_query('SELECT 1', on_job_submitted=callback) - - assert len(received) == 1 - assert received[0].job_id == 'job-fwd' - - -@pytest.mark.asyncio -async def test_workspace_manager_create_skips_feature_lookup_on_default_branch(): - """ - On the default branch `has_storage_branches` short-circuits before calling - `has_feature`, so we should never trigger the underlying verify_token round trip. - """ - input_client = Mock(spec=KeboolaClient) - input_client.branch_id = None - input_client.has_feature = AsyncMock(return_value=True) - - rebound_client = Mock(spec=KeboolaClient) - rebound_client.branch_id = None - input_client.with_branch_id = AsyncMock(return_value=rebound_client) - - await WorkspaceManager.create(input_client) - - input_client.has_feature.assert_not_called() - - -def _make_cancel_test_workspace(*, cancel_job_side_effect=None) -> tuple[_SnowflakeWorkspace, AsyncMock, dict]: - """Build a `_SnowflakeWorkspace` whose `_qsclient` simulates a long-running query. - - The mocked `get_job_status` returns ``running`` until `cancel_job` is invoked, - after which it returns ``canceled`` (mimicking Query Service confirming the cancel). - The ``state`` dict lets the caller inspect whether cancellation was issued. - """ - workspace = _SnowflakeWorkspace(workspace_id=1, schema='test_schema', client=Mock(spec=KeboolaClient)) - mock_qs = AsyncMock(spec=QueryServiceClient) - workspace._qsclient = mock_qs - - mock_qs.submit_job.return_value = 'job-abc-123' - - state = {'cancelled': False} - - async def get_status(job_id: str): - return {'status': 'canceled' if state['cancelled'] else 'running'} - - async def default_cancel(job_id: str, reason: str): - state['cancelled'] = True - return {} - - mock_qs.get_job_status.side_effect = get_status - mock_qs.cancel_job.side_effect = cancel_job_side_effect or default_cancel - return workspace, mock_qs, state - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('cancel_job_side_effect_factory', 'expect_cancel_call'), - [ - (None, True), - ( - lambda: HTTPStatusError( - 'cancel failed', - request=Mock(spec=Request), - response=Mock(spec=Response, status_code=500, text='boom'), - ), - True, - ), - ], - ids=['backend_cancel_succeeds', 'backend_cancel_fails'], -) -async def test_execute_query_cancellation_propagates_to_backend( - cancel_job_side_effect_factory, expect_cancel_call: bool -): - """Client cancellation (MCP `notifications/cancelled`) must trigger `cancel_job` on - the Snowflake side. If the backend cancel itself fails, the original CancelledError - must still propagate so the SDK can finalize the request cleanly.""" - side_effect = cancel_job_side_effect_factory() if cancel_job_side_effect_factory else None - workspace, mock_qs, _state = _make_cancel_test_workspace(cancel_job_side_effect=side_effect) - - task = asyncio.create_task(workspace.execute_query('SELECT 1')) - # Yield to let the task enter the poll loop and issue at least one get_job_status. - await asyncio.sleep(0.05) - - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - - if expect_cancel_call: - mock_qs.cancel_job.assert_called_once_with('job-abc-123', reason='Client cancelled the request') diff --git a/tests/tools/__init__.py b/tests/tools/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/tools/components/__init__.py b/tests/tools/components/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/tools/components/conftest.py b/tests/tools/components/conftest.py deleted file mode 100644 index dcce1b8f8..000000000 --- a/tests/tools/components/conftest.py +++ /dev/null @@ -1,190 +0,0 @@ -from typing import Any - -import pytest -from mcp.server.fastmcp import Context -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.client import KeboolaClient - - -@pytest.fixture -def mock_components() -> list[dict[str, Any]]: - """Mock result of `component_list`""" - return [ - { - 'id': 'keboola.ex-aws-s3', - 'name': 'AWS S3 Extractor', - 'type': 'extractor', - 'description': 'Extract data from AWS S3', - 'version': '1', - }, - { - 'id': 'keboola.wr-google-drive', - 'name': 'Google Drive Writer', - 'type': 'writer', - 'description': 'Write data to Google Drive', - 'version': '1', - }, - { - 'id': 'keboola.app-google-drive', - 'name': 'Google Drive Application', - 'type': 'application', - 'description': 'Application for Google Drive', - 'version': '1', - }, - { - 'id': 'keboola.snowflake-transformation', - 'name': 'Snowflake Transformation', - 'type': 'transformation', - 'description': 'Snowflake SQL transformation', - 'version': '1', - }, - ] - - -@pytest.fixture -def mock_configurations() -> list[dict[str, Any]]: - """Mock result of `configuration_list`""" - return [ - { - 'id': '123', - 'name': 'My Config', - 'description': 'Test configuration', - 'created': '2024-01-01T00:00:00Z', - 'isDisabled': False, - 'isDeleted': False, - 'version': 1, - 'configuration': {}, - }, - { - 'id': '456', - 'name': 'My Config 2', - 'description': 'Test configuration 2', - 'created': '2024-01-01T00:00:00Z', - 'isDisabled': True, - 'isDeleted': True, - 'version': 2, - 'configuration': {}, - }, - ] - - -@pytest.fixture -def mock_component() -> dict[str, Any]: - """Mock result of `component_detail`""" - return { - 'id': 'keboola.ex-aws-s3', - 'name': 'AWS S3 Extractor', - 'type': 'extractor', - 'description': 'Extract data from AWS S3', - 'longDescription': 'Extract data from AWS S3 looooooooong', - 'categories': ['extractor'], - 'version': 1, - 'created': '2024-01-01T00:00:00Z', - 'data': {'data1': 'data1', 'data2': 'data2'}, - 'component_flags': ['flag1', 'flag2'], - 'configurationSchema': {}, - 'configurationDescription': 'Extract data from AWS S3', - 'emptyConfiguration': {}, - 'rootConfigurationExamples': [{'foo': 'root'}], - 'rowConfigurationExamples': [{'foo': 'row'}], - } - - -@pytest.fixture -def mock_tf_component() -> dict[str, Any]: - """Mock result of `component_detail` for a transformation component""" - return { - 'componentId': 'keboola.google-bigquery-transformation', - 'componentType': 'transformation', - 'componentName': 'Google BigQuery', - 'componentCategories': [], - 'description': "BigQuery is Google's fully managed, serverless data warehouse", - 'longDescription': 'Application which runs KBC transformations', - 'documentationUrl': 'https://help.keboola.com/transformations/bigquery', - 'documentation': '---\ntitle: Google BigQuery Transformation\npermalink: /transformations/bigquery/\n---', - 'configurationSchema': {}, - 'configurationRowSchema': {}, - 'configurationDescription': None, - 'rootConfigurationExamples': [], - 'rowConfigurationExamples': [], - 'componentFlags': [ - 'genericDockerUI', - 'genericDockerUI-tableOutput', - 'genericCodeBlocksUI', - 'genericVariablesUI', - 'genericDockerUI-tableInput', - ], - 'data': {}, - } - - -@pytest.fixture -def mock_configuration() -> dict[str, Any]: - """Mock mock_configuration tool.""" - return { - 'id': '123', - 'name': 'My Config', - 'description': 'Test configuration', - 'created': '2024-01-01T00:00:00Z', - 'isDisabled': False, - 'isDeleted': False, - 'version': 1, - 'configuration': {}, - 'rows': [{'id': '1', 'name': 'Row 1', 'version': 1}, {'id': '2', 'name': 'Row 2', 'version': 1}], - } - - -@pytest.fixture -def mock_tf_configuration() -> dict[str, Any]: - """Mock mock_configuration tool.""" - return { - 'id': '124', - 'name': 'My Transformation', - 'description': 'Test transformation configuration', - 'created': '2024-01-01T00:00:00Z', - 'isDisabled': False, - 'isDeleted': False, - 'version': 1, - 'configuration': { - 'parameters': { - 'blocks': [ - { - 'name': 'Blocks', - 'codes': [{'name': 'Code 1', 'script': ['SELECT * FROM customers;', 'SELECT * FROM orders;']}], - }, - ], - }, - 'storage': { - 'input': {'tables': []}, - 'output': {'tables': [{'source': 'customers', 'destination': 'out.c-my-transformation.customers'}]}, - }, - }, - } - - -@pytest.fixture -def mock_metadata() -> list[dict[str, Any]]: - """Mock mock_component_configuration tool.""" - return [ - { - 'id': '1', - 'key': 'test-key', - 'value': 'test-value', - 'provider': 'user', - 'timestamp': '2024-01-01T00:00:00Z', - } - ] - - -@pytest.fixture -def mock_branch_id() -> str: - return 'default' - - -@pytest.fixture -def mcp_context_components_configs(mocker: MockerFixture, mcp_context_client: Context, mock_branch_id: str) -> Context: - keboola_client = mcp_context_client.session.state[KeboolaClient.STATE_KEY] - keboola_client.storage_client.branch_id = mock_branch_id - - return mcp_context_client diff --git a/tests/tools/components/test_sql_utils.py b/tests/tools/components/test_sql_utils.py deleted file mode 100644 index 0095e733b..000000000 --- a/tests/tools/components/test_sql_utils.py +++ /dev/null @@ -1,899 +0,0 @@ -""" -Tests for SQL splitting and joining utilities. - -Ported from the Keboola UI's splitSqlQueries.test.ts to ensure -the Python implementation matches the production-proven JavaScript logic. -""" - -import pytest - -from keboola_mcp_server.tools.components.sql_utils import ( - format_sql, - join_sql_statements, - split_sql_statements, -) - - -@pytest.mark.parametrize( - ('input_sql', 'expected', 'timeout_seconds', 'test_id'), - [ - # Simple queries - ( - '\nSELECT 1;\nSelect 2;\nSELECT 3;', - ['SELECT 1;', 'Select 2;', 'SELECT 3;'], - 1.0, - 'simple_queries', - ), - # Multi-line comments with /* */ syntax - ( - '\nSELECT 1;\n/*\n Select 2;\n*/\nSELECT 3;', - ['SELECT 1;', '/*\n Select 2;\n*/\nSELECT 3;'], - 1.0, - 'multi_line_comments', - ), - # Single line comments with -- syntax - ( - '\nSELECT 1;\n-- Select 2;\nSELECT 3;', - ['SELECT 1;', '-- Select 2;\nSELECT 3;'], - 1.0, - 'single_line_comment_double_dash', - ), - # Single line comments with # syntax - ( - '\nSELECT 1;\n# Select 2;\nSELECT 3;', - ['SELECT 1;', '# Select 2;\nSELECT 3;'], - 1.0, - 'single_line_comment_hash', - ), - # Single line comments with // syntax - ( - '\nSELECT 1;\n// Select 2;\nSELECT 3;', - ['SELECT 1;', '// Select 2;\nSELECT 3;'], - 1.0, - 'single_line_comment_double_slash', - ), - # Dollar-quoted blocks with $$ syntax - ( - '\nSELECT 1;\nexecute immediate $$\n SELECT 2;\n SELECT 3;\n$$;', - ['SELECT 1;', 'execute immediate $$\n SELECT 2;\n SELECT 3;\n$$;'], - 1.0, - 'dollar_quoted_blocks', - ), - # Empty string - ( - '', - [], - 1.0, - 'empty_string', - ), - # Whitespace only - ( - ' ', - [], - 1.0, - 'whitespace_only', - ), - # Single statement without semicolon - ( - 'SELECT 1', - ['SELECT 1'], - 1.0, - 'single_statement_no_semicolon', - ), - # Single statement with semicolon - ( - 'SELECT 1;', - ['SELECT 1;'], - 1.0, - 'single_statement_with_semicolon', - ), - # Semicolons in single-quoted strings - ( - "SELECT 'test;test' AS col1; SELECT 2;", - ["SELECT 'test;test' AS col1;", 'SELECT 2;'], - 1.0, - 'semicolons_in_single_quoted_strings', - ), - # Semicolons in double-quoted strings - ( - 'SELECT "test;test" AS col1; SELECT 2;', - ['SELECT "test;test" AS col1;', 'SELECT 2;'], - 1.0, - 'semicolons_in_double_quoted_strings', - ), - # Escaped quotes in strings - ( - "SELECT 'it\\'s a test'; SELECT 2;", - ["SELECT 'it\\'s a test';", 'SELECT 2;'], - 1.0, - 'escaped_quotes', - ), - # Complex query with timeout - ( - ( - 'SELECT 1;\n-- Comment line\nexecute immediate $$\n SELECT 2;\n ' - "SELECT 'value;still string';\n$$;\nSELECT 3;\n" - '-- Another comment\nSELECT "double" as col;\n' - ), - [ - 'SELECT 1;', - ('-- Comment line\nexecute immediate $$\n SELECT 2;\n ' "SELECT 'value;still string';\n$$;"), - 'SELECT 3;', - '-- Another comment\nSELECT "double" as col;', - ], - 1.0, - 'complex_query_with_timeout', - ), - # Nested dollar quotes - ( - 'CREATE FUNCTION f() $$ SELECT $$nested$$; $$;', - ['CREATE FUNCTION f() $$ SELECT $$nested$$; $$;'], - 1.0, - 'nested_dollar_quotes', - ), - # Mixed single and double quotes - ( - "SELECT 'single', \"double\"; SELECT 2;", - ["SELECT 'single', \"double\";", 'SELECT 2;'], - 1.0, - 'mixed_quotes', - ), - # Windows-style line endings (carriage returns) - ( - 'SELECT 1;\r\nSELECT 2;\r\n', - ['SELECT 1;', 'SELECT 2;'], - 1.0, - 'carriage_returns', - ), - # Complex SQL with division operators and table names containing dashes - ( - ( - 'CREATE TABLE `top_20_products_revenue` AS\n' - 'SELECT\n' - '/* comment */\n' - 'ROW_NUMBER() OVER (ORDER BY SUM(CAST(`line_items_quantity` AS INT64) *' - ' CAST(`line_items_price` AS FLOAT64)) DESC) as revenue_rank,\n' - '`line_items_product_id` as product_id,\n' - '`line_items_title` as product_title,\n' - 'COUNT(DISTINCT `line_items_variant_id`) as variant_count,\n' - 'SUM(CAST(`line_items_quantity` AS INT64)) as total_quantity_sold,\n' - 'ROUND(SUM(CAST(`line_items_quantity` AS INT64) * CAST(`line_items_price` AS FLOAT64)), 2)' - ' as total_revenue,\n' - 'ROUND(AVG(CAST(`line_items_price` AS FLOAT64)), 2) as avg_unit_price,\n' - 'COUNT(*) as total_orders,\n' - 'ROUND(SUM(CAST(`line_items_quantity` AS INT64) * CAST(`line_items_price` AS FLOAT64)) /' - ' SUM(CAST(`line_items_quantity` AS INT64)), 2) as revenue_per_unit,\n' - 'MIN(`created_at`) as first_sale_date,\n' - 'MAX(`updated_at`) as last_sale_date,\n' - 'CURRENT_TIMESTAMP() as report_generated_at\n' - 'FROM `in.c-kds-team-ex-shopify-01k368x27c4gpd4k5v0nwmcn98.orders`\n' - '-- comment\n' - "WHERE `financial_status` IN ('paid', 'partially_paid')\n" - 'AND `line_items_product_id` IS NOT NULL\n' - 'GROUP BY `line_items_product_id`, `line_items_title`\n' - 'QUALIFY revenue_rank <= 20\n' - 'ORDER BY revenue_rank' - ), - [ - ( - 'CREATE TABLE `top_20_products_revenue` AS\n' - 'SELECT\n' - '/* comment */\n' - 'ROW_NUMBER() OVER (ORDER BY SUM(CAST(`line_items_quantity` AS INT64)' - ' * CAST(`line_items_price` AS FLOAT64)) DESC) as revenue_rank,\n' - '`line_items_product_id` as product_id,\n' - '`line_items_title` as product_title,\n' - 'COUNT(DISTINCT `line_items_variant_id`) as variant_count,\n' - 'SUM(CAST(`line_items_quantity` AS INT64)) as total_quantity_sold,\n' - 'ROUND(SUM(CAST(`line_items_quantity` AS INT64) * CAST(`line_items_price` AS FLOAT64)), 2)' - ' as total_revenue,\n' - 'ROUND(AVG(CAST(`line_items_price` AS FLOAT64)), 2) as avg_unit_price,\n' - 'COUNT(*) as total_orders,\n' - 'ROUND(SUM(CAST(`line_items_quantity` AS INT64) * CAST(`line_items_price` AS FLOAT64)) /' - ' SUM(CAST(`line_items_quantity` AS INT64)), 2) as revenue_per_unit,\n' - 'MIN(`created_at`) as first_sale_date,\n' - 'MAX(`updated_at`) as last_sale_date,\n' - 'CURRENT_TIMESTAMP() as report_generated_at\n' - 'FROM `in.c-kds-team-ex-shopify-01k368x27c4gpd4k5v0nwmcn98.orders`\n' - '-- comment\n' - "WHERE `financial_status` IN ('paid', 'partially_paid')\n" - 'AND `line_items_product_id` IS NOT NULL\n' - 'GROUP BY `line_items_product_id`, `line_items_title`\n' - 'QUALIFY revenue_rank <= 20\n' - 'ORDER BY revenue_rank' - ) - ], - 1.0, - 'complex_create_table_with_division_and_dashes', - ), - # Empty strings (single and double quotes) - ( - "SELECT '' AS empty1, \"\" AS empty2; SELECT 2;", - ["SELECT '' AS empty1, \"\" AS empty2;", 'SELECT 2;'], - 1.0, - 'empty_quoted_strings', - ), - # Strings with escaped backslashes - ( - "SELECT 'C:\\\\path\\\\to\\\\file' AS path; SELECT 2;", - ["SELECT 'C:\\\\path\\\\to\\\\file' AS path;", 'SELECT 2;'], - 1.0, - 'strings_with_escaped_backslashes', - ), - # Double-quoted strings with escaped quotes - ( - 'SELECT "test\\"quoted\\"value" AS col; SELECT 2;', - ['SELECT "test\\"quoted\\"value" AS col;', 'SELECT 2;'], - 1.0, - 'double_quoted_with_escaped_quotes', - ), - # Strings containing newlines - ( - "SELECT 'line1\nline2\nline3' AS multiline; SELECT 2;", - ["SELECT 'line1\nline2\nline3' AS multiline;", 'SELECT 2;'], - 1.0, - 'strings_with_newlines', - ), - # Multiple consecutive escaped quotes - ( - "SELECT 'test\\'\\'\\'value' AS col; SELECT 2;", - ["SELECT 'test\\'\\'\\'value' AS col;", 'SELECT 2;'], - 1.0, - 'multiple_escaped_quotes', - ), - # Multi-line comment with only asterisks - ( - 'SELECT 1; /* **** */ SELECT 2;', - ['SELECT 1;', '/* **** */ SELECT 2;'], - 1.0, - 'block_comment_all_asterisks', - ), - # Empty multi-line comment - ( - 'SELECT 1; /**/ SELECT 2;', - ['SELECT 1;', '/**/ SELECT 2;'], - 1.0, - 'empty_block_comment', - ), - # Multi-line comment with asterisks in middle - ( - 'SELECT 1; /* comment with *** asterisks */ SELECT 2;', - ['SELECT 1;', '/* comment with *** asterisks */ SELECT 2;'], - 1.0, - 'block_comment_with_asterisks', - ), - # Comments that look like they might be nested (but aren't) - ( - 'SELECT 1; /* comment /* not nested */ */ SELECT 2;', - ['SELECT 1;', '/* comment /* not nested */ */ SELECT 2;'], - 1.0, - 'block_comment_pseudo_nested', - ), - # Dollar-quoted block with special characters - ( - 'SELECT 1; $$ SELECT "test"; -- comment; $$; SELECT 2;', - ['SELECT 1;', '$$ SELECT "test"; -- comment; $$;', 'SELECT 2;'], - 1.0, - 'dollar_quoted_with_special_chars', - ), - # Dollar-quoted block at start - ( - '$$ SELECT 1; $$; SELECT 2;', - ['$$ SELECT 1; $$;', 'SELECT 2;'], - 1.0, - 'dollar_quoted_at_start', - ), - # Multiple dollar signs in a row (not dollar quotes) - ( - 'SELECT $1, $2, $3; SELECT 2;', - ['SELECT $1, $2, $3;', 'SELECT 2;'], - 1.0, - 'multiple_dollar_signs', - ), - # Division operator in arithmetic - ( - 'SELECT 10 / 2 AS result; SELECT 20 / 4;', - ['SELECT 10 / 2 AS result;', 'SELECT 20 / 4;'], - 1.0, - 'division_operators', - ), - # Negative numbers - ( - 'SELECT -1, -2.5, -10 / 2; SELECT 2;', - ['SELECT -1, -2.5, -10 / 2;', 'SELECT 2;'], - 1.0, - 'negative_numbers', - ), - # Table name with dash (not a comment) - ( - 'SELECT * FROM table-name; SELECT 2;', - ['SELECT * FROM table-name;', 'SELECT 2;'], - 1.0, - 'table_name_with_dash', - ), - # Comment at start of statement - ( - '-- Leading comment\nSELECT 1; SELECT 2;', - ['-- Leading comment\nSELECT 1;', 'SELECT 2;'], - 1.0, - 'comment_at_start', - ), - # Comment at end of statement - ( - 'SELECT 1; -- Trailing comment\nSELECT 2;', - ['SELECT 1;', '-- Trailing comment\nSELECT 2;'], - 1.0, - 'comment_at_end', - ), - # Hash comment at start - ( - '# Leading hash comment\nSELECT 1; SELECT 2;', - ['# Leading hash comment\nSELECT 1;', 'SELECT 2;'], - 1.0, - 'hash_comment_at_start', - ), - # C-style comment at start - ( - '// Leading slash comment\nSELECT 1; SELECT 2;', - ['// Leading slash comment\nSELECT 1;', 'SELECT 2;'], - 1.0, - 'slash_comment_at_start', - ), - # Multiple consecutive statements with comments - ( - 'SELECT 1; /* comment */ SELECT 2; -- comment\nSELECT 3;', - ['SELECT 1;', '/* comment */ SELECT 2;', '-- comment\nSELECT 3;'], - 1.0, - 'multiple_statements_with_comments', - ), - # String containing comment-like text - ( - "SELECT '-- not a comment' AS col; SELECT 2;", - ["SELECT '-- not a comment' AS col;", 'SELECT 2;'], - 1.0, - 'string_with_comment_like_text', - ), - # String containing hash - ( - "SELECT 'price #123' AS col; SELECT 2;", - ["SELECT 'price #123' AS col;", 'SELECT 2;'], - 1.0, - 'string_with_hash', - ), - # String containing slashes - ( - "SELECT 'path/to/file' AS col; SELECT 2;", - ["SELECT 'path/to/file' AS col;", 'SELECT 2;'], - 1.0, - 'string_with_slashes', - ), - # String containing dollar signs - ( - "SELECT 'cost $100' AS col; SELECT 2;", - ["SELECT 'cost $100' AS col;", 'SELECT 2;'], - 1.0, - 'string_with_dollar_signs', - ), - # Mixed quotes and comments - ( - "SELECT 'single' AS s, \"double\" AS d; -- comment\nSELECT 2;", - ["SELECT 'single' AS s, \"double\" AS d;", '-- comment\nSELECT 2;'], - 1.0, - 'mixed_quotes_and_comments', - ), - # Statement with only whitespace before semicolon - ( - 'SELECT 1 ; SELECT 2;', - ['SELECT 1 ;', 'SELECT 2;'], - 1.0, - 'whitespace_before_semicolon', - ), - # Statement with tabs and spaces - ( - '\tSELECT 1;\tSELECT 2;\nSELECT 3;', - ['SELECT 1;', 'SELECT 2;', 'SELECT 3;'], - 1.0, - 'tabs_and_spaces', - ), - # Multiple semicolons (should split) - ( - 'SELECT 1;; SELECT 2;', - ['SELECT 1;', 'SELECT 2;'], - 1.0, - 'multiple_semicolons', - ), - # Unicode characters in strings - ( - "SELECT 'café' AS name, '🚀' AS emoji; SELECT 2;", - ["SELECT 'café' AS name, '🚀' AS emoji;", 'SELECT 2;'], - 1.0, - 'unicode_in_strings', - ), - # Unicode characters in SQL - ( - 'SELECT 1; SELECT 2; -- café comment', - ['SELECT 1;', 'SELECT 2;', '-- café comment'], - 1.0, - 'unicode_in_comments', - ), - # Very long statement (tests performance) - ( - 'SELECT ' + 'x' * 1000 + '; SELECT 2;', - ['SELECT ' + 'x' * 1000 + ';', 'SELECT 2;'], - 1.0, - 'very_long_statement', - ), - # Statement with only a comment - ( - '-- Only comment\nSELECT 1;', - ['-- Only comment\nSELECT 1;'], - 1.0, - 'comment_only_statement', - ), - # Block comment only statement - ( - '/* Only comment */\nSELECT 1;', - ['/* Only comment */\nSELECT 1;'], - 1.0, - 'block_comment_only_statement', - ), - # Multiple block comments - ( - 'SELECT 1; /* comment1 */ SELECT 2; /* comment2 */ SELECT 3;', - ['SELECT 1;', '/* comment1 */ SELECT 2;', '/* comment2 */ SELECT 3;'], - 1.0, - 'multiple_block_comments', - ), - # Dollar-quoted with nested dollar signs - ( - 'SELECT 1; $$ SELECT $variable; $$; SELECT 2;', - ['SELECT 1;', '$$ SELECT $variable; $$;', 'SELECT 2;'], - 1.0, - 'dollar_quoted_with_nested_dollar', - ), - # Complex arithmetic with division and subtraction - ( - 'SELECT (100 - 20) / 2 AS result; SELECT 2;', - ['SELECT (100 - 20) / 2 AS result;', 'SELECT 2;'], - 1.0, - 'complex_arithmetic', - ), - # Mixed line endings - ( - 'SELECT 1;\rSELECT 2;\nSELECT 3;\r\nSELECT 4;', - ['SELECT 1;', 'SELECT 2;', 'SELECT 3;', 'SELECT 4;'], - 1.0, - 'mixed_line_endings', - ), - # Comment with Windows line ending - ( - 'SELECT 1; -- comment\r\nSELECT 2;', - ['SELECT 1;', '-- comment\r\nSELECT 2;'], - 1.0, - 'comment_with_crlf', - ), - # Hash comment with Windows line ending - ( - 'SELECT 1; # comment\r\nSELECT 2;', - ['SELECT 1;', '# comment\r\nSELECT 2;'], - 1.0, - 'hash_comment_with_crlf', - ), - # Multiple statements without semicolons - ( - 'SELECT 1\nSELECT 2\nSELECT 3', - ['SELECT 1\nSELECT 2\nSELECT 3'], - 1.0, - 'multiple_statements_no_semicolons', - ), - # Statement with comment and no semicolon - ( - 'SELECT 1 -- comment\nSELECT 2', - ['SELECT 1 -- comment\nSELECT 2'], - 1.0, - 'statement_with_comment_no_semicolon', - ), - # Empty string between statements - ( - 'SELECT 1;\n\n\nSELECT 2;', - ['SELECT 1;', 'SELECT 2;'], - 1.0, - 'empty_lines_between_statements', - ), - # Statement starting with whitespace - ( - ' SELECT 1; SELECT 2;', - ['SELECT 1;', 'SELECT 2;'], - 1.0, - 'leading_whitespace', - ), - # Block comment spanning multiple lines with complex content - ( - 'SELECT 1; /*\n * Multi-line\n * comment\n * with stars\n */ SELECT 2;', - ['SELECT 1;', '/*\n * Multi-line\n * comment\n * with stars\n */ SELECT 2;'], - 1.0, - 'multi_line_block_comment_with_stars', - ), - ], -) -@pytest.mark.asyncio -async def test_split_sql_statements(input_sql, expected, timeout_seconds, test_id): - """Test SQL splitting with various inputs and scenarios.""" - result = await split_sql_statements(input_sql, timeout_seconds=timeout_seconds) - assert result == expected - - -@pytest.mark.parametrize( - ('statements', 'expected', 'test_id'), - [ - # Empty list - ( - [], - '', - 'empty_list', - ), - # Single statement - ( - ['SELECT 1'], - 'SELECT 1\n\n', - 'single_statement', - ), - # Multiple statements - ( - ['SELECT 1', 'SELECT 2', 'SELECT 3'], - 'SELECT 1\n\nSELECT 2\n\nSELECT 3\n\n', - 'multiple_statements', - ), - # Preserve existing semicolons - ( - ['SELECT 1;', 'SELECT 2;'], - 'SELECT 1;\n\nSELECT 2;\n\n', - 'existing_semicolons', - ), - # Mixed statements (with and without semicolons) - ( - ['SELECT 1;', 'SELECT 2', 'SELECT 3'], - 'SELECT 1;\n\nSELECT 2\n\nSELECT 3\n\n', - 'mixed_statements', - ), - # Filter empty statements - ( - ['SELECT 1', '', ' ', 'SELECT 2'], - 'SELECT 1\n\nSELECT 2\n\n', - 'filter_empty_statements', - ), - # Preserve internal whitespace - ( - ['SELECT \n 1'], - 'SELECT \n 1\n\n', - 'preserve_whitespace', - ), - # Statement with trailing whitespace - ( - ['SELECT 1; ', 'SELECT 2 '], - 'SELECT 1;\n\nSELECT 2\n\n', - 'trailing_whitespace', - ), - # Multiple trailing spaces and tabs - ( - ['SELECT 1 \t ', ' \t SELECT 2'], - 'SELECT 1\n\nSELECT 2\n\n', - 'mixed_whitespace', - ), - # Multiple empty strings (should be filtered) - ( - ['SELECT 1', '', ' ', '\t'], - 'SELECT 1\n\n', - 'with_multiple_empty', - ), - # Pure comment statement (comments are preserved like any other statement) - ( - ['-- This is just a comment', 'SELECT 1'], - '-- This is just a comment\n\nSELECT 1\n\n', - 'pure_comment_statement', - ), - # Multi-line statement with comments (preserved as-is) - ( - ['SELECT a -- comment 1\n, b -- comment 2\nFROM table'], - 'SELECT a -- comment 1\n, b -- comment 2\nFROM table\n\n', - 'multiline_with_comments', - ), - ], -) -def test_join_sql_statements(statements, expected, test_id): - """Test SQL joining with various inputs and scenarios.""" - result = join_sql_statements(statements) - assert result == expected - - -@pytest.mark.parametrize( - ('original', 'test_id'), - [ - # Simple queries - ( - 'SELECT 1;\nSELECT 2;\nSELECT 3;', - 'simple_queries', - ), - # With comments - ( - 'SELECT 1;\n-- comment\nSELECT 2;', - 'with_comments', - ), - # With dollar-quoted blocks - ( - 'SELECT 1;\nexecute immediate $$\n SELECT 2;\n$$;', - 'with_dollar_quotes', - ), - # Complex SQL - ( - "CREATE TABLE test (id INT);\nINSERT INTO test VALUES (1);\nSELECT * FROM test WHERE name = 'test;test';", - 'complex_sql', - ), - ], -) -@pytest.mark.asyncio -async def test_validate_round_trip(original, test_id): - """ - Test round-trip validation: split(join(split(x))) == split(x). - - This ensures that splitting and joining logic is consistent. - """ - # Split original - split_original = await split_sql_statements(original) - - # Join and split again - joined = join_sql_statements(split_original) - split_again = await split_sql_statements(joined) - - # Verify round-trip consistency - assert split_original == split_again - - -@pytest.mark.parametrize( - ('input_sql', 'dialect', 'expected', 'test_id'), - [ - # Pure comment - should not get a semicolon - ( - '-- This is a comment', - 'snowflake', - '-- This is a comment', - 'pure_line_comment', - ), - # Pure block comment - should not get a semicolon - ( - '/* This is a comment */', - 'snowflake', - '/* This is a comment */', - 'pure_block_comment', - ), - # Single statement without semicolon - should add semicolon - ( - 'SELECT 1', - 'snowflake', - 'SELECT\n 1;', - 'single_statement_no_semicolon', - ), - # Single statement with semicolon - should preserve semicolon - ( - 'SELECT 1;', - 'snowflake', - 'SELECT\n 1;', - 'single_statement_with_semicolon', - ), - # Multiple statements without semicolons - should add semicolons - ( - 'SELECT 1; SELECT 2', - 'snowflake', - 'SELECT\n 1;\n\nSELECT\n 2;', - 'multiple_statements_no_semicolons', - ), - # Multiple statements with semicolons - should preserve semicolons - ( - 'SELECT 1; SELECT 2;', - 'snowflake', - 'SELECT\n 1;\n\nSELECT\n 2;', - 'multiple_statements_with_semicolons', - ), - # Statements with inline comment - comment preserved, semicolon added - ( - 'SELECT 1;\n-- comment\nSELECT 2', - 'snowflake', - 'SELECT\n 1;\n\n/* comment */\nSELECT\n 2;', - 'statements_with_inline_comment', - ), - # Statements with inline comment - comment preserved, semicolon added - ( - 'SELECT 1;\n// comment\nSELECT 2', - 'snowflake', - 'SELECT\n 1;\n\n/* comment */\nSELECT\n 2;', - 'statements_with_inline_comment_cpp', - ), - # Complex statement - properly formatted with semicolon - ( - 'SELECT a, b FROM table WHERE x > 10', - 'snowflake', - 'SELECT\n a,\n b\nFROM table\nWHERE\n x > 10;', - 'complex_statement', - ), - # BigQuery dialect tests - # Pure comment - should not get a semicolon - ( - '-- This is a comment', - 'bigquery', - '-- This is a comment', - 'bigquery_pure_line_comment', - ), - # Pure block comment - should not get a semicolon - ( - '/* This is a comment */', - 'bigquery', - '/* This is a comment */', - 'bigquery_pure_block_comment', - ), - # Single statement without semicolon - should add semicolon - ( - 'SELECT 1', - 'bigquery', - 'SELECT\n 1;', - 'bigquery_single_statement_no_semicolon', - ), - # Single statement with semicolon - should preserve semicolon - ( - 'SELECT 1;', - 'bigquery', - 'SELECT\n 1;', - 'bigquery_single_statement_with_semicolon', - ), - # Multiple statements - should add semicolons - ( - 'SELECT 1; SELECT 2', - 'bigquery', - 'SELECT\n 1;\n\nSELECT\n 2;', - 'bigquery_multiple_statements', - ), - # Statements with inline comment - comment preserved, semicolon added - ( - 'SELECT 1;\n-- comment\nSELECT 2', - 'bigquery', - 'SELECT\n 1;\n\n/* comment */\nSELECT\n 2;', - 'bigquery_statements_with_inline_comment', - ), - # Complex statement - properly formatted with semicolon - ( - 'SELECT a, b FROM table WHERE x > 10', - 'bigquery', - 'SELECT\n a,\n b\nFROM table\nWHERE\n x > 10;', - 'bigquery_complex_statement', - ), - # BigQuery-specific: backtick-quoted identifiers - ( - 'SELECT * FROM `project.dataset.table` WHERE id > 10', - 'bigquery', - 'SELECT\n *\nFROM `project.dataset.table`\nWHERE\n id > 10;', - 'bigquery_backtick_identifiers', - ), - # BigQuery-specific: STRUCT syntax - ( - 'SELECT STRUCT(1 AS a, 2 AS b) AS my_struct', - 'bigquery', - 'SELECT\n STRUCT(1 AS a, 2 AS b) AS my_struct;', - 'bigquery_struct_syntax', - ), - # BigQuery-specific: ARRAY syntax (preserved as-is) - ( - 'SELECT [1, 2, 3] AS my_array', - 'bigquery', - 'SELECT\n [1, 2, 3] AS my_array;', - 'bigquery_array_syntax', - ), - ], -) -def test_format_sql(input_sql, dialect, expected, test_id): - """Test SQL formatting with semicolon and comment handling.""" - result = format_sql(input_sql, dialect) - assert result == expected - - -@pytest.mark.parametrize( - ('input_sql', 'dialect', 'test_id'), - [ - # Invalid dialect handling - ( - 'SELECT 1', - 'invalid_dialect_name', - 'invalid_dialect', - ), - ( - 'SELECT * FROM table', - 'nonexistent_sql_dialect', - 'nonexistent_dialect', - ), - # SQL that sqlglot cannot parse - ( - 'SELECT FROM WHERE', - 'snowflake', - 'malformed_sql_missing_table', - ), - ( - 'SELECT * FROM', - 'bigquery', - 'malformed_sql_incomplete', - ), - ( - 'SELECT * FROM table WHERE x =', - 'snowflake', - 'malformed_sql_incomplete_where', - ), - ( - 'CREATE TABLE (id INT)', - 'bigquery', - 'malformed_sql_missing_table_name', - ), - ( - 'SELECT * FROM table WHERE x = (SELECT', - 'snowflake', - 'malformed_sql_unclosed_subquery', - ), - ( - 'SELECT * FROM table WHERE x = "unclosed string', - 'bigquery', - 'malformed_sql_unclosed_string', - ), - ( - "SELECT * FROM table WHERE x = 'unclosed string", - 'snowflake', - 'malformed_sql_unclosed_single_quote', - ), - ( - 'SELECT * FROM table WHERE x = /* unclosed comment', - 'bigquery', - 'malformed_sql_unclosed_comment', - ), - ( - 'SELECT * FROM table WHERE x = $$ unclosed dollar quote', - 'snowflake', - 'malformed_sql_unclosed_dollar_quote', - ), - # Unsupported comment: '#' - ( - 'SELECT 1;\n-- comment1\nSELECT 2;\n# comment2\nSELECT 3;\n// comment3', - 'snowflake', - 'unsupported_comment_hash', - ), - # Empty strings and whitespace-only input - ( - '', - 'snowflake', - 'empty_string', - ), - ( - '', - 'bigquery', - 'empty_string_bigquery', - ), - ( - ' ', - 'snowflake', - 'whitespace_only_spaces', - ), - ( - '\t\t', - 'bigquery', - 'whitespace_only_tabs', - ), - ( - '\n\n\n', - 'snowflake', - 'whitespace_only_newlines', - ), - ( - ' \t\n\r ', - 'bigquery', - 'whitespace_only_mixed', - ), - ], -) -def test_format_sql_error(input_sql, dialect, test_id): - result = format_sql(input_sql, dialect) - # On error, format_sql returns the original SQL unchanged - assert result == input_sql diff --git a/tests/tools/components/test_tf_update.py b/tests/tools/components/test_tf_update.py deleted file mode 100644 index 8b6a43033..000000000 --- a/tests/tools/components/test_tf_update.py +++ /dev/null @@ -1,1255 +0,0 @@ -""" -Tests for transformation parameter update functions. - -Tests all operations for modifying SQL transformation parameters including -block and code management, script updates, and string replacements. -""" - -import copy - -import pytest - -from keboola_mcp_server.tools.components.model import ( - SimplifiedTfBlocks, - TfAddBlock, - TfAddCode, - TfAddScript, - TfRemoveBlock, - TfRemoveCode, - TfRenameBlock, - TfRenameCode, - TfSetCode, - TfStrReplace, -) -from keboola_mcp_server.tools.components.tf_update import ( - add_block, - add_code, - add_script, - remove_block, - remove_code, - rename_block, - rename_code, - set_code, - str_replace, -) - - -@pytest.fixture -def sample_params(): - """Sample transformation parameters with blocks and codes.""" - return { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - { - 'id': 'b1', - 'name': 'Block B', - 'codes': [ - {'id': 'b1.c0', 'name': 'Code Z', 'script': 'SELECT * FROM table3'}, - ], - }, - ] - } - - -@pytest.fixture -def empty_params(): - """Empty transformation parameters.""" - return {'blocks': []} - - -# ============================================================================ -# ADD_BLOCK TESTS -# ============================================================================ - - -@pytest.mark.parametrize( - ('initial_params', 'operation', 'expected_params', 'expected_message'), - [ - # Add block to end of existing blocks - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Existing Block', 'codes': []}, - ] - }, - TfAddBlock( - op='add_block', - block=SimplifiedTfBlocks.Block(name='New Block', codes=[]), - position='end', - ), - { - 'blocks': [ - {'id': 'b0', 'name': 'Existing Block', 'codes': []}, - {'name': 'New Block', 'codes': []}, - ] - }, - 'Added block with name "New Block"', - ), - # Add block to start of existing blocks - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Existing Block', 'codes': []}, - ] - }, - TfAddBlock( - op='add_block', - block=SimplifiedTfBlocks.Block(name='New Block', codes=[]), - position='start', - ), - { - 'blocks': [ - {'name': 'New Block', 'codes': []}, - {'id': 'b0', 'name': 'Existing Block', 'codes': []}, - ] - }, - 'Added block with name "New Block"', - ), - # Add block with multiple codes - ( - {'blocks': []}, - TfAddBlock( - op='add_block', - block=SimplifiedTfBlocks.Block( - name='Multi Code Block', - codes=[ - SimplifiedTfBlocks.Block.Code( - name='Code 1', - script=( - 'SELECT u.id, u.name, COUNT(o.id) as order_count ' - 'FROM users u LEFT JOIN orders o ON u.id = o.user_id ' - "WHERE u.created_at > '2024-01-01' " - 'GROUP BY u.id, u.name HAVING COUNT(o.id) > 5' - ), - ), - SimplifiedTfBlocks.Block.Code( - name='Code 2', - script=( - 'SELECT p.product_name, SUM(oi.quantity * oi.price) as revenue ' - 'FROM products p INNER JOIN order_items oi ON p.id = oi.product_id ' - 'GROUP BY p.product_name ORDER BY revenue DESC LIMIT 10' - ), - ), - ], - ), - ), - { - 'blocks': [ - { - 'name': 'Multi Code Block', - 'codes': [ - { - 'name': 'Code 1', - 'script': ( - 'SELECT u.id, u.name, COUNT(o.id) as order_count ' - 'FROM users u LEFT JOIN orders o ON u.id = o.user_id ' - "WHERE u.created_at > '2024-01-01' " - 'GROUP BY u.id, u.name HAVING COUNT(o.id) > 5' - ), - }, - { - 'name': 'Code 2', - 'script': ( - 'SELECT p.product_name, SUM(oi.quantity * oi.price) as revenue ' - 'FROM products p INNER JOIN order_items oi ON p.id = oi.product_id ' - 'GROUP BY p.product_name ORDER BY revenue DESC LIMIT 10' - ), - }, - ], - }, - ] - }, - 'Added block with name "Multi Code Block"', - ), - ], -) -def test_add_block(initial_params, operation, expected_params, expected_message): - """Test adding blocks to transformation parameters.""" - params = copy.deepcopy(initial_params) - result_params, result_msg = add_block(params, operation, 'snowflake') - assert result_params == expected_params - assert result_msg == expected_message - - -@pytest.mark.parametrize( - ('initial_params', 'block_name', 'error_match'), - [ - # Params without blocks key - ({}, 'First Block', "Invalid parameters: must contain 'blocks' key"), - # Params with other keys but no blocks - ({'other_key': 'value'}, 'First Block', "Invalid parameters: must contain 'blocks' key"), - # Empty block name - ({'blocks': []}, '', 'Invalid operation: block name cannot be empty'), - # Whitespace-only block names - ({'blocks': []}, ' ', 'Invalid operation: block name cannot be empty'), - ({'blocks': []}, '\t', 'Invalid operation: block name cannot be empty'), - ({'blocks': []}, '\n', 'Invalid operation: block name cannot be empty'), - ], -) -def test_add_block_error(initial_params, block_name, error_match): - """Test error cases when adding blocks.""" - params = copy.deepcopy(initial_params) - operation = TfAddBlock( - op='add_block', - block=SimplifiedTfBlocks.Block( - name=block_name, - codes=[SimplifiedTfBlocks.Block.Code(name='First Code', script='SELECT 1')], - ), - position='end', - ) - - with pytest.raises(ValueError, match=error_match): - add_block(params, operation, 'snowflake') - - -# ============================================================================ -# REMOVE_BLOCK TESTS -# ============================================================================ - - -@pytest.mark.parametrize( - ('initial_params', 'operation', 'expected_params'), - [ - # Remove first block - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': []}, - {'id': 'b1', 'name': 'Block B', 'codes': []}, - ] - }, - TfRemoveBlock(op='remove_block', block_id='b0'), - { - 'blocks': [ - {'id': 'b1', 'name': 'Block B', 'codes': []}, - ] - }, - ), - # Remove last block - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': []}, - {'id': 'b1', 'name': 'Block B', 'codes': []}, - ] - }, - TfRemoveBlock(op='remove_block', block_id='b1'), - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': []}, - ] - }, - ), - # Remove middle block - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': []}, - {'id': 'b1', 'name': 'Block B', 'codes': []}, - {'id': 'b2', 'name': 'Block C', 'codes': []}, - ] - }, - TfRemoveBlock(op='remove_block', block_id='b1'), - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': []}, - {'id': 'b2', 'name': 'Block C', 'codes': []}, - ] - }, - ), - # Remove only block - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Only Block', 'codes': []}, - ] - }, - TfRemoveBlock(op='remove_block', block_id='b0'), - {'blocks': []}, - ), - ], -) -def test_remove_block_success(initial_params, operation, expected_params): - """Test successfully removing blocks from transformation parameters.""" - params = copy.deepcopy(initial_params) - result_params, result_msg = remove_block(params, operation, 'snowflake') - assert result_params == expected_params - assert result_msg == '' - - -@pytest.mark.parametrize( - ('initial_params', 'block_id_to_remove'), - [ - # Block not found - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': []}, - ] - }, - 'nonexistent', - ), - # Empty blocks list - ( - {'blocks': []}, - 'b0', - ), - ], -) -def test_remove_block_error(initial_params, block_id_to_remove): - """Test error cases when removing blocks.""" - params = copy.deepcopy(initial_params) - operation = TfRemoveBlock(op='remove_block', block_id=block_id_to_remove) - - with pytest.raises(ValueError, match=f"Block with id '{block_id_to_remove}' does not exist"): - remove_block(params, operation, 'snowflake') - - -# ============================================================================ -# RENAME_BLOCK TESTS -# ============================================================================ - - -@pytest.mark.parametrize( - ('initial_params', 'operation', 'expected_params'), - [ - # Rename first block - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': [{'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT 1'}]}, - {'id': 'b1', 'name': 'Block B', 'codes': []}, - ] - }, - TfRenameBlock(op='rename_block', block_id='b0', block_name='Renamed Block A'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Renamed Block A', - 'codes': [{'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT 1'}], - }, - {'id': 'b1', 'name': 'Block B', 'codes': []}, - ] - }, - ), - # Rename second block - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': []}, - {'id': 'b1', 'name': 'Block B', 'codes': []}, - ] - }, - TfRenameBlock(op='rename_block', block_id='b1', block_name='Renamed Block B'), - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': []}, - {'id': 'b1', 'name': 'Renamed Block B', 'codes': []}, - ] - }, - ), - # Rename with special characters - ( - { - 'blocks': [ - {'id': 'b0', 'name': 'Block A', 'codes': []}, - ] - }, - TfRenameBlock(op='rename_block', block_id='b0', block_name='Block with Special-Chars_123'), - { - 'blocks': [ - {'id': 'b0', 'name': 'Block with Special-Chars_123', 'codes': []}, - ] - }, - ), - ], -) -def test_rename_block_success(initial_params, operation, expected_params): - """Test successfully renaming blocks.""" - params = copy.deepcopy(initial_params) - result_params, result_msg = rename_block(params, operation, 'snowflake') - assert result_params == expected_params - assert result_msg == '' - - -@pytest.mark.parametrize( - ('block_id', 'block_name', 'error_match'), - [ - # Non-existent block IDs - ('nonexistent', 'New Name', "Block with id 'nonexistent' does not exist"), - ('b999', 'New Name', "Block with id 'b999' does not exist"), - # Empty block name - ('b0', '', 'Invalid operation: block name cannot be empty'), - # Whitespace-only block names - ('b0', ' ', 'Invalid operation: block name cannot be empty'), - ('b0', '\t', 'Invalid operation: block name cannot be empty'), - ('b0', '\n', 'Invalid operation: block name cannot be empty'), - ], -) -def test_rename_block_error(sample_params, block_id, block_name, error_match): - """Test error cases when renaming blocks.""" - params = copy.deepcopy(sample_params) - operation = TfRenameBlock(op='rename_block', block_id=block_id, block_name=block_name) - - with pytest.raises(ValueError, match=error_match): - rename_block(params, operation, 'snowflake') - - -# ============================================================================ -# ADD_CODE TESTS -# ============================================================================ - - -@pytest.mark.parametrize( - ('initial_params', 'operation', 'expected_params', 'expected_message'), - [ - # Add code to end - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - ] - }, - TfAddCode( - op='add_code', - block_id='b0', - code=SimplifiedTfBlocks.Block.Code( - name='New Code at End', - script=('SELECT col1 FROM table1;'), - ), - position='end', - ), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - {'name': 'New Code at End', 'script': 'SELECT col1 FROM table1;'}, - ], - }, - ] - }, - 'Added code with name "New Code at End"', - ), - # Add code to start - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - ] - }, - TfAddCode( - op='add_code', - block_id='b0', - code=SimplifiedTfBlocks.Block.Code( - name='New Code at Start', - script=( - 'SELECT DISTINCT category, AVG(price) OVER (PARTITION BY category) as avg_price ' - 'FROM products WHERE in_stock = true ORDER BY category' - ), - ), - position='start', - ), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - { - 'name': 'New Code at Start', - 'script': ( - 'SELECT DISTINCT category, AVG(price) OVER (PARTITION BY category) as avg_price ' - 'FROM products WHERE in_stock = true ORDER BY category' - ), - }, - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - ] - }, - 'Added code with name "New Code at Start"', - ), - ], -) -def test_add_code_success(initial_params, operation, expected_params, expected_message): - """Test successfully adding code to blocks.""" - params = copy.deepcopy(initial_params) - result_params, result_msg = add_code(params, operation, sql_dialect='snowflake') - assert result_params == expected_params - assert result_msg == expected_message - - -@pytest.mark.parametrize( - ('block_id', 'code_name', 'error_match'), - [ - # Non-existent block IDs - ('nonexistent', 'Test Code', "Block with id 'nonexistent' does not exist"), - ('b999', 'Test Code', "Block with id 'b999' does not exist"), - # Empty code name - ('b0', '', 'Invalid operation: code name cannot be empty'), - # Whitespace-only code names - ('b0', ' ', 'Invalid operation: code name cannot be empty'), - ('b0', '\t', 'Invalid operation: code name cannot be empty'), - ('b0', '\n', 'Invalid operation: code name cannot be empty'), - ], -) -def test_add_code_error(sample_params, block_id, code_name, error_match): - """Test error cases when adding code to blocks.""" - params = copy.deepcopy(sample_params) - operation = TfAddCode( - op='add_code', - block_id=block_id, - code=SimplifiedTfBlocks.Block.Code(name=code_name, script='SELECT 1'), - position='end', - ) - - with pytest.raises(ValueError, match=error_match): - add_code(params, operation, sql_dialect='snowflake') - - -# ============================================================================ -# REMOVE_CODE TESTS -# ============================================================================ - - -@pytest.mark.parametrize( - ('initial_params', 'operation', 'expected_params'), - [ - # Remove first code - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - ] - }, - TfRemoveCode(op='remove_code', block_id='b0', code_id='b0.c0'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - ] - }, - ), - # Remove only code in block - ( - { - 'blocks': [ - { - 'id': 'b1', - 'name': 'Block B', - 'codes': [ - {'id': 'b1.c0', 'name': 'Code Z', 'script': 'SELECT * FROM table3'}, - ], - }, - ] - }, - TfRemoveCode(op='remove_code', block_id='b1', code_id='b1.c0'), - { - 'blocks': [ - { - 'id': 'b1', - 'name': 'Block B', - 'codes': [], - }, - ] - }, - ), - ], -) -def test_remove_code_success(initial_params, operation, expected_params): - """Test successfully removing code from blocks.""" - params = copy.deepcopy(initial_params) - result_params, result_msg = remove_code(params, operation, sql_dialect='snowflake') - assert result_params == expected_params - assert result_msg == '' - - -@pytest.mark.parametrize( - ('block_id', 'code_id'), - [ - ('b0', 'nonexistent'), - ('nonexistent', 'b0.c0'), - ('b0', 'b1.c0'), - ], -) -def test_remove_code_error(sample_params, block_id, code_id): - """Test error cases when removing code from blocks.""" - params = copy.deepcopy(sample_params) - operation = TfRemoveCode(op='remove_code', block_id=block_id, code_id=code_id) - - with pytest.raises(ValueError, match=f"Code with id '{code_id}' in block '{block_id}' does not exist"): - remove_code(params, operation, sql_dialect='snowflake') - - -# ============================================================================ -# RENAME_CODE TESTS -# ============================================================================ - - -@pytest.mark.parametrize( - ('initial_params', 'operation', 'expected_params'), - [ - # Rename first code - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - ] - }, - TfRenameCode(op='rename_code', block_id='b0', code_id='b0.c0', code_name='Renamed Code X'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Renamed Code X', 'script': 'SELECT * FROM table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - ] - }, - ), - # Rename second code - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - { - 'id': 'b1', - 'name': 'Block B', - 'codes': [ - {'id': 'b1.c0', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - {'id': 'b1.c1', 'name': 'Code Z', 'script': 'SELECT * FROM table3'}, - ], - }, - ] - }, - TfRenameCode(op='rename_code', block_id='b1', code_id='b1.c1', code_name='Renamed Code Z'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - { - 'id': 'b1', - 'name': 'Block B', - 'codes': [ - {'id': 'b1.c0', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - {'id': 'b1.c1', 'name': 'Renamed Code Z', 'script': 'SELECT * FROM table3'}, - ], - }, - ] - }, - ), - ], -) -def test_rename_code_success(initial_params, operation, expected_params): - """Test successfully renaming code in blocks.""" - params = copy.deepcopy(initial_params) - result_params, result_msg = rename_code(params, operation, sql_dialect='snowflake') - assert result_params == expected_params - assert result_msg == '' - - -@pytest.mark.parametrize( - ('block_id', 'code_id', 'code_name', 'error_match'), - [ - # Non-existent code IDs - ('b0', 'nonexistent', 'New Name', "Code with id 'nonexistent' in block 'b0' does not exist"), - ('nonexistent', 'b0.c0', 'New Name', "Code with id 'b0.c0' in block 'nonexistent' does not exist"), - ('b0', 'b1.c0', 'New Name', "Code with id 'b1.c0' in block 'b0' does not exist"), - # Empty code name - ('b0', 'b0.c0', '', 'Invalid operation: code name cannot be empty'), - # Whitespace-only code names - ('b0', 'b0.c0', ' ', 'Invalid operation: code name cannot be empty'), - ('b0', 'b0.c0', '\t', 'Invalid operation: code name cannot be empty'), - ('b0', 'b0.c0', '\n', 'Invalid operation: code name cannot be empty'), - ], -) -def test_rename_code_error(sample_params, block_id, code_id, code_name, error_match): - """Test error cases when renaming code in blocks.""" - params = copy.deepcopy(sample_params) - operation = TfRenameCode(op='rename_code', block_id=block_id, code_id=code_id, code_name=code_name) - - with pytest.raises(ValueError, match=error_match): - rename_code(params, operation, sql_dialect='snowflake') - - -# ============================================================================ -# SET_CODE TESTS -# ============================================================================ - - -@pytest.mark.parametrize( - ('initial_params', 'operation', 'expected_params', 'expected_message'), - [ - # Set code script - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - ] - }, - TfSetCode(op='set_code', block_id='b0', code_id='b0.c0', script='SELECT * FROM new_table'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM new_table'}, - ], - }, - ] - }, - "Changed code with id 'b0.c0' in block 'b0'", - ), - # Set multiline script - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - ] - }, - TfSetCode(op='set_code', block_id='b0', code_id='b0.c0', script='SELECT *\nFROM table1\nWHERE col = 1'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT *\nFROM table1\nWHERE col = 1'}, - ], - }, - ] - }, - "Changed code with id 'b0.c0' in block 'b0'", - ), - ], -) -def test_set_code_success(initial_params, operation, expected_params, expected_message): - """Test successfully setting code script.""" - params = copy.deepcopy(initial_params) - result_params, result_msg = set_code(params, operation, sql_dialect='snowflake') - assert result_params == expected_params - assert result_msg == expected_message - - -@pytest.mark.parametrize( - ('block_id', 'code_id', 'script', 'error_match'), - [ - ('b0', 'nonexistent', 'SELECT 1', "Code with id 'nonexistent' in block 'b0' does not exist"), - ('nonexistent', 'b0.c0', 'SELECT 1', "Code with id 'b0.c0' in block 'nonexistent' does not exist"), - ('b0', 'b0.c0', '', 'Invalid operation: script cannot be empty'), - ('b0', 'b0.c0', ' ', 'Invalid operation: script cannot be empty'), - ], -) -def test_set_code_error(sample_params, block_id, code_id, script, error_match): - """Test error cases when setting code script.""" - params = copy.deepcopy(sample_params) - operation = TfSetCode(op='set_code', block_id=block_id, code_id=code_id, script=script) - - with pytest.raises(ValueError, match=error_match): - set_code(params, operation, sql_dialect='snowflake') - - -# ============================================================================ -# ADD_SCRIPT TESTS -# ============================================================================ - - -@pytest.mark.parametrize( - ('initial_params', 'operation', 'expected_params', 'expected_message'), - [ - # Append to existing script - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - ] - }, - TfAddScript(op='add_script', block_id='b0', code_id='b0.c0', script='WHERE col = 1'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - { - 'id': 'b0.c0', - 'name': 'Code X', - 'script': 'SELECT * FROM table1 WHERE col = 1', - }, - ], - }, - ] - }, - "Added script to code with id 'b0.c0' in block 'b0'", - ), - # Prepend to existing script - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - ] - }, - TfAddScript( - op='add_script', block_id='b0', code_id='b0.c0', script='SELECT * FROM table0;', position='start' - ), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - { - 'id': 'b0.c0', - 'name': 'Code X', - 'script': 'SELECT * FROM table0; SELECT * FROM table1', - }, - ], - }, - ] - }, - "Added script to code with id 'b0.c0' in block 'b0'", - ), - # Prepend to existing script (creates invalid SQL, not reformatted) - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - ] - }, - TfAddScript( - op='add_script', block_id='b0', code_id='b0.c0', script='SELECT * FROM table0', position='start' - ), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - { - 'id': 'b0.c0', - 'name': 'Code X', - 'script': 'SELECT * FROM table0 SELECT * FROM table1', - }, - ], - }, - ] - }, - "Added script to code with id 'b0.c0' in block 'b0'", - ), - # Append to empty script - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': ''}, - ], - }, - ] - }, - TfAddScript(op='add_script', block_id='b0', code_id='b0.c0', script='SELECT 1', position='end'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT 1'}, - ], - }, - ] - }, - "Added script to code with id 'b0.c0' in block 'b0'", - ), - ], -) -def test_add_script_success(initial_params, operation, expected_params, expected_message): - """Test successfully adding script to code.""" - params = copy.deepcopy(initial_params) - result_params, result_msg = add_script(params, operation, sql_dialect='snowflake') - assert result_params == expected_params - assert result_msg == expected_message - - -@pytest.mark.parametrize( - ('block_id', 'code_id', 'script', 'error_match'), - [ - # Non-existent code IDs - ('b0', 'nonexistent', 'SELECT 1', "Code with id 'nonexistent' in block 'b0' does not exist"), - ('nonexistent', 'b0.c0', 'SELECT 1', "Code with id 'b0.c0' in block 'nonexistent' does not exist"), - # Empty script - ('b0', 'b0.c0', '', 'Invalid operation: script cannot be empty'), - # Whitespace-only scripts - ('b0', 'b0.c0', ' ', 'Invalid operation: script cannot be empty'), - ('b0', 'b0.c0', '\t', 'Invalid operation: script cannot be empty'), - ('b0', 'b0.c0', '\n', 'Invalid operation: script cannot be empty'), - ], -) -def test_add_script_error(sample_params, block_id, code_id, script, error_match): - """Test error cases when adding script to code.""" - params = copy.deepcopy(sample_params) - operation = TfAddScript( - op='add_script', - block_id=block_id, - code_id=code_id, - script=script, - position='end', - ) - - with pytest.raises(ValueError, match=error_match): - add_script(params, operation, sql_dialect='snowflake') - - -# ============================================================================ -# STR_REPLACE TESTS -# ============================================================================ - - -@pytest.mark.parametrize( - ('initial_params', 'operation', 'expected_params', 'expected_msg'), - [ - # Replace in all blocks and codes - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - { - 'id': 'b1', - 'name': 'Block B', - 'codes': [ - {'id': 'b1.c0', 'name': 'Code Z', 'script': 'SELECT * FROM table3'}, - ], - }, - ] - }, - TfStrReplace(op='str_replace', block_id=None, code_id=None, search_for='FROM', replace_with='IN'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * IN table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * IN table2'}, - ], - }, - { - 'id': 'b1', - 'name': 'Block B', - 'codes': [ - {'id': 'b1.c0', 'name': 'Code Z', 'script': 'SELECT * IN table3'}, - ], - }, - ] - }, - 'Replaced 3 occurrences of "FROM" in the transformation', - ), - # Replace in specific block - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - { - 'id': 'b1', - 'name': 'Block B', - 'codes': [ - {'id': 'b1.c0', 'name': 'Code Z', 'script': 'SELECT * FROM table3'}, - ], - }, - ] - }, - TfStrReplace(op='str_replace', block_id='b0', code_id=None, search_for='FROM', replace_with='IN'), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * IN table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * IN table2'}, - ], - }, - { - 'id': 'b1', - 'name': 'Block B', - 'codes': [ - {'id': 'b1.c0', 'name': 'Code Z', 'script': 'SELECT * FROM table3'}, - ], - }, - ] - }, - 'Replaced 2 occurrences of "FROM" in block "b0"', - ), - # Replace in specific code - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - ] - }, - TfStrReplace( - op='str_replace', block_id='b0', code_id='b0.c0', search_for='table1', replace_with='new_table1' - ), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM new_table1'}, - {'id': 'b0.c1', 'name': 'Code Y', 'script': 'SELECT * FROM table2'}, - ], - }, - ] - }, - 'Replaced 1 occurrence of "table1" in code "b0.c0", block "b0"', - ), - # Replace with empty string - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'SELECT * FROM table1'}, - ], - }, - ] - }, - TfStrReplace(op='str_replace', block_id='b0', code_id='b0.c0', search_for='SELECT * ', replace_with=''), - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block A', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code X', 'script': 'FROM table1'}, - ], - }, - ] - }, - 'Replaced 1 occurrence of "SELECT * " in code "b0.c0", block "b0"', - ), - ], -) -def test_str_replace_success(initial_params, operation, expected_params, expected_msg): - """Test successfully replacing strings in scripts.""" - params = copy.deepcopy(initial_params) - result_params, result_msg = str_replace(params, operation, sql_dialect='snowflake') - assert result_params == expected_params - assert result_msg == expected_msg - - -@pytest.mark.parametrize( - ('block_id', 'code_id', 'search_for', 'replace_with', 'error_match'), - [ - # Empty search string - ('b0', 'b0.c0', '', 'replacement', 'Invalid operation: search string is empty'), - # Search and replace are the same - ('b0', 'b0.c0', 'table', 'table', 'Invalid operation: search string and replace string are the same'), - # Search string not found - ('b0', 'b0.c0', 'nonexistent', 'replacement', 'Search string "nonexistent" not found'), - # Invalid block ID - ('nonexistent', None, 'table', 'new_table', 'No scripts found'), - # Invalid code ID - ('b0', 'nonexistent', 'table', 'new_table', 'No scripts found'), - ], -) -def test_str_replace_error(sample_params, block_id, code_id, search_for, replace_with, error_match): - """Test error cases when replacing strings in scripts.""" - params = copy.deepcopy(sample_params) - operation = TfStrReplace( - op='str_replace', - block_id=block_id, - code_id=code_id, - search_for=search_for, - replace_with=replace_with, - ) - - with pytest.raises(ValueError, match=error_match): - str_replace(params, operation, sql_dialect='snowflake') - - -# ============================================================================ -# INTEGRATION TESTS -# ============================================================================ - - -def test_multiple_operations_sequence(sample_params): - """Test applying multiple operations in sequence.""" - params = copy.deepcopy(sample_params) - - # 1. Add a new block - params, _ = add_block( - params, - TfAddBlock( - op='add_block', - block=SimplifiedTfBlocks.Block( - name='New Block', - codes=[SimplifiedTfBlocks.Block.Code(name='New Code', script='SELECT 1')], - ), - position='end', - ), - 'snowflake', - ) - assert len(params['blocks']) == 3 - - # 2. Rename an existing block - params, _ = rename_block( - params, TfRenameBlock(op='rename_block', block_id='b0', block_name='Renamed Block A'), sql_dialect='snowflake' - ) - assert params['blocks'][0]['name'] == 'Renamed Block A' - - # 3. Add code to existing block - params, _ = add_code( - params, - TfAddCode( - op='add_code', - block_id='b0', - code=SimplifiedTfBlocks.Block.Code(name='Additional Code', script='SELECT * FROM new_table'), - position='end', - ), - sql_dialect='snowflake', - ) - - # 4. Replace string in all scripts - params, _ = str_replace( - params, - TfStrReplace(op='str_replace', block_id=None, code_id=None, search_for='FROM', replace_with='IN'), - sql_dialect='snowflake', - ) - - # Verify final state - assert len(params['blocks']) == 3 - assert len(params['blocks'][0]['codes']) == 3 # Original 2 + 1 added - # Verify string replacement worked - assert 'IN' in params['blocks'][0]['codes'][0]['script'] - - -def test_operations_preserve_unaffected_data(sample_params): - """Test that operations don't modify unrelated blocks or codes.""" - params = copy.deepcopy(sample_params) - - # Store original second block - original_second_block = copy.deepcopy(params['blocks'][1]) - - # Modify first block - params, _ = rename_block( - params, TfRenameBlock(op='rename_block', block_id='b0', block_name='Modified Block'), sql_dialect='snowflake' - ) - - # Verify second block unchanged - assert params['blocks'][1] == original_second_block diff --git a/tests/tools/components/test_tools.py b/tests/tools/components/test_tools.py deleted file mode 100644 index ba01e47e9..000000000 --- a/tests/tools/components/test_tools.py +++ /dev/null @@ -1,2669 +0,0 @@ -import asyncio -from typing import Any, Callable -from unittest.mock import MagicMock - -import httpx -import pytest -from fastmcp.exceptions import ToolError -from mcp.server.fastmcp import Context -from pydantic import ValidationError -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.client import ( - CONDITIONAL_FLOW_COMPONENT_ID, - DATA_APP_COMPONENT_ID, - ORCHESTRATOR_COMPONENT_ID, - KeboolaClient, -) -from keboola_mcp_server.clients.encryption import REDACTED_SECRET_VALUE -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.components.api_models import ConfigurationAPIResponse -from keboola_mcp_server.tools.components.model import ( - Component, - ComponentCapabilities, - ComponentSummary, - ComponentType, - ComponentWithConfigs, - ConfigParamRemove, - ConfigParamReplace, - ConfigParamSet, - ConfigParamUpdate, - ConfigSummary, - ConfigToolOutput, - Configuration, - ConfigurationRoot, - ConfigurationRootSummary, - ConfigurationRow, - FullConfigId, - GetComponentsOutput, - GetConfigsDetailOutput, - GetConfigsListOutput, - SimplifiedTfBlocks, - TfParamUpdate, - TfRenameBlock, - TfSetCode, - TfStrReplace, - VariableDefinition, -) -from keboola_mcp_server.tools.components.tools import ( - add_config_row, - create_config, - create_sql_transformation, - get_components, - get_config_examples, - get_configs, - run_sync_action, - update_config, - update_config_row, - update_sql_transformation, -) -from keboola_mcp_server.tools.components.utils import ( - BIGQUERY_TRANSFORMATION_ID, - FOLDER_SUPPORTING_COMPONENT_IDS, - SNOWFLAKE_TRANSFORMATION_ID, - VARIABLES_COMPONENT_ID, - clean_bucket_name, -) -from keboola_mcp_server.workspace import WorkspaceManager - -# ============================================================================ -# get_configs TESTS -# ============================================================================ - - -@pytest.fixture -def assert_get_configs_list() -> Callable[ - [ - GetConfigsListOutput, - list[dict[str, Any]], - list[dict[str, Any]], - ], - None, -]: - """Assert that the get_configs tool (list mode) returns the correct components and configurations.""" - - def _assert_get_configs_list( - result: GetConfigsListOutput, - components: list[dict[str, Any]], - configurations: list[dict[str, Any]], - ): - components_with_configs = result.components_with_configs - - assert len(components_with_configs) == len(components) - # assert basics - assert all(isinstance(component, ComponentWithConfigs) for component in components_with_configs) - assert all(isinstance(component.component, ComponentSummary) for component in components_with_configs) - assert all(isinstance(component.configs, list) for component in components_with_configs) - assert all( - all(isinstance(config, ConfigSummary) for config in component.configs) - for component in components_with_configs - ) - # assert component list details - assert all( - returned.component.component_id == expected['id'] - for returned, expected in zip(components_with_configs, components) - ) - assert all( - returned.component.component_name == expected['name'] - for returned, expected in zip(components_with_configs, components) - ) - assert all( - returned.component.component_type == expected['type'] - for returned, expected in zip(components_with_configs, components) - ) - assert all(not hasattr(returned.component, 'version') for returned in components_with_configs) - - # assert configurations list details - assert all(len(component.configs) == len(configurations) for component in components_with_configs) - assert all( - all(isinstance(config.configuration_root, ConfigurationRootSummary) for config in component.configs) - for component in components_with_configs - ) - # use zip to iterate over the result and mock_configurations since we artificially mock the .get method - assert all( - all( - config.configuration_root.configuration_id == expected['id'] - for config, expected in zip(component.configs, configurations) - ) - for component in components_with_configs - ) - assert all( - all( - config.configuration_root.name == expected['name'] - for config, expected in zip(component.configs, configurations) - ) - for component in components_with_configs - ) - - return _assert_get_configs_list - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('component_types', 'expected_types', 'expected_mock_comp_idxs'), - [ - # No filter - should retrieve all component types (including transformation) - # Order: application, extractor, transformation, writer - ([], ['application', 'extractor', 'transformation', 'writer'], [2, 0, 3, 1]), - # Single type - extractor only - (['extractor'], ['extractor'], [0]), - # Single type - writer only - (['writer'], ['writer'], [1]), - # Single type - application only - (['application'], ['application'], [2]), - # Single type - transformation only - (['transformation'], ['transformation'], [3]), - # Multiple types - extractor and writer - # Order: extractor, writer - (['extractor', 'writer'], ['extractor', 'writer'], [0, 1]), - # Multiple types - extractor, writer, and application - # Order: application, extractor, writer - (['extractor', 'writer', 'application'], ['application', 'extractor', 'writer'], [2, 0, 1]), - ], -) -async def test_get_configs_by_types( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_components: list[dict[str, Any]], - mock_configurations: list[dict[str, Any]], - assert_get_configs_list: Callable[[GetConfigsListOutput, list[dict[str, Any]], list[dict[str, Any]]], None], - component_types: list[ComponentType], - expected_types: list[ComponentType], - expected_mock_comp_idxs: list[int], -): - """ - Test get_configs (list mode) when component types are provided with various filters. - The expected_mock_comp_idxs are the indices of mock_components that should be returned. - """ - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - # Create a mapping of component_type to the matching mock component - component_type_map = {comp['type']: comp for comp in mock_components} - - # Create a side_effect function that returns the correct component based on component_type - async def mock_component_list(component_type: ComponentType, include: list[str] | None = None): - # Return matching component or empty list if no transformation exists - if component_type in component_type_map: - return [{**component_type_map[component_type], 'configurations': mock_configurations}] - return [] - - keboola_client.storage_client.component_list = mocker.AsyncMock(side_effect=mock_component_list) - - result = await get_configs(ctx=context, component_types=component_types) - - # Verify we get the list output type - assert isinstance(result, GetConfigsListOutput) - - # Get the expected components based on the indices - expected_components = [mock_components[i] for i in expected_mock_comp_idxs] - assert_get_configs_list(result, expected_components, mock_configurations) - - # Verify the calls were made with the correct arguments (in sorted order) - expected_calls = [mocker.call(component_type=comp_type, include=['configuration']) for comp_type in expected_types] - keboola_client.storage_client.component_list.assert_has_calls(expected_calls) - - -@pytest.mark.asyncio -async def test_get_configs_by_component_ids( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_configurations: list[dict[str, Any]], - mock_component: dict[str, Any], - assert_get_configs_list: Callable[[GetConfigsListOutput, list[dict[str, Any]], list[dict[str, Any]]], None], -): - """Test get_configs (list mode) when component IDs are provided.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=mock_configurations) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - - result = await get_configs(ctx=context, component_ids=[mock_component['id']]) - - # Verify we get the list output type - assert isinstance(result, GetConfigsListOutput) - - assert_get_configs_list(result, [mock_component], mock_configurations) - - # Verify the calls were made with the correct arguments - keboola_client.storage_client.configuration_list.assert_called_once_with(component_id=mock_component['id']) - keboola_client.storage_client.component_detail.assert_called_once_with(component_id=mock_component['id']) - - -@pytest.mark.asyncio -async def test_get_configs_detail( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_configuration: dict[str, Any], - mock_component: dict[str, Any], - mock_metadata: list[dict[str, Any]], -): - """Test get_configs (detail mode) when specific configs are provided.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - # Get URL components from context for link assertions - storage_api_url = keboola_client.storage_api_url - project_id = await keboola_client.storage_client.project_id() - base_url = f'{storage_api_url}/admin/projects/{project_id}' - - mock_ai_service = mocker.MagicMock() - mock_ai_service.get_component_detail = mocker.AsyncMock(return_value=mock_component) - - keboola_client.ai_service_client = mock_ai_service - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - # mock the configuration_detail method to return the mock_configuration - # simulate the response from the API - keboola_client.storage_client.configuration_detail = mocker.AsyncMock( - return_value={**mock_configuration, 'component': mock_component, 'configurationMetadata': mock_metadata} - ) - - configs = [FullConfigId(component_id=mock_component['id'], configuration_id=mock_configuration['id'])] - result = await get_configs(ctx=context, configs=configs) - - # Verify we get the detail output type - assert isinstance(result, GetConfigsDetailOutput) - assert len(result.configs) == 1 - - config = result.configs[0] - assert config.configuration_root.configuration_id == mock_configuration['id'] - assert config.configuration_root.name == mock_configuration['name'] - assert config.component is not None - assert config.component.component_id == mock_component['id'] - assert config.component.component_name == mock_component['name'] - - # Verify links - assert set(config.links) == { - Link( - type='ui-detail', - title=f'Configuration: {mock_configuration["name"]}', - url=f'{base_url}/components/{mock_component["id"]}/{mock_configuration["id"]}', - ), - Link( - type='ui-dashboard', - title=f'Component "{mock_component["id"]}" Configurations Dashboard', - url=f'{base_url}/components/{mock_component["id"]}', - ), - } - - # Verify the calls were made with the correct arguments - keboola_client.storage_client.configuration_detail.assert_called_once_with( - component_id=mock_component['id'], configuration_id=mock_configuration['id'] - ) - - -@pytest.mark.asyncio -async def test_get_configs_detail_multiple( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_configuration: dict[str, Any], - mock_component: dict[str, Any], - mock_metadata: list[dict[str, Any]], -): - """Test get_configs (detail mode) when multiple specific configs are provided.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - # Create a second configuration - mock_configuration_2 = {**mock_configuration, 'id': '456', 'name': 'My Config 2'} - - mock_ai_service = mocker.MagicMock() - mock_ai_service.get_component_detail = mocker.AsyncMock(return_value=mock_component) - - keboola_client.ai_service_client = mock_ai_service - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - - # Return different configs based on the configuration_id - async def mock_config_detail(component_id: str, configuration_id: str): - if configuration_id == mock_configuration['id']: - return {**mock_configuration, 'component': mock_component, 'configurationMetadata': mock_metadata} - else: - return {**mock_configuration_2, 'component': mock_component, 'configurationMetadata': mock_metadata} - - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_config_detail) - - configs = [ - FullConfigId(component_id=mock_component['id'], configuration_id=mock_configuration['id']), - FullConfigId(component_id=mock_component['id'], configuration_id=mock_configuration_2['id']), - ] - result = await get_configs(ctx=context, configs=configs) - - # Verify we get the detail output type with multiple configs - assert isinstance(result, GetConfigsDetailOutput) - assert len(result.configs) == 2 - - # Verify both configs are present - config_ids = {c.configuration_root.configuration_id for c in result.configs} - assert config_ids == {mock_configuration['id'], mock_configuration_2['id']} - - # Verify each config has the expected data - for config in result.configs: - assert isinstance(config, Configuration) - assert config.component is not None - assert config.component.component_id == mock_component['id'] - - -@pytest.mark.asyncio -async def test_get_configs_detail_transformation( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_tf_configuration: dict[str, Any], - mock_tf_component: dict[str, Any], - mock_metadata: list[dict[str, Any]], -): - """ - Test get_configs (detail mode) for transformations. - We test that the transformation parameters are correctly simplified and IDs are added. - """ - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - mock_ai_service = mocker.MagicMock() - mock_ai_service.get_component_detail = mocker.AsyncMock(return_value=mock_tf_component) - - keboola_client.ai_service_client = mock_ai_service - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_tf_component) - # mock the configuration_detail method to return the mock_configuration - # simulate the response from the API - keboola_client.storage_client.configuration_detail = mocker.AsyncMock( - return_value={**mock_tf_configuration, 'component': mock_tf_component, 'configurationMetadata': mock_metadata} - ) - - configs = [ - FullConfigId(component_id=mock_tf_component['componentId'], configuration_id=mock_tf_configuration['id']) - ] - result = await get_configs(ctx=context, configs=configs) - - # Verify we get the detail output type - assert isinstance(result, GetConfigsDetailOutput) - assert len(result.configs) == 1 - - config = result.configs[0] - assert isinstance(config, Configuration) - assert config.configuration_root.parameters == { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Blocks', - 'codes': [ - {'id': 'b0.c0', 'name': 'Code 1', 'script': 'SELECT * FROM customers;\n\nSELECT * FROM orders;\n\n'} - ], - } - ], - } - - -@pytest.mark.asyncio -async def test_get_configs_detail_ignores_other_params( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_configuration: dict[str, Any], - mock_component: dict[str, Any], - mock_metadata: list[dict[str, Any]], -): - """Test that get_configs (detail mode) ignores component_types and component_ids when configs is provided.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - mock_ai_service = mocker.MagicMock() - mock_ai_service.get_component_detail = mocker.AsyncMock(return_value=mock_component) - - keboola_client.ai_service_client = mock_ai_service - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock( - return_value={**mock_configuration, 'component': mock_component, 'configurationMetadata': mock_metadata} - ) - keboola_client.storage_client.component_list = mocker.AsyncMock() - keboola_client.storage_client.configuration_list = mocker.AsyncMock() - - configs = [FullConfigId(component_id=mock_component['id'], configuration_id=mock_configuration['id'])] - - # Provide all params, but configs takes precedence - result = await get_configs( - ctx=context, - component_types=['extractor', 'writer'], # Should be ignored - component_ids=['some-other-component'], # Should be ignored - configs=configs, # This should be used - ) - - # Verify we get the detail output type - assert isinstance(result, GetConfigsDetailOutput) - assert len(result.configs) == 1 - - # Verify that component_list and configuration_list were NOT called (because configs takes precedence) - keboola_client.storage_client.component_list.assert_not_called() - keboola_client.storage_client.configuration_list.assert_not_called() - - # Verify configuration_detail was called for the specified config - keboola_client.storage_client.configuration_detail.assert_called_once_with( - component_id=mock_component['id'], configuration_id=mock_configuration['id'] - ) - - -@pytest.mark.asyncio -async def test_get_configs_detail_empty_list_processors( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_metadata: list[dict[str, Any]], -): - """get_configs must not raise when the Storage API returns processors=[] on a row.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - mock_ai_service = mocker.MagicMock() - mock_ai_service.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.ai_service_client = mock_ai_service - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - - config_with_empty_processors = { - 'id': '123', - 'name': 'My Config', - 'description': 'Test', - 'version': 1, - 'isDisabled': False, - 'isDeleted': False, - 'configuration': {'processors': []}, - 'rows': [ - { - 'id': 'row-1', - 'name': 'Row 1', - 'version': 1, - 'configuration': {'parameters': {}, 'processors': []}, - } - ], - } - keboola_client.storage_client.configuration_detail = mocker.AsyncMock( - return_value={ - **config_with_empty_processors, - 'component': mock_component, - 'configurationMetadata': mock_metadata, - } - ) - - configs = [FullConfigId(component_id=mock_component['id'], configuration_id='123')] - result = await get_configs(ctx=context, configs=configs) - - assert isinstance(result, GetConfigsDetailOutput) - assert len(result.configs) == 1 - config = result.configs[0] - config_root = config.configuration_root - assert config_root.configuration_id == '123' - assert config_root.processors is None - assert config.configuration_rows is not None - assert len(config.configuration_rows) == 1 - assert config.configuration_rows[0].processors is None - - -@pytest.mark.parametrize( - ('processors_value', 'expected'), - [ - ([], None), - (None, None), - ('omit', None), - ({'after': [{'definition': {'component': 'x'}}]}, {'after': [{'definition': {'component': 'x'}}]}), - ], - ids=['empty_list_normalized', 'none_passthrough', 'field_omitted', 'dict_passthrough'], -) -def test_configuration_root_processors_normalization(processors_value: Any, expected: Any) -> None: - """ConfigurationRoot.from_api_response normalizes Storage API processors=[] but preserves dicts.""" - api_config = ConfigurationAPIResponse.model_validate( - { - 'componentId': 'keboola.ex-aws-s3', - 'id': '123', - 'name': 'My Config', - 'version': 1, - 'configuration': {'processors': processors_value} if processors_value != 'omit' else {}, - 'metadata': [], - } - ) - root = ConfigurationRoot.from_api_response(api_config) - assert root.processors == expected - - -@pytest.mark.parametrize( - 'invalid_processors', - [ - ['unexpected'], - [{'after': []}], - ], - ids=['non_empty_list_of_str', 'non_empty_list_of_dict'], -) -def test_configuration_root_rejects_non_empty_list_processors(invalid_processors: list) -> None: - """A non-empty list at the root must fail Pydantic validation, not be silently accepted.""" - api_config = ConfigurationAPIResponse.model_validate( - { - 'componentId': 'keboola.ex-aws-s3', - 'id': '123', - 'name': 'My Config', - 'version': 1, - 'configuration': {'processors': invalid_processors}, - 'metadata': [], - } - ) - with pytest.raises(ValidationError): - ConfigurationRoot.from_api_response(api_config) - - -@pytest.mark.parametrize( - 'invalid_processors', - [ - ['unexpected'], - [{'after': []}], - ], - ids=['non_empty_list_of_str', 'non_empty_list_of_dict'], -) -def test_configuration_row_rejects_non_empty_list_processors(invalid_processors: list) -> None: - """A non-empty list on a row must fail Pydantic validation, not be silently accepted.""" - with pytest.raises(ValidationError): - ConfigurationRow.from_api_row_data( - row_data={ - 'id': 'row-1', - 'name': 'Row 1', - 'version': 1, - 'configuration': {'parameters': {}, 'processors': invalid_processors}, - }, - component_id='keboola.ex-aws-s3', - configuration_id='123', - ) - - -def test_configuration_root_redacts_plaintext_secrets() -> None: - """Plaintext '#'-values are masked on reads while 'KBC::' ciphers pass through unchanged.""" - api_config = ConfigurationAPIResponse.model_validate( - { - 'componentId': 'keboola.ex-aws-s3', - 'id': '123', - 'name': 'My Config', - 'version': 1, - 'configuration': { - 'parameters': {'user': 'admin', '#password': 'plain-secret', '#token': 'KBC::ProjectSecure::abcd'}, - 'processors': {'after': [{'definition': {'component': 'x'}, 'parameters': {'#key': 'plain'}}]}, - }, - 'metadata': [], - } - ) - root = ConfigurationRoot.from_api_response(api_config) - assert root.parameters == { - 'user': 'admin', - '#password': REDACTED_SECRET_VALUE, - '#token': 'KBC::ProjectSecure::abcd', - } - assert root.processors == { - 'after': [{'definition': {'component': 'x'}, 'parameters': {'#key': REDACTED_SECRET_VALUE}}] - } - - -def test_configuration_row_redacts_plaintext_secrets() -> None: - """Plaintext '#'-values in row parameters are masked on reads.""" - row = ConfigurationRow.from_api_row_data( - row_data={ - 'id': 'row-1', - 'name': 'Row 1', - 'version': 1, - 'configuration': { - 'parameters': {'#api_key': 'plain-secret', '#token': 'KBC::ProjectSecure::abcd', 'period': 'daily'} - }, - }, - component_id='keboola.ex-aws-s3', - configuration_id='123', - ) - assert row.parameters == { - '#api_key': REDACTED_SECRET_VALUE, - '#token': 'KBC::ProjectSecure::abcd', - 'period': 'daily', - } - - -@pytest.mark.parametrize( - ('metadata', 'expected_folder'), - [ - ([{'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'Analytics', 'provider': 'user'}], 'Analytics'), - ([], ''), - ], - ids=['folder_in_metadata', 'no_metadata'], -) -def test_get_configs_includes_folder(metadata: list[dict], expected_folder: str) -> None: - """ConfigurationRootSummary.from_api_response extracts folder from metadata.""" - api_config = ConfigurationAPIResponse.model_validate( - { - 'componentId': 'keboola.ex-aws-s3', - 'id': '123', - 'name': 'My Config', - 'version': 1, - 'configuration': {}, - 'metadata': metadata, - } - ) - summary = ConfigurationRootSummary.from_api_response(api_config) - assert summary.folder == expected_folder - - -@pytest.mark.asyncio -async def test_get_components( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_components: list[dict[str, Any]], -): - """Test get_components tool fetches components concurrently.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - component_ids = [comp['id'] for comp in mock_components[:3]] - - # Get URL components from context - storage_api_url = keboola_client.storage_api_url - project_id = await keboola_client.storage_client.project_id() - base_url = f'{storage_api_url}/admin/projects/{project_id}' - - # Track call order to verify concurrent execution - call_order: list[str] = [] - - async def mock_fetch_component(client: KeboolaClient, component_id: str): - from keboola_mcp_server.tools.components.api_models import ComponentAPIResponse - - call_order.append(component_id) - # Find the matching mock component - for comp in mock_components: - if comp['id'] == component_id: - return ComponentAPIResponse.model_validate(comp) - raise ValueError(f'Component {component_id} not found') - - mocker.patch( - 'keboola_mcp_server.tools.components.tools.fetch_component', - side_effect=mock_fetch_component, - ) - - result = await get_components(ctx=context, component_ids=component_ids) - - # Verify all components were fetched - assert set(call_order) == set(component_ids) - - # Build expected components - expected_components = [ - Component( - component_id=mock_components[0]['id'], - component_name=mock_components[0]['name'], - component_type=mock_components[0]['type'], - component_categories=[], - capabilities=ComponentCapabilities(), - links=[ - Link( - type='ui-dashboard', - title=f'{mock_components[0]["name"]} Configurations Dashboard', - url=f'{base_url}/components/{mock_components[0]["id"]}', - ) - ], - ), - Component( - component_id=mock_components[1]['id'], - component_name=mock_components[1]['name'], - component_type=mock_components[1]['type'], - component_categories=[], - capabilities=ComponentCapabilities(), - links=[ - Link( - type='ui-dashboard', - title=f'{mock_components[1]["name"]} Configurations Dashboard', - url=f'{base_url}/components/{mock_components[1]["id"]}', - ) - ], - ), - Component( - component_id=mock_components[2]['id'], - component_name=mock_components[2]['name'], - component_type=mock_components[2]['type'], - component_categories=[], - capabilities=ComponentCapabilities(), - links=[ - Link( - type='ui-dashboard', - title=f'{mock_components[2]["name"]} Configurations Dashboard', - url=f'{base_url}/components/{mock_components[2]["id"]}', - ) - ], - ), - ] - - expected_output = GetComponentsOutput( - components=expected_components, - links=[ - Link( - type='ui-dashboard', - title='Used Components Dashboard', - url=f'{base_url}/components/configurations', - ) - ], - ) - - assert result == expected_output - - -@pytest.mark.parametrize( - ('sql_dialect', 'expected_component_id', 'expected_configuration_id'), - [ - ('Snowflake', 'keboola.snowflake-transformation', '1234'), - ('BigQuery', 'keboola.google-bigquery-transformation', '5678'), - ], -) -@pytest.mark.asyncio -async def test_create_sql_transformation( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], - sql_dialect: str, - expected_component_id: str, - expected_configuration_id: str, -): - """Test create_sql_transformation tool.""" - context = mcp_context_components_configs - - # Mock the WorkspaceManager - workspace_manager = WorkspaceManager.from_state(context.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value=sql_dialect) - # Mock the KeboolaClient - keboola_client = KeboolaClient.from_state(context.session.state) - component = mock_component - component['id'] = expected_component_id - configuration = mock_configuration - configuration['id'] = expected_configuration_id - - # Set up the mock for ai_service_client - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) - keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=configuration) - - transformation_name = mock_configuration['name'] - bucket_name = clean_bucket_name(transformation_name) - description = mock_configuration['description'] - code_blocks = [ - SimplifiedTfBlocks.Block.Code(name='Code 0', script='SELECT * FROM test'), - SimplifiedTfBlocks.Block.Code(name='Code 1', script='SELECT * FROM test2; SELECT * FROM test3;'), - ] - created_table_name = 'test_table_1' - - # Test the create_sql_transformation tool - new_transformation_configuration = await create_sql_transformation( - ctx=context, - name=transformation_name, - description=description, - sql_code_blocks=code_blocks, - created_table_names=[created_table_name], - ) - - assert isinstance(new_transformation_configuration, ConfigToolOutput) - assert new_transformation_configuration.component_id == expected_component_id - assert new_transformation_configuration.configuration_id == mock_configuration['id'] - assert new_transformation_configuration.description == mock_configuration['description'] - assert new_transformation_configuration.version == mock_configuration['version'] - - raw_code_blocks = await asyncio.gather(*[b.to_raw_code() for b in code_blocks]) - keboola_client.storage_client.configuration_create.assert_called_once_with( - component_id=expected_component_id, - name=transformation_name, - description=description, - configuration={ - 'parameters': { - 'blocks': [ - { - 'name': 'Blocks', - 'codes': [b.model_dump() for b in raw_code_blocks], - } - ] - }, - 'storage': { - 'input': {'tables': []}, - 'output': { - 'tables': [ - { - 'source': created_table_name, - 'destination': f'out.c-{bucket_name}.{created_table_name}', - } - ] - }, - }, - }, - ) - - -@pytest.mark.parametrize('sql_dialect', ['Unknown']) -@pytest.mark.asyncio -async def test_create_sql_transformation_fail( - mocker: MockerFixture, - sql_dialect: str, - mcp_context_components_configs: Context, -): - """Test create_sql_transformation tool which should raise an error if the sql dialect is unknown.""" - context = mcp_context_components_configs - workspace_manager = WorkspaceManager.from_state(context.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value=sql_dialect) - - with pytest.raises(ValueError, match='Unsupported SQL dialect'): - _ = await create_sql_transformation( - ctx=context, - name='test_name', - description='test_description', - sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='Code 0', script='SELECT * FROM test')], - ) - - -@pytest.mark.parametrize( - ('folder', 'tf_count', 'tf_folders', 'expect_folder_metadata', 'expect_hint'), - [ - ('Analytics', 0, [], True, False), - (' Analytics ', 0, [], True, False), # whitespace stripped - ('', 5, [], False, False), - ('', 25, ['Analytics'], False, True), - ('', 25, [], False, True), - ], - ids=[ - 'folder_provided', - 'folder_whitespace_stripped', - 'no_folder_few', - 'no_folder_many_with_folders', - 'no_folder_many_no_folders', - ], -) -@pytest.mark.asyncio -async def test_create_sql_transformation_folder( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], - folder: str, - tf_count: int, - tf_folders: list[str], - expect_folder_metadata: bool, - expect_hint: bool, -) -> None: - """Test folder metadata and change_summary hint for create_sql_transformation.""" - context = mcp_context_components_configs - workspace_manager = WorkspaceManager.from_state(context.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') - keboola_client = KeboolaClient.from_state(context.session.state) - mock_component['id'] = 'keboola.snowflake-transformation' - mock_configuration['id'] = '9999' - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_configuration) - mocker.patch( - 'keboola_mcp_server.tools.components.tools.get_config_folders', - mocker.AsyncMock(return_value=(tf_count, tf_folders, False)), - ) - - result = await create_sql_transformation( - ctx=context, - name='Test', - description='desc', - sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='c', script='SELECT 1;')], - folder=folder, - ) - - assert isinstance(result, ConfigToolOutput) - metadata_calls = [ - call - for call in keboola_client.storage_client.configuration_metadata_update.call_args_list - if call.kwargs.get('metadata', {}).get(MetadataField.CONFIGURATION_FOLDER_NAME) - ] - if expect_folder_metadata: - assert len(metadata_calls) == 1 - assert metadata_calls[0].kwargs['metadata'] == {MetadataField.CONFIGURATION_FOLDER_NAME: folder.strip()} - else: - assert len(metadata_calls) == 0 - if expect_hint: - assert result.change_summary is not None - assert str(tf_count) in result.change_summary - else: - assert result.change_summary is None - - -@pytest.mark.parametrize( - ('folder', 'tf_count', 'tf_folders', 'expect_folder_metadata', 'expect_folder_delete', 'expect_hint'), - [ - ('Sales', 0, [], True, False, False), - (' Sales ', 0, [], True, False, False), # whitespace stripped - (None, 5, [], False, False, False), - (None, 25, ['Analytics'], False, False, True), - (None, 25, [], False, False, True), - ('', 5, [], False, True, False), # empty string → delete - ], - ids=[ - 'folder_provided', - 'folder_whitespace_stripped', - 'no_folder_few', - 'no_folder_many_with_folders', - 'no_folder_many_no_folders', - 'folder_empty_deletes', - ], -) -@pytest.mark.asyncio -async def test_update_sql_transformation_folder( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], - folder: Any, - tf_count: int, - tf_folders: list[str], - expect_folder_metadata: bool, - expect_folder_delete: bool, - expect_hint: bool, -) -> None: - """Test folder metadata and change_summary hint for update_sql_transformation.""" - context = mcp_context_components_configs - workspace_manager = WorkspaceManager.from_state(context.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') - keboola_client = KeboolaClient.from_state(context.session.state) - mock_component['id'] = 'keboola.snowflake-transformation' - configuration_id = 'cfg-folder-test' - existing = { - 'id': configuration_id, - 'name': 'T', - 'description': 'D', - 'configuration': {'parameters': {'blocks': []}, 'storage': {}}, - 'version': 1, - } - updated = {**existing, 'version': 2} - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=updated) - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_metadata_get = mocker.AsyncMock( - return_value=[{'id': 'meta-1', 'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'OldFolder'}] - ) - keboola_client.storage_client.configuration_metadata_delete = mocker.AsyncMock() - mocker.patch( - 'keboola_mcp_server.tools.components.tools.get_config_folders', - mocker.AsyncMock(return_value=(tf_count, tf_folders, False)), - ) - - result = await update_sql_transformation( - context, - change_description='test', - configuration_id=configuration_id, - folder=folder, - ) - - assert isinstance(result, ConfigToolOutput) - metadata_calls = [ - call - for call in keboola_client.storage_client.configuration_metadata_update.call_args_list - if call.kwargs.get('metadata', {}).get(MetadataField.CONFIGURATION_FOLDER_NAME) - ] - if expect_folder_metadata: - assert len(metadata_calls) == 1 - assert metadata_calls[0].kwargs['metadata'] == {MetadataField.CONFIGURATION_FOLDER_NAME: folder.strip()} - else: - assert len(metadata_calls) == 0 - if expect_folder_delete: - keboola_client.storage_client.configuration_metadata_delete.assert_called_once_with( - component_id='keboola.snowflake-transformation', - configuration_id=configuration_id, - metadata_id='meta-1', - ) - else: - keboola_client.storage_client.configuration_metadata_delete.assert_not_called() - if expect_hint: - assert result.change_summary is not None - assert str(tf_count) in result.change_summary - else: - assert result.change_summary is None - - -@pytest.mark.parametrize( - ('folder', 'patched_fn'), - [ - ('Sales', 'set_configuration_folder_metadata'), - ('', 'clear_configuration_folder_metadata'), - ], - ids=['set_raises', 'clear_raises'], -) -@pytest.mark.asyncio -async def test_update_sql_transformation_folder_metadata_error_is_swallowed( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], - folder: str, - patched_fn: str, -) -> None: - """Metadata errors in update_sql_transformation are swallowed and logged, not raised.""" - context = mcp_context_components_configs - workspace_manager = WorkspaceManager.from_state(context.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') - keboola_client = KeboolaClient.from_state(context.session.state) - mock_component['id'] = 'keboola.snowflake-transformation' - configuration_id = 'cfg-error-test' - existing = { - 'id': configuration_id, - 'name': 'T', - 'description': 'D', - 'configuration': {'parameters': {'blocks': []}, 'storage': {}}, - 'version': 1, - } - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={**existing, 'version': 2}) - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_metadata_get = mocker.AsyncMock( - return_value=[{'id': 'meta-1', 'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'OldFolder'}] - ) - keboola_client.storage_client.configuration_metadata_delete = mocker.AsyncMock() - mocker.patch( - f'keboola_mcp_server.tools.components.tools.{patched_fn}', - mocker.AsyncMock(side_effect=RuntimeError('metadata API unavailable')), - ) - mocker.patch( - 'keboola_mcp_server.tools.components.tools.get_config_folders', - mocker.AsyncMock(return_value=(0, [], False)), - ) - - result = await update_sql_transformation( - context, - change_description='test', - configuration_id=configuration_id, - folder=folder, - ) - - assert isinstance(result, ConfigToolOutput) - assert result.success is True - - -@pytest.mark.parametrize( - ('sql_dialect', 'expected_component_id', 'parameter_updates', 'storage', 'expected_config'), - [ - pytest.param( - 'Snowflake', - 'keboola.snowflake-transformation', - [ - TfRenameBlock(op='rename_block', block_id='b0', block_name='Updated Blocks'), - TfSetCode( - op='set_code', - block_id='b0', - code_id='b0.c0', - script='SELECT 1;SELECT * FROM new_table;', - ), - ], - {'output': {'tables': []}}, - { - 'parameters': { - 'blocks': [ - { - 'name': 'Updated Blocks', - 'codes': [{'name': 'Existing Code', 'script': ['SELECT 1;', 'SELECT * FROM new_table;']}], - } - ] - }, - 'storage': {'output': {'tables': []}}, - 'other_field': 'should_be_preserved', - }, - id='snowflake_rename_block_and_set_code', - ), - pytest.param( - 'BigQuery', - 'keboola.google-bigquery-transformation', - [ - TfStrReplace( - op='str_replace', - block_id='b0', - code_id='b0.c0', - search_for='SELECT 1', - replace_with='SELECT 2', - ), - ], - None, - { - 'parameters': { - 'blocks': [{'name': 'Existing', 'codes': [{'name': 'Existing Code', 'script': ['SELECT 2;']}]}] - }, - 'storage': {'input': {'tables': ['existing_table']}}, - 'other_field': 'should_be_preserved', - }, - id='bigquery_str_replace', - ), - pytest.param( - 'Snowflake', - 'keboola.snowflake-transformation', - None, - {'output': {'tables': []}}, - { - 'parameters': { - 'blocks': [{'name': 'Existing', 'codes': [{'name': 'Existing Code', 'script': ['SELECT 1;']}]}] - }, - 'storage': {'output': {'tables': []}}, - 'other_field': 'should_be_preserved', - }, - id='snowflake_storage_only', - ), - ], -) -@pytest.mark.asyncio -async def test_update_sql_transformation( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], - sql_dialect: str, - expected_component_id: str, - parameter_updates: list[TfParamUpdate] | None, - storage: dict[str, Any] | None, - expected_config: dict[str, Any], -): - """ - Test update_sql_transformation tool with transformation-specific parameter_updates. - """ - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - # Mock the WorkspaceManager - workspace_manager = WorkspaceManager.from_state(context.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value=sql_dialect) - - component_id = mock_component['id'] = expected_component_id - configuration_id = 'test-config-id' - - existing_configuration = { - 'id': configuration_id, - 'name': 'Existing Transformation', - 'description': 'Existing description', - 'configuration': { - 'parameters': { - 'blocks': [{'name': 'Existing', 'codes': [{'name': 'Existing Code', 'script': ['SELECT 1;']}]}] - }, - 'storage': {'input': {'tables': ['existing_table']}}, - 'other_field': 'should_be_preserved', - }, - 'version': 1, - } - - updated_name = 'Updated Transformation' - updated_description = 'Updated transformation description' - updated_configuration = { - **existing_configuration, - 'name': updated_name, - 'description': updated_description, - 'version': 2, - } - - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing_configuration) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=updated_configuration) - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - - new_change_description = 'Test transformation update' - - result = await update_sql_transformation( - context, - change_description=new_change_description, - configuration_id=configuration_id, - name=updated_name, - description=updated_description, - parameter_updates=parameter_updates, - storage=storage, - ) - - assert isinstance(result, ConfigToolOutput) - assert result.component_id == component_id - assert result.configuration_id == configuration_id - assert result.description == updated_description - assert result.success is True - assert result.timestamp is not None - assert result.version == updated_configuration['version'] - - keboola_client.ai_service_client.get_component_detail.assert_called_with(component_id=expected_component_id) - keboola_client.storage_client.configuration_update.assert_called_once_with( - component_id=component_id, - configuration_id=configuration_id, - change_description=new_change_description, - configuration=expected_config, - updated_name=updated_name, - updated_description=updated_description, - ) - - -@pytest.mark.asyncio -async def test_update_sql_transformation_wrong_component_type( - mocker: MockerFixture, - mcp_context_components_configs: Context, -) -> None: - """ - update_sql_transformation should raise ToolError with actionable guidance when the - configuration belongs to a Python/R transformation (Storage returns 404 for the SQL - component + config-ID combination). - """ - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - workspace_manager = WorkspaceManager.from_state(context.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') - - # Simulate Storage returning 404: the config exists under python-transformation-v2, - # not under keboola.snowflake-transformation. - mock_response = MagicMock(spec=httpx.Response) - mock_response.status_code = 404 - keboola_client.storage_client.configuration_detail = mocker.AsyncMock( - side_effect=httpx.HTTPStatusError('Not Found', request=MagicMock(), response=mock_response) - ) - - with pytest.raises(ToolError) as exc_info: - await update_sql_transformation( - context, - change_description='update python transformation', - configuration_id='python-config-id', - ) - - error_msg = str(exc_info.value) - assert 'python-config-id' in error_msg - assert 'keboola.snowflake-transformation' in error_msg - assert 'update_config' in error_msg - assert 'keboola.python-transformation-v2' in error_msg - - -@pytest.mark.asyncio -async def test_get_config_examples( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], -): - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - # Setup mock to return test data - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - - text = await get_config_examples(component_id='keboola.ex-aws-s3', ctx=context) - assert text == """# Configuration Examples for `keboola.ex-aws-s3` - -## Root Configuration Examples - -1. Root Configuration: -```json -{ - "foo": "root" -} -``` - -## Row Configuration Examples - -1. Row Configuration: -```json -{ - "foo": "row" -} -``` - -""" - - -@pytest.mark.asyncio -async def test_create_config( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], -): - """Test create_component_root_configuration tool.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - component_id = mock_component['id'] - configuration = mock_configuration - configuration['id'] = 'test-config-id' - - # Set up the mock for ai_service_client and storage_client - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=configuration) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - - name = 'Test Configuration' - description = 'Test configuration description' - parameters = {'test_param': 'test_value'} - storage = {'input': {'tables': []}} - - # Test the create_component_root_configuration tool - result = await create_config( - ctx=context, - name=name, - description=description, - component_id=component_id, - parameters=parameters, - storage=storage, - ) - - assert isinstance(result, ConfigToolOutput) - assert result.component_id == component_id - assert result.configuration_id == configuration['id'] - assert result.description == description - assert result.success is True - assert result.timestamp is not None - assert result.version == configuration['version'] - - keboola_client.ai_service_client.get_component_detail.assert_called_once_with(component_id=component_id) - keboola_client.storage_client.configuration_create.assert_called_once_with( - component_id=component_id, - name=name, - description=description, - configuration={'storage': storage, 'parameters': parameters}, - ) - - -@pytest.mark.asyncio -async def test_add_config_row( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], -): - """Test create_component_row_configuration tool.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - component_id = mock_component['id'] - configuration_id = 'test-config-id' - row_configuration = {'id': 'test-row-id', 'name': 'Test Row', 'version': 1} - - # Set up the mock for ai_service_client and storage_client - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_row_create = mocker.AsyncMock(return_value=row_configuration) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - - name = 'Test Row Configuration' - description = 'Test row configuration description' - parameters = {'row_param': 'row_value'} - storage = {} - - # Test the create_component_row_configuration tool - result = await add_config_row( - ctx=context, - name=name, - description=description, - component_id=component_id, - configuration_id=configuration_id, - parameters=parameters, - storage=storage, - ) - - assert isinstance(result, ConfigToolOutput) - assert result.component_id == component_id - assert result.configuration_id == configuration_id - assert result.description == description - assert result.success is True - assert result.timestamp is not None - assert result.version == row_configuration['version'] - - keboola_client.ai_service_client.get_component_detail.assert_called_once_with(component_id=component_id) - keboola_client.storage_client.configuration_row_create.assert_called_once_with( - component_id=component_id, - config_id=configuration_id, - name=name, - description=description, - configuration={'storage': storage, 'parameters': parameters}, - ) - - -@pytest.mark.parametrize( - ('parameter_updates', 'storage', 'expected_config'), - [ - pytest.param( - [ - ConfigParamSet(op='set', path='api_key', value='new_api_key'), - ConfigParamReplace(op='str_replace', path='database.host', search_for='old', replace_with='new'), - ConfigParamRemove(op='remove', path='deprecated_field'), - ], - None, - { - 'parameters': { - 'api_key': 'new_api_key', - 'database': {'host': 'new_host', 'port': 5432}, - 'existing_param': 'existing_value', - # 'deprecated_field' is removed - }, - 'storage': {'input': {'tables': ['existing_table']}}, - 'other_field': 'should_be_preserved', - }, - id='parameter_updates_only1', - ), - pytest.param( - [ - ConfigParamRemove(op='remove', path='existing_param'), - ConfigParamSet(op='set', path='updated_param', value='updated_value'), - ], - None, - { - 'parameters': { - 'api_key': 'old_api_key', - 'database': {'host': 'old_host', 'port': 5432}, - 'deprecated_field': 'old_value', - 'updated_param': 'updated_value', - # 'existing_param' is removed - }, - 'storage': {'input': {'tables': ['existing_table']}}, - 'other_field': 'should_be_preserved', - }, - id='parameter_updates_only2', - ), - pytest.param( - [ - ConfigParamRemove(op='remove', path='existing_param'), - ConfigParamSet(op='set', path='updated_param', value='updated_value'), - ], - {'output': {'tables': []}}, - { - 'parameters': { - 'api_key': 'old_api_key', - 'database': {'host': 'old_host', 'port': 5432}, - 'deprecated_field': 'old_value', - 'updated_param': 'updated_value', - # 'existing_param' is removed - }, - 'storage': {'output': {'tables': []}}, - 'other_field': 'should_be_preserved', - }, - id='both_parameter_updates_and_storage', - ), - pytest.param( - None, - {'output': {'tables': []}}, - { - 'parameters': { - 'api_key': 'old_api_key', - 'database': {'host': 'old_host', 'port': 5432}, - 'deprecated_field': 'old_value', - 'existing_param': 'existing_value', - }, - 'storage': {'output': {'tables': []}}, - 'other_field': 'should_be_preserved', - }, - id='storage_only', - ), - ], -) -@pytest.mark.asyncio -async def test_update_config( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - parameter_updates: list[ConfigParamUpdate] | None, - storage: dict[str, Any] | None, - expected_config: dict[str, Any], -): - """Test update_component_root_configuration tool with parameter_updates.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - component_id = mock_component['id'] - configuration_id = 'test-config-id' - - existing_configuration = { - 'id': configuration_id, - 'name': 'Existing Config', - 'description': 'Existing description', - 'configuration': { - 'parameters': { - 'api_key': 'old_api_key', - 'database': {'host': 'old_host', 'port': 5432}, - 'deprecated_field': 'old_value', - 'existing_param': 'existing_value', - }, - 'storage': {'input': {'tables': ['existing_table']}}, - 'other_field': 'should_be_preserved', - }, - 'version': 1, - } - - updated_name = 'Updated Configuration' - updated_description = 'Updated configuration description' - updated_configuration = { - **existing_configuration, - 'name': updated_name, - 'description': updated_description, - 'version': 2, - } - - # Set up the mock for ai_service_client and storage_client - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing_configuration) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=updated_configuration) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - - change_description = 'Test update with parameter updates' - - # Test the update_component_root_configuration tool with parameter_updates - result = await update_config( - ctx=context, - name=updated_name, - description=updated_description, - change_description=change_description, - component_id=component_id, - configuration_id=configuration_id, - parameter_updates=parameter_updates, - storage=storage, - ) - - assert isinstance(result, ConfigToolOutput) - assert result.component_id == component_id - assert result.configuration_id == configuration_id - assert result.description == updated_description - assert result.success is True - assert result.timestamp is not None - assert result.version == updated_configuration['version'] - - keboola_client.ai_service_client.get_component_detail.assert_called_once_with(component_id=component_id) - keboola_client.storage_client.configuration_update.assert_called_once_with( - component_id=component_id, - configuration_id=configuration_id, - configuration=expected_config, - change_description=change_description, - updated_name=updated_name, - updated_description=updated_description, - ) - - -@pytest.mark.parametrize( - ('folder', 'cfg_count', 'cfg_folders', 'expect_folder_metadata', 'expect_folder_delete', 'expect_hint'), - [ - ('Analytics', 0, [], True, False, False), - (' Analytics ', 0, [], True, False, False), - (None, 5, [], False, False, False), - (None, 25, ['Analytics'], False, False, True), - (None, 25, [], False, False, True), - ('', 5, [], False, True, False), - ], - ids=[ - 'folder_provided', - 'folder_whitespace_stripped', - 'no_folder_few', - 'no_folder_many_with_folders', - 'no_folder_many_no_folders', - 'folder_empty_deletes', - ], -) -@pytest.mark.asyncio -async def test_update_config_folder( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - folder: Any, - cfg_count: int, - cfg_folders: list[str], - expect_folder_metadata: bool, - expect_folder_delete: bool, - expect_hint: bool, -) -> None: - """Test folder metadata is set/cleared and folder hint is returned by update_config.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - component_id = 'keboola.python-transformation-v2' - mock_component['id'] = component_id - configuration_id = 'cfg-folder-test' - existing = { - 'id': configuration_id, - 'name': 'My Python Transformation', - 'description': 'desc', - 'configuration': {'parameters': {}, 'storage': {}}, - 'version': 1, - } - updated = {**existing, 'version': 2} - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=updated) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - keboola_client.storage_client.configuration_metadata_get = mocker.AsyncMock( - return_value=[{'id': 'meta-1', 'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'OldFolder'}] - ) - keboola_client.storage_client.configuration_metadata_delete = mocker.AsyncMock() - mocker.patch( - 'keboola_mcp_server.tools.components.utils.get_config_folders', - mocker.AsyncMock(return_value=(cfg_count, cfg_folders, False)), - ) - - result = await update_config( - ctx=context, - change_description='test', - component_id=component_id, - configuration_id=configuration_id, - folder=folder, - ) - - assert isinstance(result, ConfigToolOutput) - metadata_calls = [ - call - for call in keboola_client.storage_client.configuration_metadata_update.call_args_list - if call.kwargs.get('metadata', {}).get(MetadataField.CONFIGURATION_FOLDER_NAME) - ] - if expect_folder_metadata: - assert len(metadata_calls) == 1 - assert metadata_calls[0].kwargs['metadata'] == {MetadataField.CONFIGURATION_FOLDER_NAME: folder.strip()} - else: - assert len(metadata_calls) == 0 - if expect_folder_delete: - keboola_client.storage_client.configuration_metadata_delete.assert_called_once_with( - component_id=component_id, configuration_id=configuration_id, metadata_id='meta-1' - ) - else: - keboola_client.storage_client.configuration_metadata_delete.assert_not_called() - if expect_hint: - assert result.change_summary is not None - assert str(cfg_count) in result.change_summary - else: - assert result.change_summary is None - - -@pytest.mark.parametrize( - 'folder', - [None, 'Analytics', ''], - ids=['folder_none', 'folder_provided', 'folder_empty'], -) -@pytest.mark.asyncio -async def test_update_config_folder_skipped_for_extractor( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - folder: Any, -) -> None: - """Folder logic is entirely skipped for components not in FOLDER_SUPPORTING_COMPONENT_IDS.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - component_id = 'keboola.ex-aws-s3' - assert component_id not in FOLDER_SUPPORTING_COMPONENT_IDS - mock_component['id'] = component_id - configuration_id = 'cfg-s3-test' - existing = { - 'id': configuration_id, - 'name': 'My S3 Extractor', - 'description': 'desc', - 'configuration': {'parameters': {}, 'storage': {}}, - 'version': 1, - } - updated = {**existing, 'version': 2} - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=existing) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=updated) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - keboola_client.storage_client.configuration_metadata_delete = mocker.AsyncMock() - mock_get_config_folders = mocker.patch( - 'keboola_mcp_server.tools.components.utils.get_config_folders', - mocker.AsyncMock(), - ) - - result = await update_config( - ctx=context, - change_description='test', - component_id=component_id, - configuration_id=configuration_id, - folder=folder, - ) - - assert isinstance(result, ConfigToolOutput) - mock_get_config_folders.assert_not_called() - folder_metadata_calls = [ - call - for call in keboola_client.storage_client.configuration_metadata_update.call_args_list - if call.kwargs.get('metadata', {}).get(MetadataField.CONFIGURATION_FOLDER_NAME) - ] - assert len(folder_metadata_calls) == 0 - keboola_client.storage_client.configuration_metadata_delete.assert_not_called() - assert result.change_summary is None - - -@pytest.mark.parametrize( - ('tool_fn', 'tool_args', 'component_id', 'message'), - [ - ( - update_config, - {'configuration_id': 'foo', 'change_description': 'bar'}, - DATA_APP_COMPONENT_ID, - 'Use the data applications tools.', - ), - ( - update_config, - {'configuration_id': 'foo', 'change_description': 'bar'}, - CONDITIONAL_FLOW_COMPONENT_ID, - 'Use the flows tools.', - ), - ( - update_config, - {'configuration_id': 'foo', 'change_description': 'bar'}, - ORCHESTRATOR_COMPONENT_ID, - 'Use the flows tools.', - ), - ( - update_config, - {'configuration_id': 'foo', 'change_description': 'bar'}, - BIGQUERY_TRANSFORMATION_ID, - 'Use the SQL transformation tools.', - ), - ( - update_config, - {'configuration_id': 'foo', 'change_description': 'bar'}, - SNOWFLAKE_TRANSFORMATION_ID, - 'Use the SQL transformation tools.', - ), - ( - update_config_row, - {'configuration_id': 'foo', 'configuration_row_id': 'bar', 'change_description': 'baz'}, - DATA_APP_COMPONENT_ID, - 'Use the data applications tools.', - ), - ( - update_config_row, - {'configuration_id': 'foo', 'configuration_row_id': 'bar', 'change_description': 'baz'}, - CONDITIONAL_FLOW_COMPONENT_ID, - 'Use the flows tools.', - ), - ( - update_config_row, - {'configuration_id': 'foo', 'configuration_row_id': 'bar', 'change_description': 'baz'}, - ORCHESTRATOR_COMPONENT_ID, - 'Use the flows tools.', - ), - ( - update_config_row, - {'configuration_id': 'foo', 'configuration_row_id': 'bar', 'change_description': 'baz'}, - BIGQUERY_TRANSFORMATION_ID, - 'Use the SQL transformation tools.', - ), - ( - update_config_row, - {'configuration_id': 'foo', 'configuration_row_id': 'bar', 'change_description': 'baz'}, - SNOWFLAKE_TRANSFORMATION_ID, - 'Use the SQL transformation tools.', - ), - ( - create_config, - {'name': 'foo', 'description': 'bar', 'parameters': {}}, - DATA_APP_COMPONENT_ID, - 'Use the data applications tools.', - ), - ( - create_config, - {'name': 'foo', 'description': 'bar', 'parameters': {}}, - CONDITIONAL_FLOW_COMPONENT_ID, - 'Use the flows tools.', - ), - ( - create_config, - {'name': 'foo', 'description': 'bar', 'parameters': {}}, - ORCHESTRATOR_COMPONENT_ID, - 'Use the flows tools.', - ), - ( - create_config, - {'name': 'foo', 'description': 'bar', 'parameters': {}}, - BIGQUERY_TRANSFORMATION_ID, - 'Use the SQL transformation tools.', - ), - ( - create_config, - {'name': 'foo', 'description': 'bar', 'parameters': {}}, - SNOWFLAKE_TRANSFORMATION_ID, - 'Use the SQL transformation tools.', - ), - ( - add_config_row, - {'name': 'foo', 'description': 'bar', 'configuration_id': 'baz', 'parameters': {}}, - DATA_APP_COMPONENT_ID, - 'Use the data applications tools.', - ), - ( - add_config_row, - {'name': 'foo', 'description': 'bar', 'configuration_id': 'baz', 'parameters': {}}, - CONDITIONAL_FLOW_COMPONENT_ID, - 'Use the flows tools.', - ), - ( - add_config_row, - {'name': 'foo', 'description': 'bar', 'configuration_id': 'baz', 'parameters': {}}, - ORCHESTRATOR_COMPONENT_ID, - 'Use the flows tools.', - ), - ( - add_config_row, - {'name': 'foo', 'description': 'bar', 'configuration_id': 'baz', 'parameters': {}}, - BIGQUERY_TRANSFORMATION_ID, - 'Use the SQL transformation tools.', - ), - ( - add_config_row, - {'name': 'foo', 'description': 'bar', 'configuration_id': 'baz', 'parameters': {}}, - SNOWFLAKE_TRANSFORMATION_ID, - 'Use the SQL transformation tools.', - ), - ], -) -@pytest.mark.asyncio -async def test_generic_tools_reject_specialized_components( - tool_fn: Callable[..., Any], - tool_args: dict[str, Any], - component_id: str, - message: str, - mcp_context_components_configs: Context, -): - m = f'The "{tool_fn.__name__}" tool cannot be used with {component_id} component. {message}' - with pytest.raises(ValueError, match=m): - await tool_fn( - ctx=mcp_context_components_configs, - component_id=component_id, - **tool_args, - ) - - -@pytest.mark.parametrize( - ('parameter_updates', 'storage', 'expected_config'), - [ - pytest.param( - [ - ConfigParamRemove(op='remove', path='existing_param'), - ConfigParamSet(op='set', path='updated_param', value='updated_value'), - ], - {'output': {'tables': []}}, - { - 'parameters': {'updated_param': 'updated_value'}, - 'storage': {'output': {'tables': []}}, - 'other_field': 'should_be_preserved', - }, - id='both_parameter_updates_and_storage', - ), - pytest.param( - [ - ConfigParamRemove(op='remove', path='existing_param'), - ConfigParamSet(op='set', path='updated_param', value='updated_value'), - ], - None, - { - 'parameters': {'updated_param': 'updated_value'}, - 'storage': {'input': {'tables': ['existing_table']}}, - 'other_field': 'should_be_preserved', - }, - id='parameter_updates_only', - ), - pytest.param( - None, - {'output': {'tables': []}}, - { - 'parameters': {'existing_param': 'existing_value'}, - 'storage': {'output': {'tables': []}}, - 'other_field': 'should_be_preserved', - }, - id='storage_only', - ), - ], -) -@pytest.mark.asyncio -async def test_update_config_row( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - parameter_updates: list[ConfigParamUpdate] | None, - storage: dict[str, Any] | None, - expected_config: dict[str, Any], -): - """Test update_component_row_configuration tool with parameter_updates.""" - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - component_id = mock_component['id'] - configuration_id = 'test-config-id' - configuration_row_id = 'test-row-id' - - existing_row_configuration = { - 'configuration': { - 'parameters': {'existing_param': 'existing_value'}, - 'storage': {'input': {'tables': ['existing_table']}}, - 'other_field': 'should_be_preserved', - }, - 'version': 1, - } - - updated_name = 'Updated Row Configuration' - updated_description = 'Updated row configuration description' - updated_row_configuration = { - **existing_row_configuration, - 'name': updated_name, - 'description': updated_description, - 'version': 2, - } - - # Set up the mock for ai_service_client and storage_client - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_row_detail = mocker.AsyncMock(return_value=existing_row_configuration) - keboola_client.storage_client.configuration_row_update = mocker.AsyncMock(return_value=updated_row_configuration) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - - change_description = 'Test row update' - - # Test the update_component_row_configuration tool with parameter_updates - result = await update_config_row( - ctx=context, - name=updated_name, - description=updated_description, - change_description=change_description, - component_id=component_id, - configuration_id=configuration_id, - configuration_row_id=configuration_row_id, - parameter_updates=parameter_updates, - storage=storage, - ) - - assert isinstance(result, ConfigToolOutput) - assert result.component_id == component_id - assert result.configuration_id == configuration_id - assert result.description == updated_description - assert result.success is True - assert result.timestamp is not None - assert result.version == updated_row_configuration['version'] - - keboola_client.ai_service_client.get_component_detail.assert_called_once_with(component_id=component_id) - keboola_client.storage_client.configuration_row_update.assert_called_once_with( - component_id=component_id, - config_id=configuration_id, - configuration_row_id=configuration_row_id, - configuration=expected_config, - change_description=change_description, - updated_name=updated_name, - updated_description=updated_description, - is_disabled=None, - ) - - -# ============================================================================ -# run_sync_action TESTS -# ============================================================================ - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ( - 'root_params', - 'root_storage', - 'root_runtime', - 'root_authorization', - 'row_params', - 'row_storage', - 'expected_params', - 'expected_storage', - 'expected_runtime', - 'expected_authorization', - ), - [ - # No row - uses root config only - ( - {'host': 'db.example.com', 'port': 3306}, - {'input': {'tables': []}}, - None, - None, - None, - None, - {'host': 'db.example.com', 'port': 3306}, - {'input': {'tables': []}}, - None, - None, - ), - # With row - row params override root params (shallow merge) - ( - {'host': 'db.example.com', 'port': 3306, 'database': 'prod'}, - {'input': {'tables': [{'source': 't1'}]}}, - None, - None, - {'database': 'staging', 'schema': 'public'}, - {'input': {'tables': [{'source': 't2'}]}}, - {'host': 'db.example.com', 'port': 3306, 'database': 'staging', 'schema': 'public'}, - {'input': {'tables': [{'source': 't2'}]}}, - None, - None, - ), - # With row - empty root, row provides all - ( - {}, - {}, - None, - None, - {'key': 'value'}, - {'output': {'tables': []}}, - {'key': 'value'}, - {'output': {'tables': []}}, - None, - None, - ), - # Root has runtime.image_tag - must be forwarded so the runner picks up the pinned tag - ( - {'host': 'db.example.com'}, - {'input': {'tables': []}}, - {'image_tag': '1.2.3'}, - None, - None, - None, - {'host': 'db.example.com'}, - {'input': {'tables': []}}, - {'image_tag': '1.2.3'}, - None, - ), - # Root runtime is preserved alongside row overrides for parameters/storage - ( - {'host': 'db.example.com'}, - {'input': {'tables': []}}, - {'image_tag': '4.5.6', 'safe': True}, - None, - {'database': 'staging'}, - {'input': {'tables': [{'source': 't2'}]}}, - {'host': 'db.example.com', 'database': 'staging'}, - {'input': {'tables': [{'source': 't2'}]}}, - {'image_tag': '4.5.6', 'safe': True}, - None, - ), - # Root has OAuth authorization - must be forwarded so sync-actions resolves credentials - ( - {'sheets': []}, - {}, - None, - {'oauth_api': {'id': 'creds-1', 'version': 3}}, - None, - None, - {'sheets': []}, - {}, - None, - {'oauth_api': {'id': 'creds-1', 'version': 3}}, - ), - # Authorization is preserved alongside runtime and row overrides - ( - {'sheets': []}, - {'input': {'tables': []}}, - {'image_tag': '1.0.0'}, - {'oauth_api': {'id': 'creds-2', 'version': 3}}, - {'sheets': [{'id': 1}]}, - {'input': {'tables': [{'source': 't1'}]}}, - {'sheets': [{'id': 1}]}, - {'input': {'tables': [{'source': 't1'}]}}, - {'image_tag': '1.0.0'}, - {'oauth_api': {'id': 'creds-2', 'version': 3}}, - ), - ], - ids=[ - 'no-row', - 'row-overrides-root', - 'empty-root-with-row', - 'root-runtime-image-tag', - 'root-runtime-with-row', - 'root-authorization-oauth', - 'root-authorization-with-runtime-and-row', - ], -) -async def test_run_sync_action( - mcp_context_components_configs: Context, - root_params: dict, - root_storage: dict, - root_runtime: dict | None, - root_authorization: dict | None, - row_params: dict | None, - row_storage: dict | None, - expected_params: dict, - expected_storage: dict, - expected_runtime: dict | None, - expected_authorization: dict | None, -): - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - component_id = 'keboola.ex-db-mysql' - configuration_id = '123' - action_name = 'testConnection' - expected_response = {'status': 'ok'} - - root_configuration: dict[str, Any] = { - 'parameters': root_params, - 'storage': root_storage, - } - if root_runtime is not None: - root_configuration['runtime'] = root_runtime - if root_authorization is not None: - root_configuration['authorization'] = root_authorization - - keboola_client.storage_client.configuration_detail.return_value = { - 'id': configuration_id, - 'componentId': component_id, - 'name': 'Test Config', - 'version': 1, - 'isDisabled': False, - 'isDeleted': False, - 'configuration': root_configuration, - 'rows': [], - } - - if row_params and row_storage: - row_id = 'row-1' - keboola_client.storage_client.configuration_row_detail.return_value = { - 'id': row_id, - 'configuration': { - 'parameters': row_params, - 'storage': row_storage, - }, - } - else: - row_id = None - - keboola_client.sync_actions_client.execute_action.return_value = expected_response - - result = await run_sync_action( - ctx=context, - action_name=action_name, - component_id=component_id, - configuration_id=configuration_id, - configuration_row_id=row_id, - ) - - assert result == expected_response - - expected_config_data: dict[str, Any] = { - 'parameters': expected_params, - 'storage': expected_storage, - } - if expected_runtime is not None: - expected_config_data['runtime'] = expected_runtime - if expected_authorization is not None: - expected_config_data['authorization'] = expected_authorization - - keboola_client.storage_client.configuration_detail.assert_called_once_with(component_id, configuration_id) - keboola_client.sync_actions_client.execute_action.assert_called_once_with( - component_id=component_id, - action=action_name, - config_data=expected_config_data, - ) - - if row_id: - keboola_client.storage_client.configuration_row_detail.assert_called_once_with( - component_id, configuration_id, row_id - ) - - -# ============================================================================ -# variables TESTS -# ============================================================================ - -_VARS_CONFIG_ID = 'vars-cfg-1' -_PARENT_CFG_ID = 'parent-cfg-1' -_PARENT_COMPONENT_ID = SNOWFLAKE_TRANSFORMATION_ID - - -_DEFAULT_ROW_ID = 'default-row-1' -_CREATED_ROW_ID = 'created-row-1' - - -def _make_vars_config( - component_id: str = _PARENT_COMPONENT_ID, - config_id: str = _PARENT_CFG_ID, - *, - with_default_row: bool = False, -) -> dict[str, Any]: - rows = ( - [{'id': _DEFAULT_ROW_ID, 'name': 'Default Values', 'configuration': {'values': []}}] if with_default_row else [] - ) - return { - 'id': _VARS_CONFIG_ID, - 'name': f'Variables definition for {component_id}/{config_id}', - 'configuration': {'variables': [{'name': 'env', 'type': 'string'}]}, - 'rows': rows, - } - - -def _make_parent_config(extra: dict | None = None) -> dict[str, Any]: - cfg = {'parameters': {'blocks': []}, 'storage': {}} - if extra: - cfg.update(extra) - return { - 'id': _PARENT_CFG_ID, - 'name': 'My Transformation', - 'description': 'desc', - 'configuration': cfg, - 'version': 2, - } - - -@pytest.mark.parametrize( - ( - 'existing_vars_configs', - 'has_default_value', - 'expect_create', - 'expect_row_create', - 'expect_row_update', - 'expect_row_clear', - ), - [ - # No existing vars config → create; no default value - ([], False, True, False, False, False), - # No existing vars config → create; with default value → also create row - ([], True, True, True, False, False), - # Existing vars config → update; no default value - ([_make_vars_config()], False, False, False, False, False), - # Existing vars config → update; with default value → create row (no existing row) - ([_make_vars_config()], True, False, True, False, False), - # Existing vars config WITH a Default Values row → update row instead of create - ([_make_vars_config(with_default_row=True)], True, False, False, True, False), - # Existing vars config WITH a Default Values row, no new default → clear the row - ([_make_vars_config(with_default_row=True)], False, False, False, False, True), - ], - ids=[ - 'new-no-default', - 'new-with-default', - 'existing-no-default', - 'existing-with-default', - 'existing-row-update', - 'existing-row-clear-default', - ], -) -@pytest.mark.asyncio -async def test_create_sql_transformation_variables( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], - existing_vars_configs: list[dict[str, Any]], - has_default_value: bool, - expect_create: bool, - expect_row_create: bool, - expect_row_update: bool, - expect_row_clear: bool, -) -> None: - context = mcp_context_components_configs - workspace_manager = WorkspaceManager.from_state(context.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') - keboola_client = KeboolaClient.from_state(context.session.state) - - component = mock_component - component['id'] = SNOWFLAKE_TRANSFORMATION_ID - configuration = mock_configuration - configuration['id'] = _PARENT_CFG_ID - - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) - keboola_client.storage_client.configuration_create = mocker.AsyncMock( - side_effect=( - [configuration, {**_make_vars_config(), 'id': _VARS_CONFIG_ID}] if expect_create else [configuration] - ) - ) - keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=existing_vars_configs) - vars_link_version = 3 - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={'version': vars_link_version}) - keboola_client.storage_client.configuration_row_create = mocker.AsyncMock(return_value={'id': _CREATED_ROW_ID}) - keboola_client.storage_client.configuration_row_update = mocker.AsyncMock(return_value={}) - - async def detail_side_effect(*args: Any, **kwargs: Any) -> dict[str, Any]: - cid = args[0] if args else kwargs.get('component_id') - if cid == VARIABLES_COMPONENT_ID and existing_vars_configs: - return existing_vars_configs[0] - return _make_parent_config() - - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=detail_side_effect) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - - var_def = VariableDefinition(name='env', type='string', default_value='prod' if has_default_value else None) - - result = await create_sql_transformation( - ctx=context, - name=mock_configuration['name'], - description=mock_configuration['description'], - sql_code_blocks=[SimplifiedTfBlocks.Block.Code(name='Block', script='SELECT 1')], - variables=[var_def], - ) - - assert result.success is True - - # Verify vars config was created or updated. - if expect_create: - # Second create call is for vars config. - assert keboola_client.storage_client.configuration_create.call_count == 2 - vars_call = keboola_client.storage_client.configuration_create.call_args_list[1] - assert vars_call.kwargs['component_id'] == VARIABLES_COMPONENT_ID - assert vars_call.kwargs['configuration'] == {'variables': [{'name': 'env', 'type': 'string'}]} - else: - assert keboola_client.storage_client.configuration_create.call_count == 1 - keboola_client.storage_client.configuration_update.assert_any_call( - component_id=VARIABLES_COMPONENT_ID, - configuration_id=_VARS_CONFIG_ID, - configuration={'variables': [{'name': 'env', 'type': 'string'}]}, - change_description='Update variable definitions', - ) - - row_cfg = {'values': [{'name': 'env', 'value': 'prod'}]} - - if expect_row_create: - keboola_client.storage_client.configuration_row_create.assert_called_once_with( - component_id=VARIABLES_COMPONENT_ID, - config_id=_VARS_CONFIG_ID, - name='Default Values', - description='', - configuration=row_cfg, - ) - keboola_client.storage_client.configuration_row_update.assert_not_called() - elif expect_row_update: - keboola_client.storage_client.configuration_row_update.assert_called_once_with( - component_id=VARIABLES_COMPONENT_ID, - config_id=_VARS_CONFIG_ID, - configuration_row_id=_DEFAULT_ROW_ID, - configuration=row_cfg, - change_description='Update default variable values', - ) - keboola_client.storage_client.configuration_row_create.assert_not_called() - elif expect_row_clear: - keboola_client.storage_client.configuration_row_update.assert_called_once_with( - component_id=VARIABLES_COMPONENT_ID, - config_id=_VARS_CONFIG_ID, - configuration_row_id=_DEFAULT_ROW_ID, - configuration={'values': []}, - change_description='Clear default variable values', - ) - keboola_client.storage_client.configuration_row_create.assert_not_called() - else: - keboola_client.storage_client.configuration_row_create.assert_not_called() - keboola_client.storage_client.configuration_row_update.assert_not_called() - - # Verify parent config was patched with variables_id and variables_values_id. - parent_update_calls = [ - c - for c in keboola_client.storage_client.configuration_update.call_args_list - if c.kwargs.get('component_id') == SNOWFLAKE_TRANSFORMATION_ID - ] - assert len(parent_update_calls) == 1 - assert parent_update_calls[0].kwargs['configuration']['variables_id'] == _VARS_CONFIG_ID - expected_values_id = _CREATED_ROW_ID if expect_row_create else (_DEFAULT_ROW_ID if expect_row_update else None) - if expected_values_id: - assert parent_update_calls[0].kwargs['configuration']['variables_values_id'] == expected_values_id - else: - assert 'variables_values_id' not in parent_update_calls[0].kwargs['configuration'] - - # Verify UPDATED_BY_MCP metadata is stamped with the version from the vars-link parent update. - updated_by_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{vars_link_version}' - metadata_calls = keboola_client.storage_client.configuration_metadata_update.call_args_list - updated_by_calls = [c for c in metadata_calls if c.kwargs.get('metadata', {}).get(updated_by_key) == 'true'] - assert len(updated_by_calls) == 1 - assert updated_by_calls[0].kwargs['component_id'] == SNOWFLAKE_TRANSFORMATION_ID - assert updated_by_calls[0].kwargs['configuration_id'] == _PARENT_CFG_ID - - -@pytest.mark.parametrize( - ('variables', 'existing_vars_configs', 'expect_vars_update', 'expect_parent_update'), - [ - # None → leave unchanged, no vars API calls. - (None, [], False, False), - # Empty list, no existing vars config → no-op. - ([], [], False, False), - # Empty list, existing vars config → delete vars config + remove variables_id from parent. - ([], [_make_vars_config()], False, True), - # Empty list, existing vars config WITH Default Values row → same: delete vars config. - ([], [_make_vars_config(with_default_row=True)], False, True), - # Non-empty list → update vars config + patch parent. - ([VariableDefinition(name='env', type='string')], [], True, True), - ], - ids=['none-no-op', 'empty-no-existing', 'empty-delete', 'empty-delete-with-row', 'set-vars'], -) -@pytest.mark.asyncio -async def test_update_sql_transformation_variables( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], - variables: list[VariableDefinition] | None, - existing_vars_configs: list[dict[str, Any]], - expect_vars_update: bool, - expect_parent_update: bool, -) -> None: - context = mcp_context_components_configs - workspace_manager = WorkspaceManager.from_state(context.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') - keboola_client = KeboolaClient.from_state(context.session.state) - - component = mock_component - component['id'] = SNOWFLAKE_TRANSFORMATION_ID - existing_raw = _make_parent_config({'variables_id': _VARS_CONFIG_ID} if existing_vars_configs else None) - - has_default_row = any(r.get('name') == 'Default Values' for c in existing_vars_configs for r in c.get('rows', [])) - vars_config = {**_make_vars_config(with_default_row=has_default_row), 'id': _VARS_CONFIG_ID} - - async def detail_side_effect(*args: Any, **kwargs: Any) -> dict[str, Any]: - cid = args[0] if args else kwargs.get('component_id') - return vars_config if cid == VARIABLES_COMPONENT_ID else existing_raw - - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=component) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=detail_side_effect) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=existing_raw) - keboola_client.storage_client.configuration_delete = mocker.AsyncMock(return_value=None) - keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=existing_vars_configs) - keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=vars_config) - keboola_client.storage_client.configuration_row_create = mocker.AsyncMock(return_value={}) - keboola_client.storage_client.configuration_row_update = mocker.AsyncMock(return_value={}) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - - result = await update_sql_transformation( - ctx=context, - change_description='test update', - configuration_id=_PARENT_CFG_ID, - variables=variables, - ) - - assert result.success is True - - vars_update_calls = [ - c - for c in keboola_client.storage_client.configuration_update.call_args_list - if c.kwargs.get('component_id') == VARIABLES_COMPONENT_ID - ] - vars_create_calls = [ - c - for c in keboola_client.storage_client.configuration_create.call_args_list - if c.kwargs.get('component_id') == VARIABLES_COMPONENT_ID - ] - # The single parent update call (vars link is now folded into the main PUT). - all_parent_updates = [ - c - for c in keboola_client.storage_client.configuration_update.call_args_list - if c.kwargs.get('component_id') == SNOWFLAKE_TRANSFORMATION_ID - ] - assert len(all_parent_updates) == 1 - main_cfg = all_parent_updates[0].kwargs['configuration'] - - if not expect_vars_update: - assert not vars_update_calls - assert not vars_create_calls - - if variables is not None and len(variables) > 0 and expect_vars_update: - # Setting vars: either created or updated, and variables_id embedded in main PUT. - assert vars_update_calls or vars_create_calls - assert 'variables_id' in main_cfg - - if variables == [] and existing_vars_configs: - # Deletion: vars config is deleted, not updated; variables_id removed from main PUT. - vars_delete_calls = [ - c - for c in keboola_client.storage_client.configuration_delete.call_args_list - if c.kwargs.get('component_id') == VARIABLES_COMPONENT_ID - ] - assert vars_delete_calls - assert not vars_update_calls - assert 'variables_id' not in main_cfg - - -@pytest.mark.parametrize( - ('variables', 'existing_vars_configs', 'expect_vars_api_calls'), - [ - # No variables → no vars API calls. - ([], [], False), - # With variables → vars API calls. - ([VariableDefinition(name='token', type='vault')], [], True), - ], - ids=['no-vars', 'with-vars'], -) -@pytest.mark.asyncio -async def test_create_config_variables( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - mock_configuration: dict[str, Any], - variables: list[VariableDefinition], - existing_vars_configs: list[dict[str, Any]], - expect_vars_api_calls: bool, -) -> None: - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - component_id = mock_component['id'] - configuration = mock_configuration - configuration['id'] = _PARENT_CFG_ID - - vars_link_version = 3 - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=configuration) - keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=existing_vars_configs) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={'version': vars_link_version}) - keboola_client.storage_client.configuration_row_create = mocker.AsyncMock(return_value={}) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=_make_parent_config()) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - - result = await create_config( - ctx=context, - name=mock_configuration['name'], - description=mock_configuration['description'], - component_id=component_id, - parameters={'key': 'val'}, - variables=variables, - ) - - assert result.success is True - - vars_create_calls = [ - c - for c in keboola_client.storage_client.configuration_create.call_args_list - if c.kwargs.get('component_id') == VARIABLES_COMPONENT_ID - ] - if expect_vars_api_calls: - assert len(vars_create_calls) == 1 - assert vars_create_calls[0].kwargs['configuration']['variables'][0]['name'] == variables[0].name - updated_by_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{vars_link_version}' - metadata_calls = keboola_client.storage_client.configuration_metadata_update.call_args_list - updated_by_calls = [c for c in metadata_calls if c.kwargs.get('metadata', {}).get(updated_by_key) == 'true'] - assert len(updated_by_calls) == 1 - assert updated_by_calls[0].kwargs['component_id'] == component_id - assert updated_by_calls[0].kwargs['configuration_id'] == _PARENT_CFG_ID - else: - assert not vars_create_calls - - -_GENERIC_COMPONENT_ID = 'keboola.ex-generic-v2' - - -@pytest.mark.parametrize( - ('variables', 'existing_vars_configs', 'expect_vars_update', 'expect_parent_update'), - [ - # None → leave unchanged, no vars API calls. - (None, [], False, False), - # Empty list, no existing vars config → no-op (parent has no variables_id to unlink). - ([], [], False, False), - # Empty list, existing vars config → delete vars config + remove variables_id from parent. - ([], [_make_vars_config(_GENERIC_COMPONENT_ID)], False, True), - # Empty list, existing vars config WITH Default Values row → same: delete vars config. - ([], [_make_vars_config(_GENERIC_COMPONENT_ID, with_default_row=True)], False, True), - # Non-empty list → update vars config + patch parent. - ([VariableDefinition(name='env', type='string')], [], True, True), - ], - ids=['none-no-op', 'empty-no-existing', 'empty-delete', 'empty-delete-with-row', 'set-vars'], -) -@pytest.mark.asyncio -async def test_update_config_variables( - mocker: MockerFixture, - mcp_context_components_configs: Context, - mock_component: dict[str, Any], - variables: list[VariableDefinition] | None, - existing_vars_configs: list[dict[str, Any]], - expect_vars_update: bool, - expect_parent_update: bool, -) -> None: - context = mcp_context_components_configs - keboola_client = KeboolaClient.from_state(context.session.state) - - component_id = _GENERIC_COMPONENT_ID - configuration_id = _PARENT_CFG_ID - existing_raw = _make_parent_config({'variables_id': _VARS_CONFIG_ID} if existing_vars_configs else None) - - has_default_row = any(r.get('name') == 'Default Values' for c in existing_vars_configs for r in c.get('rows', [])) - vars_config = {**_make_vars_config(_GENERIC_COMPONENT_ID, with_default_row=has_default_row), 'id': _VARS_CONFIG_ID} - - async def detail_side_effect(*args: Any, **kwargs: Any) -> dict[str, Any]: - cid = args[0] if args else kwargs.get('component_id') - return vars_config if cid == VARIABLES_COMPONENT_ID else existing_raw - - keboola_client.ai_service_client = mocker.MagicMock() - keboola_client.ai_service_client.get_component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.component_detail = mocker.AsyncMock(return_value=mock_component) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=detail_side_effect) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=existing_raw) - keboola_client.storage_client.configuration_delete = mocker.AsyncMock(return_value=None) - keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=existing_vars_configs) - keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=vars_config) - keboola_client.storage_client.configuration_row_create = mocker.AsyncMock(return_value={}) - keboola_client.storage_client.configuration_row_update = mocker.AsyncMock(return_value={}) - keboola_client.storage_client.configuration_metadata_update = mocker.AsyncMock() - - mock_component['id'] = component_id - - result = await update_config( - ctx=context, - component_id=component_id, - configuration_id=configuration_id, - change_description='test update', - variables=variables, - ) - - assert result.success is True - - vars_update_calls = [ - c - for c in keboola_client.storage_client.configuration_update.call_args_list - if c.kwargs.get('component_id') == VARIABLES_COMPONENT_ID - ] - vars_create_calls = [ - c - for c in keboola_client.storage_client.configuration_create.call_args_list - if c.kwargs.get('component_id') == VARIABLES_COMPONENT_ID - ] - # The single parent update call (vars link is now folded into the main PUT). - all_parent_updates = [ - c - for c in keboola_client.storage_client.configuration_update.call_args_list - if c.kwargs.get('component_id') == component_id - ] - assert len(all_parent_updates) == 1 - main_cfg = all_parent_updates[0].kwargs['configuration'] - - if not expect_vars_update: - assert not vars_update_calls - assert not vars_create_calls - - if variables is not None and len(variables) > 0 and expect_vars_update: - assert vars_update_calls or vars_create_calls - assert 'variables_id' in main_cfg - - if variables == [] and existing_vars_configs: - # Deletion: vars config is deleted, not updated; variables_id removed from main PUT. - vars_delete_calls = [ - c - for c in keboola_client.storage_client.configuration_delete.call_args_list - if c.kwargs.get('component_id') == VARIABLES_COMPONENT_ID - ] - assert vars_delete_calls - assert not vars_update_calls - assert 'variables_id' not in main_cfg diff --git a/tests/tools/components/test_utils.py b/tests/tools/components/test_utils.py deleted file mode 100644 index 1e1aee55f..000000000 --- a/tests/tools/components/test_utils.py +++ /dev/null @@ -1,1571 +0,0 @@ -import re -from typing import Any, Sequence -from unittest.mock import AsyncMock, MagicMock - -import pytest - -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.tools.components.model import ( - ALL_COMPONENT_TYPES, - ComponentType, - ConfigParamListAppend, - ConfigParamRemove, - ConfigParamReplace, - ConfigParamSet, - ConfigParamUpdate, - SimplifiedTfBlocks, - TfAddBlock, - TfAddCode, - TfParamUpdate, - TfRemoveCode, - TfRenameBlock, - TfRenameCode, - TfSetCode, - TfStrReplace, - TransformationConfiguration, -) -from keboola_mcp_server.tools.components.utils import ( - _apply_param_update, - _normalize_jsonpath, - clean_bucket_name, - clear_configuration_folder_metadata, - create_transformation_configuration, - expand_component_types, - get_config_folders, - set_configuration_folder_metadata, - set_nested_value, - structure_summary, - update_params, - update_transformation_parameters, -) - - -@pytest.mark.parametrize( - ('component_type', 'expected'), - [ - (['extractor', 'writer'], ('extractor', 'writer')), - (['writer', 'extractor', 'writer', 'extractor'], ('extractor', 'writer')), - ([], ALL_COMPONENT_TYPES), - (None, ALL_COMPONENT_TYPES), - ], -) -def test_expand_component_types( - component_type: Sequence[ComponentType], - expected: list[ComponentType], -): - """Test list_component_configurations tool with core component.""" - assert expand_component_types(component_type) == expected - - -@pytest.mark.parametrize( - ('codes', 'transformation_name', 'output_tables', 'expected'), - [ - # testing with multiple sql statements and no output table mappings - # it should not create any output tables - ( - [SimplifiedTfBlocks.Block.Code(name='Code 0', script='SELECT * FROM test;\nSELECT * FROM test2;')], - 'test name', - [], - TransformationConfiguration( - parameters=TransformationConfiguration.Parameters( - blocks=[ - TransformationConfiguration.Parameters.Block( - name='Blocks', - codes=[ - TransformationConfiguration.Parameters.Block.Code( - name='Code 0', - script=['SELECT * FROM test;', 'SELECT * FROM test2;'], - ) - ], - ) - ] - ), - storage=TransformationConfiguration.Storage( - input=TransformationConfiguration.Storage.Destination(tables=[]), - output=TransformationConfiguration.Storage.Destination(tables=[]), - ), - ), - ), - # testing with multiple sql statements and output table mappings - # it should create output tables according to the mappings - ( - [ - SimplifiedTfBlocks.Block.Code( - name='Code 0', - script=( - 'CREATE OR REPLACE TABLE "test_table_1" AS SELECT * FROM "test";\n' - '-- comment\n' - 'CREATE OR REPLACE TABLE "test_table_2" AS SELECT * FROM "test";' - ), - ) - ], - 'test name two', - ['test_table_1', 'test_table_2'], - TransformationConfiguration( - parameters=TransformationConfiguration.Parameters( - blocks=[ - TransformationConfiguration.Parameters.Block( - name='Blocks', - codes=[ - TransformationConfiguration.Parameters.Block.Code( - name='Code 0', - script=[ - 'CREATE OR REPLACE TABLE "test_table_1" AS SELECT * FROM "test";', - '-- comment\n' - 'CREATE OR REPLACE TABLE "test_table_2" AS SELECT * FROM "test";', - ], - ) - ], - ) - ] - ), - storage=TransformationConfiguration.Storage( - input=TransformationConfiguration.Storage.Destination(tables=[]), - output=TransformationConfiguration.Storage.Destination( - tables=[ - TransformationConfiguration.Storage.Destination.Table( - source='test_table_1', - destination='out.c-test-name-two.test_table_1', - ), - TransformationConfiguration.Storage.Destination.Table( - source='test_table_2', - destination='out.c-test-name-two.test_table_2', - ), - ] - ), - ), - ), - ), - # testing with single sql statement and output table mappings - ( - [ - SimplifiedTfBlocks.Block.Code( - name='Code 0', - script='CREATE OR REPLACE TABLE "test_table_1" AS SELECT * FROM "test";', - ) - ], - 'test name', - ['test_table_1'], - TransformationConfiguration( - parameters=TransformationConfiguration.Parameters( - blocks=[ - TransformationConfiguration.Parameters.Block( - name='Blocks', - codes=[ - TransformationConfiguration.Parameters.Block.Code( - name='Code 0', - script=['CREATE OR REPLACE TABLE "test_table_1" AS SELECT * FROM "test";'], - ) - ], - ) - ] - ), - storage=TransformationConfiguration.Storage( - input=TransformationConfiguration.Storage.Destination(tables=[]), - output=TransformationConfiguration.Storage.Destination( - tables=[ - TransformationConfiguration.Storage.Destination.Table( - source='test_table_1', - destination='out.c-test-name.test_table_1', - ), - ] - ), - ), - ), - ), - ], -) -@pytest.mark.asyncio -async def test_create_transformation_configuration( - codes: list[SimplifiedTfBlocks.Block.Code], - transformation_name: str, - output_tables: list[str], - expected: TransformationConfiguration, -): - """Test create_transformation_configuration function which should return the correct transformation configuration - given the codes, transformation_name and output_tables.""" - - configuration = await create_transformation_configuration( - codes=codes, - transformation_name=transformation_name, - output_tables=output_tables, - sql_dialect='snowflake', - ) - - assert configuration == expected - - -@pytest.mark.parametrize( - ('input_str', 'expected_str'), - [ - ('!@#$%^&*()+,./;\'[]"\\`', ''), - ('a_-', 'a_-'), - ('1234567890', '1234567890'), - ('test_table_1', 'test_table_1'), - ('test:-Table-1!', 'test-Table-1'), - ('test Test', 'test-Test'), - ('__test_test', 'test_test'), - ('--test-test', '--test-test'), # it is allowed - ('+ěščřžýáíé', 'escrzyaie'), - ], -) -def test_clean_bucket_name(input_str: str, expected_str: str): - """Test clean_bucket_name function.""" - assert clean_bucket_name(input_str) == expected_str - - -@pytest.mark.parametrize( - ('path', 'expected'), - [ - # Valid unquoted name passes through unchanged - ('api_key', 'api_key'), - # Nested valid names pass through unchanged - ('database.host', 'database.host'), - # Root '$' token is preserved - ('$.api_key', '$.api_key'), - # Hash-prefix segment gets quoted - ('#anthropic_api_key', '"#anthropic_api_key"'), - # Hash in nested path: only the special segment gets quoted - ('parameters.#key', 'parameters."#key"'), - # Already double-quoted segment is not re-quoted - ('"#key"', '"#key"'), - # Already single-quoted segment is not re-quoted - ("'key'", "'key'"), - # Bracket notation is preserved (contains '[') - ('items[0].name', 'items[0].name'), - # Digit-leading segment gets quoted - ('123key', '"123key"'), - # Empty string segment gets quoted - ('', '""'), - ], -) -def test_normalize_jsonpath(path: str, expected: str): - """Test _normalize_jsonpath quotes segments with special characters for jsonpath_ng.""" - assert _normalize_jsonpath(path) == expected - - -@pytest.mark.parametrize( - ('params', 'update', 'expected'), - [ - # Test 'set' operation on simple key - ( - {'api_key': 'old_key', 'count': 42}, - ConfigParamSet(op='set', path='api_key', value='new_key'), - {'api_key': 'new_key', 'count': 42}, - ), - # Test 'set' operation on nested key - ( - {'database': {'host': 'localhost', 'port': 5432}}, - ConfigParamSet(op='set', path='database.host', value='remotehost'), - {'database': {'host': 'remotehost', 'port': 5432}}, - ), - # Test 'set' operation on new key - ( - {'api_key': 'old_key'}, - ConfigParamSet(op='set', path='new_key', value='new_value'), - {'api_key': 'old_key', 'new_key': 'new_value'}, - ), - # Test 'set' operation creating deeply nested path - ( - {'api_key': 'value'}, - ConfigParamSet(op='set', path='config.database.connection.host', value='localhost'), - {'api_key': 'value', 'config': {'database': {'connection': {'host': 'localhost'}}}}, - ), - # Test 'set' operation with different value types - list - ( - {'config': {}}, - ConfigParamSet(op='set', path='config.items', value=[1, 2, 3]), - {'config': {'items': [1, 2, 3]}}, - ), - # Test 'set' operation with different value types - boolean - ( - {'config': {}}, - ConfigParamSet(op='set', path='config.enabled', value=True), - {'config': {'enabled': True}}, - ), - # Test 'set' operation with different value types - None - ( - {'config': {}}, - ConfigParamSet(op='set', path='config.value', value=None), - {'config': {'value': None}}, - ), - # Test 'set' operation with different value types - number - ( - {'config': {}}, - ConfigParamSet(op='set', path='config.timeout', value=300), - {'config': {'timeout': 300}}, - ), - # Test 'set' operation with multiple JSONPath matches - ( - {'messages': [{'text': 'old1'}, {'text': 'old2 old3'}]}, - ConfigParamSet(op='set', path='messages[*].text', value='new'), - {'messages': [{'text': 'new'}, {'text': 'new'}]}, - ), - # Test 'set' operation with '$' (root) JSONPath - ( - {'messages': [{'text': 'old1'}, {'text': 'old2 old3'}]}, - ConfigParamSet(op='set', path='$', value={'object': 'new'}), - {'object': 'new'}, - ), - # Test 'str_replace' operation on existing string - ( - {'api_key': 'old_key_value'}, - ConfigParamReplace(op='str_replace', path='api_key', search_for='old', replace_with='new'), - {'api_key': 'new_key_value'}, - ), - # Test 'str_replace' operation with empty replace string - ( - {'api_key': 'old_key_value'}, - ConfigParamReplace(op='str_replace', path='api_key', search_for='old_', replace_with=''), - {'api_key': 'key_value'}, - ), - # Test 'str_replace' operation on nested string - ( - {'database': {'host': 'old_host_name'}}, - ConfigParamReplace(op='str_replace', path='database.host', search_for='old', replace_with='new'), - {'database': {'host': 'new_host_name'}}, - ), - # Test 'str_replace' with multiple occurrences - ( - {'message': 'old old old'}, - ConfigParamReplace(op='str_replace', path='message', search_for='old', replace_with='new'), - {'message': 'new new new'}, - ), - # Test 'str_replace' with multiple JSONPath matches - ( - {'messages': ['old1', 'old2 old3']}, - ConfigParamReplace(op='str_replace', path='messages[*]', search_for='old', replace_with='new'), - {'messages': ['new1', 'new2 new3']}, - ), - # Test 'str_replace' on list of strings - ( - {'blocks': [{'codes': [{'script': ['SELECT old', 'FROM old_table']}]}]}, - ConfigParamReplace( - op='str_replace', - path='blocks[0].codes[0].script', - search_for='old', - replace_with='new', - ), - {'blocks': [{'codes': [{'script': ['SELECT new', 'FROM new_table']}]}]}, - ), - # Test 'remove' operation on simple key - ( - {'api_key': 'value', 'count': 42}, - ConfigParamRemove(op='remove', path='api_key'), - {'count': 42}, - ), - # Test 'remove' operation on nested key - ( - {'database': {'host': 'localhost', 'port': 5432}}, - ConfigParamRemove(op='remove', path='database.port'), - {'database': {'host': 'localhost'}}, - ), - # Test 'remove' operation on entire object - ( - {'database': {'host': 'localhost', 'port': 5432}, 'api_key': 'value'}, - ConfigParamRemove(op='remove', path='database'), - {'api_key': 'value'}, - ), - # Test 'remove' operation with multiple JSONPath matches - ( - {'messages': [{'text': 'old1'}, {'text': 'old2 old3', 'metadata': {'id': 1}}]}, - ConfigParamRemove(op='remove', path='messages[*].text'), - {'messages': [{}, {'metadata': {'id': 1}}]}, - ), - # Test 'remove' operation with '$' JSONPath - it doesn't do anything - ( - {'messages': [{'text': 'old1'}, {'text': 'old2 old3'}]}, - ConfigParamRemove(op='remove', path='$'), - {'messages': [{'text': 'old1'}, {'text': 'old2 old3'}]}, - ), - # Test 'list_append' operation on simple list - ( - {'items': [1, 2, 3]}, - ConfigParamListAppend(op='list_append', path='items', value=4), - {'items': [1, 2, 3, 4]}, - ), - # Test 'list_append' operation on nested list - ( - {'config': {'values': ['a', 'b']}}, - ConfigParamListAppend(op='list_append', path='config.values', value='c'), - {'config': {'values': ['a', 'b', 'c']}}, - ), - # Test 'list_append' operation on deeply nested list (like SQL transformation structure) - ( - {'blocks': [{'codes': [{'script': ['SELECT 1']}]}]}, - ConfigParamListAppend(op='list_append', path='blocks[0].codes[0].script', value='SELECT 2'), - {'blocks': [{'codes': [{'script': ['SELECT 1', 'SELECT 2']}]}]}, - ), - # Test 'list_append' operation with multiple JSONPath matches - ( - {'messages': [{'items': [1]}, {'items': [2]}]}, - ConfigParamListAppend(op='list_append', path='messages[*].items', value=99), - {'messages': [{'items': [1, 99]}, {'items': [2, 99]}]}, - ), - # Test 'list_append' operation with different value types - dict - ( - {'config': {'entries': [{'id': 1}]}}, - ConfigParamListAppend(op='list_append', path='config.entries', value={'id': 2}), - {'config': {'entries': [{'id': 1}, {'id': 2}]}}, - ), - # Test 'set' operation on existing '#'-prefixed key - ( - {'#anthropic_api_key': 'old'}, - ConfigParamSet(op='set', path='#anthropic_api_key', value='new'), - {'#anthropic_api_key': 'new'}, - ), - # Test 'set' operation creating a new '#'-prefixed key - ( - {}, - ConfigParamSet(op='set', path='#anthropic_api_key', value='val'), - {'#anthropic_api_key': 'val'}, - ), - # Test 'set' operation on nested '#'-prefixed key - ( - {'params': {'#key': 'old'}}, - ConfigParamSet(op='set', path='params.#key', value='new'), - {'params': {'#key': 'new'}}, - ), - # Test 'str_replace' operation on '#'-prefixed key - ( - {'#key': 'old_value'}, - ConfigParamReplace(op='str_replace', path='#key', search_for='old', replace_with='new'), - {'#key': 'new_value'}, - ), - # Test 'remove' operation on '#'-prefixed key - ( - {'#key': 'value', 'other': 'data'}, - ConfigParamRemove(op='remove', path='#key'), - {'other': 'data'}, - ), - ], -) -def test_apply_param_update( - params: dict[str, Any], - update: ConfigParamUpdate, - expected: dict[str, Any], -): - """Test _apply_param_update function with valid operations.""" - result = _apply_param_update(params, update) - assert result == expected - - -@pytest.mark.parametrize( - ('params', 'update', 'expected_error'), - [ - # Test 'str_replace' operation on non-existent path - ( - {'api_key': 'value'}, - ConfigParamReplace(op='str_replace', path='nonexistent.key', search_for='old', replace_with='new'), - 'Path "nonexistent.key" does not exist', - ), - # Test 'str_replace' operation on non-string value - ( - {'count': 42}, - ConfigParamReplace(op='str_replace', path='count', search_for='4', replace_with='5'), - 'Path "count" is not a string or list of strings', - ), - # Test 'str_replace' operation on list with non-string values - ( - {'script': ['SELECT 1', 2]}, - ConfigParamReplace(op='str_replace', path='script', search_for='1', replace_with='2'), - 'Path "script" is not a string or list of strings', - ), - # Test 'str_replace' when search string is empty - ( - {'api_key': 'my_secret_key'}, - ConfigParamReplace(op='str_replace', path='api_key', search_for='', replace_with='a'), - 'Search string is empty', - ), - # Test 'str_replace' when search string not found - ( - {'api_key': 'my_secret_key'}, - ConfigParamReplace(op='str_replace', path='api_key', search_for='notfound', replace_with='new'), - 'Search string "notfound" not found in path "api_key"', - ), - # Test 'str_replace' when search string and replace string are the same - ( - {'api_key': 'my_secret_key'}, - ConfigParamReplace(op='str_replace', path='api_key', search_for='a', replace_with='a'), - 'Search string and replace string are the same: "a"', - ), - # Test 'remove' operation on non-existent path - ( - {'api_key': 'value'}, - ConfigParamRemove(op='remove', path='nonexistent_key'), - 'Path "nonexistent_key" does not exist', - ), - # Test 'remove' operation on non-existent nested path - ( - {'database': {'host': 'localhost'}}, - ConfigParamRemove(op='remove', path='database.nonexistent_field'), - 'Path "database.nonexistent_field" does not exist', - ), - # Test 'remove' operation on completely non-existent nested path - ( - {'api_key': 'value'}, - ConfigParamRemove(op='remove', path='nonexistent.nested.path'), - 'Path "nonexistent.nested.path" does not exist', - ), - # Test 'set' operation on nested value through string - ( - {'api_key': 'string_value'}, - ConfigParamSet(op='set', path='api_key.nested', value='new_value'), - 'Cannot set nested value at path "api_key.nested"', - ), - # Test 'set' operation on deeply nested value through string - ( - {'database': {'config': 'string_value'}}, - ConfigParamSet(op='set', path='database.config.host', value='localhost'), - 'Cannot set nested value at path "database.config.host"', - ), - # Test 'set' operation on nested value through number - ( - {'count': 42}, - ConfigParamSet(op='set', path='count.nested', value='new_value'), - 'Cannot set nested value at path "count.nested"', - ), - # Test 'set' operation on nested value through list - ( - {'items': [1, 2, 3]}, - ConfigParamSet(op='set', path='items.nested', value='new_value'), - 'Cannot set nested value at path "items.nested"', - ), - # Test 'set' operation on nested value through boolean - ( - {'flag': True}, - ConfigParamSet(op='set', path='flag.nested', value='new_value'), - 'Cannot set nested value at path "flag.nested"', - ), - # Test 'list_append' operation on non-existent path - ( - {'items': [1, 2, 3]}, - ConfigParamListAppend(op='list_append', path='nonexistent_list', value=4), - 'Path "nonexistent_list" does not exist', - ), - # Test 'list_append' operation on non-existent nested path - ( - {'config': {'values': [1, 2]}}, - ConfigParamListAppend(op='list_append', path='config.nonexistent', value=3), - 'Path "config.nonexistent" does not exist', - ), - # Test 'list_append' operation on non-list value (string) - ( - {'api_key': 'my_value'}, - ConfigParamListAppend(op='list_append', path='api_key', value='extra'), - 'Path "api_key" is not a list', - ), - # Test 'list_append' operation on non-list value (dict) - ( - {'config': {'host': 'localhost'}}, - ConfigParamListAppend(op='list_append', path='config', value='item'), - 'Path "config" is not a list', - ), - # Test 'list_append' operation on non-list value (number) - ( - {'count': 42}, - ConfigParamListAppend(op='list_append', path='count', value=1), - 'Path "count" is not a list', - ), - ], -) -def test_apply_param_update_errors( - params: dict[str, Any], - update: ConfigParamUpdate, - expected_error: str, -): - """Test _apply_param_update function with error cases.""" - with pytest.raises(ValueError, match=re.escape(expected_error)): - _apply_param_update(params, update) - - -@pytest.mark.parametrize( - ('params', 'updates', 'expected'), - [ - # Test with multiple operations - ( - { - 'api_key': 'old_key', - 'database': {'host': 'localhost', 'port': 5432}, - 'deprecated_field': 'old_value', - }, - [ - ConfigParamSet(op='set', path='api_key', value='new_key'), - ConfigParamReplace( - op='str_replace', path='database.host', search_for='localhost', replace_with='remotehost' - ), - ConfigParamRemove(op='remove', path='deprecated_field'), - ], - { - 'api_key': 'new_key', - 'database': {'host': 'remotehost', 'port': 5432}, - }, - ), - # Test with single update - ( - {'api_key': 'old_key'}, - [ConfigParamSet(op='set', path='api_key', value='new_key')], - {'api_key': 'new_key'}, - ), - # Test with empty updates list - ( - {'api_key': 'value'}, - [], - {'api_key': 'value'}, - ), - # Test sequential dependency - set then modify - ( - {'config': {}}, - [ - ConfigParamSet(op='set', path='config.url', value='http://old.example.com'), - ConfigParamReplace(op='str_replace', path='config.url', search_for='old', replace_with='new'), - ], - {'config': {'url': 'http://new.example.com'}}, - ), - # Test sequential dependency - set, modify, then set another dependent value - ( - {}, - [ - ConfigParamSet(op='set', path='database.host', value='localhost'), - ConfigParamSet(op='set', path='database.port', value=5432), - ConfigParamSet(op='set', path='database.ssl', value=True), - ], - {'database': {'host': 'localhost', 'port': 5432, 'ssl': True}}, - ), - # Test order matters - set, replace, then set again - ( - {'value': 'initial'}, - [ - ConfigParamReplace(op='str_replace', path='value', search_for='initial', replace_with='modified'), - ConfigParamSet(op='set', path='value', value='final'), - ], - {'value': 'final'}, - ), - ], -) -def test_update_params( - params: dict[str, Any], - updates: Sequence[ConfigParamUpdate], - expected: dict[str, Any], -): - """Test update_params function with valid operations.""" - result = update_params(params, updates) - assert result == expected - - -def test_update_params_does_not_mutate_original_dict(): - """Test that update_params does NOT mutate the original params dict.""" - params = {'api_key': 'old_key', 'count': 42} - updates = [ - ConfigParamSet(op='set', path='api_key', value='new_key'), - ConfigParamSet(op='set', path='count', value=100), - ] - - result = update_params(params, updates) - - # The function returns a new dict with updates - assert result == {'api_key': 'new_key', 'count': 100} - # The original dict is unchanged - assert params == {'api_key': 'old_key', 'count': 42} - # They are different objects - assert result is not params - - -def test_update_params_with_error_in_middle(): - """Test that update_params raises error if any update fails, and original dict is unchanged.""" - params = {'api_key': 'value', 'count': 42} - original_params = params.copy() - updates = [ - ConfigParamSet(op='set', path='api_key', value='new_key'), - ConfigParamRemove(op='remove', path='nonexistent_field'), # This will fail - ConfigParamSet(op='set', path='count', value=100), # This won't be reached - ] - - with pytest.raises(ValueError, match='Path "nonexistent_field" does not exist'): - update_params(params, updates) - - # Original dict is completely unchanged (no mutations) - assert params == original_params - assert params == {'api_key': 'value', 'count': 42} - - -@pytest.mark.parametrize( - ('data', 'path', 'value', 'expected_error'), - [ - # Test setting through string - ( - {'api_key': 'string_value'}, - 'api_key.nested', - 'new_value', - 'Cannot set nested value at path "api_key.nested": encountered non-dict value at "api_key" (type: str)', - ), - # Test setting through number - ( - {'count': 42}, - 'count.nested', - 'new_value', - 'Cannot set nested value at path "count.nested": encountered non-dict value at "count" (type: int)', - ), - # Test setting through list - ( - {'items': [1, 2, 3]}, - 'items.nested', - 'new_value', - 'Cannot set nested value at path "items.nested": encountered non-dict value at "items" (type: list)', - ), - # Test setting through boolean - ( - {'flag': True}, - 'flag.nested', - 'new_value', - 'Cannot set nested value at path "flag.nested": encountered non-dict value at "flag" (type: bool)', - ), - # Test setting through None - ( - {'value': None}, - 'value.nested', - 'new_value', - 'Cannot set nested value at path "value.nested": encountered non-dict value at "value" (type: NoneType)', - ), - # Test deeply nested path with non-dict in middle - ( - {'database': {'config': 'string_value'}}, - 'database.config.host.port', - 5432, - ( - 'Cannot set nested value at path "database.config.host.port": ' - 'encountered non-dict value at "database.config" (type: str)' - ), - ), - ], -) -def test_set_nested_value_through_non_dict_errors( - data: dict[str, Any], - path: str, - value: Any, - expected_error: str, -): - """Test _set_nested_value raises error when encountering non-dict in path.""" - with pytest.raises(ValueError, match=re.escape(expected_error)): - set_nested_value(data, path, value) - - -@pytest.mark.parametrize( - ('parameters', 'expected_markdown'), - [ - # Test with single block and single code - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Main Block', - 'codes': [ - { - 'id': 'b0.c0', - 'name': 'Select Data', - 'script': "SELECT * FROM customers WHERE status = 'active';", - } - ], - } - ] - }, - ( - '## Updated Transformation Structure\n' - '\n' - '### Block id: `b0`, name: `Main Block`\n' - '\n' - '- **Code id: `b0.c0`, name: `Select Data`** SQL snippet:\n' - '\n' - ' ```sql\n' - " SELECT * FROM customers WHERE status = 'active';\n" - ' ```\n' - ), - ), - # Test with multiple blocks and codes - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Data Extraction', - 'codes': [ - { - 'id': 'b0.c0', - 'name': 'Extract Customers', - 'script': 'SELECT id, name, email FROM customers;', - }, - { - 'id': 'b0.c1', - 'name': 'Extract Orders', - 'script': 'SELECT order_id, customer_id, amount FROM orders;', - }, - ], - }, - { - 'id': 'b1', - 'name': 'Data Transformation', - 'codes': [ - { - 'id': 'b1.c0', - 'name': 'Aggregate Data', - 'script': 'SELECT customer_id, SUM(amount) as total FROM orders GROUP BY customer_id;', - } - ], - }, - ] - }, - ( - '## Updated Transformation Structure\n' - '\n' - '### Block id: `b0`, name: `Data Extraction`\n' - '\n' - '- **Code id: `b0.c0`, name: `Extract Customers`** SQL snippet:\n' - '\n' - ' ```sql\n' - ' SELECT id, name, email FROM customers;\n' - ' ```\n' - '\n' - '- **Code id: `b0.c1`, name: `Extract Orders`** SQL snippet:\n' - '\n' - ' ```sql\n' - ' SELECT order_id, customer_id, amount FROM orders;\n' - ' ```\n' - '\n' - '### Block id: `b1`, name: `Data Transformation`\n' - '\n' - '- **Code id: `b1.c0`, name: `Aggregate Data`** SQL snippet:\n' - '\n' - ' ```sql\n' - ' SELECT customer_id, SUM(amount) as total FROM orders GROUP BY customer_id;\n' - ' ```\n' - ), - ), - # Test with multiline SQL script - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Complex Query', - 'codes': [ - { - 'id': 'b0.c0', - 'name': 'Multi-line Select', - 'script': ( - 'SELECT\n' - ' customer_id,\n' - ' SUM(amount) as total,\n' - ' COUNT(*) as order_count\n' - 'FROM orders\n' - "WHERE status = 'completed'\n" - 'GROUP BY customer_id;' - ), - } - ], - } - ] - }, - ( - '## Updated Transformation Structure\n' - '\n' - '### Block id: `b0`, name: `Complex Query`\n' - '\n' - '- **Code id: `b0.c0`, name: `Multi-line Select`** SQL snippet:\n' - '\n' - ' ```sql\n' - ' SELECT\n customer_id,\n SUM(amount) as total,\n COUNT(*) as order_count\n' - "FROM orders\nWHERE status = 'completed'\nGROUP BY customer_id;\n" - ' ```\n' - ), - ), - # Test with empty script - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Empty Block', - 'codes': [ - { - 'id': 'b0.c0', - 'name': 'Empty Code', - 'script': '', - } - ], - } - ] - }, - ( - '## Updated Transformation Structure\n' - '\n' - '### Block id: `b0`, name: `Empty Block`\n' - '\n' - '- **Code id: `b0.c0`, name: `Empty Code`** SQL snippet:\n' - '\n' - ' *Empty script*\n' - ), - ), - # Test with block containing no codes - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Block Without Codes', - 'codes': [], - } - ] - }, - ( - '## Updated Transformation Structure\n' - '\n' - '### Block id: `b0`, name: `Block Without Codes`\n' - '\n' - '*No code blocks*\n' - ), - ), - # Test with empty blocks list - ( - {'blocks': []}, - '## Updated Transformation Structure\n\nNo blocks found in transformation.\n', - ), - # Test with very long script (truncation) - ( - { - 'blocks': [ - { - 'id': 'b0', - 'name': 'Long Script Block', - 'codes': [ - { - 'id': 'b0.c0', - 'name': 'Very Long Query', - 'script': ( - 'SELECT column1, column2, column3, column4, column5, column6, ' - 'column7, column8, column9, column10, column11, column12, ' - 'column13, column14, column15, column16 FROM very_large_table ' - 'WHERE condition1 = true;' - ), - } - ], - } - ] - }, - ( - '## Updated Transformation Structure\n' - '\n' - '### Block id: `b0`, name: `Long Script Block`\n' - '\n' - '- **Code id: `b0.c0`, name: `Very Long Query`** SQL snippet:\n' - '\n' - ' ```sql\n' - ' SELECT column1, column2, column3, column4, column5, column6, column7, ' - 'column8, column9, column10, column11, column12, column13, column14, column15, ' - 'co... (53 chars truncated)\n' - ' ```\n' - ), - ), - ], -) -def test_structure_summary(parameters: dict[str, Any], expected_markdown: str): - """Test structure_summary function generates correct markdown output.""" - result = structure_summary(parameters) - assert result == expected_markdown - - -@pytest.mark.parametrize( - ('initial_params', 'updates', 'expected_params', 'expected_msg'), - [ - # String replacement without structure change - should only report replacement - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - SimplifiedTfBlocks.Block.Code(name='Code Y', script='SELECT * FROM table2'), - ], - ), - ] - ), - [ - TfStrReplace(op='str_replace', block_id=None, code_id=None, search_for='FROM', replace_with='IN'), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * IN table1'), - SimplifiedTfBlocks.Block.Code(name='Code Y', script='SELECT * IN table2'), - ], - ), - ] - ), - 'Replaced 2 occurrences of "FROM" in the transformation', - ), - # Structural change without string replacement - should only report structure - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - [ - TfAddBlock( - op='add_block', - block=SimplifiedTfBlocks.Block(name='New Block', codes=[]), - position='end', - ), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - SimplifiedTfBlocks.Block(name='New Block', codes=[]), - ] - ), - 'Added block with name "New Block"\n## Updated Transformation Structure', - ), - # Non-structural operations - should return empty message - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - [ - TfRenameBlock(op='rename_block', block_id='b0', block_name='Renamed Block'), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Renamed Block', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - '', - ), - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - [ - TfRenameCode(op='rename_code', block_id='b0', code_id='b0.c0', code_name='Renamed Code'), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Renamed Code', script='SELECT * FROM table1'), - ], - ), - ] - ), - '', - ), - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - [ - TfSetCode(op='set_code', block_id='b0', code_id='b0.c0', script='SELECT * FROM new_table'), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM new_table'), - ], - ), - ] - ), - "Changed code with id 'b0.c0' in block 'b0'", - ), - # Multiple non-structural operations - should return message from set_code - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - [ - TfRenameBlock(op='rename_block', block_id='b0', block_name='Renamed Block'), - TfSetCode(op='set_code', block_id='b0', code_id='b0.c0', script='SELECT * FROM new_table'), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Renamed Block', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM new_table'), - ], - ), - ] - ), - "Changed code with id 'b0.c0' in block 'b0'", - ), - # Structural change + string replacement - should report both - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - [ - TfAddBlock( - op='add_block', - block=SimplifiedTfBlocks.Block( - name='New Block', - codes=[SimplifiedTfBlocks.Block.Code(name='New Code', script='SELECT * FROM table2')], - ), - position='end', - ), - TfStrReplace(op='str_replace', block_id=None, code_id=None, search_for='FROM', replace_with='IN'), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * IN table1'), - ], - ), - SimplifiedTfBlocks.Block( - name='New Block', - codes=[ - SimplifiedTfBlocks.Block.Code(name='New Code', script='SELECT * IN table2'), - ], - ), - ] - ), - ( - 'Added block with name "New Block"\n' - 'Replaced 2 occurrences of "FROM" in the transformation\n## Updated Transformation Structure' - ), - ), - # Multiple string replacements - should report all - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - SimplifiedTfBlocks.Block.Code(name='Code Y', script='SELECT * FROM table2'), - ], - ), - ] - ), - [ - TfStrReplace( - op='str_replace', block_id='b0', code_id='b0.c0', search_for='table1', replace_with='new_table1' - ), - TfStrReplace( - op='str_replace', block_id='b0', code_id='b0.c1', search_for='table2', replace_with='new_table2' - ), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM new_table1'), - SimplifiedTfBlocks.Block.Code(name='Code Y', script='SELECT * FROM new_table2'), - ], - ), - ] - ), - ( - 'Replaced 1 occurrence of "table1" in code "b0.c0", block "b0"\n' - 'Replaced 1 occurrence of "table2" in code "b0.c1", block "b0"' - ), - ), - # Add code (structural) + string replacement - should report both - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - [ - TfAddCode( - op='add_code', - block_id='b0', - code=SimplifiedTfBlocks.Block.Code(name='New Code', script='SELECT * FROM table2'), - position='end', - ), - TfStrReplace(op='str_replace', block_id=None, code_id=None, search_for='FROM', replace_with='IN'), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * IN table1'), - SimplifiedTfBlocks.Block.Code(name='New Code', script='SELECT * IN table2'), - ], - ), - ] - ), - 'Added code with name "New Code"\nReplaced 2 occurrences of "FROM" in ' - 'the transformation\n## Updated Transformation Structure', - ), - # Remove code (structural) - should report structure - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - SimplifiedTfBlocks.Block.Code(name='Code Y', script='SELECT * FROM table2'), - ], - ), - ] - ), - [ - TfRemoveCode(op='remove_code', block_id='b0', code_id='b0.c0'), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code Y', script='SELECT * FROM table2'), - ], - ), - ] - ), - '## Updated Transformation Structure', - ), - # Multiple structural changes - should report structure once - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - [ - TfAddBlock( - op='add_block', - block=SimplifiedTfBlocks.Block(name='New Block', codes=[]), - position='end', - ), - TfAddCode( - op='add_code', - block_id='b0', - code=SimplifiedTfBlocks.Block.Code(name='New Code', script='SELECT 1'), - position='end', - ), - ], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - SimplifiedTfBlocks.Block.Code(name='New Code', script='SELECT 1'), - ], - ), - SimplifiedTfBlocks.Block(name='New Block', codes=[]), - ] - ), - ( - 'Added block with name "New Block"\n' - 'Added code with name "New Code"\n' - '## Updated Transformation Structure' - ), - ), - # Empty updates list - should return empty message - ( - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - [], - SimplifiedTfBlocks( - blocks=[ - SimplifiedTfBlocks.Block( - name='Block A', - codes=[ - SimplifiedTfBlocks.Block.Code(name='Code X', script='SELECT * FROM table1'), - ], - ), - ] - ), - '', - ), - ], -) -def test_update_transformation_parameters( - initial_params: SimplifiedTfBlocks, - updates: Sequence[TfParamUpdate], - expected_params: SimplifiedTfBlocks, - expected_msg: str, -): - result_params, result_msg = update_transformation_parameters(initial_params, updates, sql_dialect='snowflake') - - assert result_params == expected_params - - if '##' in expected_msg: - # For multi-line messages, check message prefix - assert result_msg.startswith(expected_msg) - else: - # For simple patterns, check exact match - assert result_msg == expected_msg - - -# ============================================================================ -# get_transformation_folders / set_configuration_folder_metadata TESTS -# ============================================================================ - - -def _make_client( - configs: list[dict[str, Any]], - folder_configs: list[dict[str, Any]], -) -> MagicMock: - """Create a minimal mock KeboolaClient for folder tests.""" - client = MagicMock() - client.storage_client.configuration_list = AsyncMock(return_value=configs) - client.storage_client.component_configurations_search = AsyncMock(return_value=folder_configs) - client.storage_client.configuration_metadata_update = AsyncMock(return_value=[]) - return client - - -@pytest.mark.parametrize( - ('all_configs', 'expected_count'), - [ - ([], 0), - ([{'id': str(i)} for i in range(5)], 5), - ], - ids=['no_configs', 'few_configs'], -) -@pytest.mark.asyncio -async def test_get_config_folders_short_circuit( - all_configs: list[dict[str, Any]], - expected_count: int, -) -> None: - """Test that configuration_list is still called (and returns early) when total < 20.""" - client = _make_client(all_configs, []) - count, folders, lower_bound = await get_config_folders(client, 'keboola.snowflake-transformation') - assert count == expected_count - assert folders == [] - assert lower_bound is False - client.storage_client.component_configurations_search.assert_called_once_with( - component_id='keboola.snowflake-transformation', - metadata_keys=[MetadataField.CONFIGURATION_FOLDER_NAME], - ) - client.storage_client.configuration_list.assert_called_once_with(component_id='keboola.snowflake-transformation') - - -@pytest.mark.asyncio -async def test_get_config_folders_skips_list_when_enough_folder_configs() -> None: - """Test that configuration_list is skipped when ≥20 configs already have folder metadata.""" - folder_configs = [ - { - 'id': str(i), - 'componentId': 'keboola.snowflake-transformation', - 'metadata': [{'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': f'Folder{i % 5}'}], - } - for i in range(22) - ] - client = _make_client([], folder_configs) # configuration_list returns [] but should not be called - count, folders, lower_bound = await get_config_folders(client, 'keboola.snowflake-transformation') - assert count == 22 - assert len(folders) == 5 # 22 configs across 5 distinct folders - assert lower_bound is True - client.storage_client.component_configurations_search.assert_called_once_with( - component_id='keboola.snowflake-transformation', - metadata_keys=[MetadataField.CONFIGURATION_FOLDER_NAME], - ) - client.storage_client.configuration_list.assert_not_called() - - -_MANY_CONFIGS = [{'id': str(i)} for i in range(25)] - - -@pytest.mark.parametrize( - ('folder_configs', 'expected_folders'), - [ - # No folder metadata on any config - ([], []), - # Two configs with distinct folders - ( - [ - { - 'id': '1', - 'componentId': 'keboola.snowflake-transformation', - 'metadata': [{'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'Analytics'}], - }, - { - 'id': '2', - 'componentId': 'keboola.snowflake-transformation', - 'metadata': [{'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'Sales'}], - }, - ], - ['Analytics', 'Sales'], - ), - # Duplicate folder names are deduplicated - ( - [ - { - 'id': '1', - 'componentId': 'keboola.snowflake-transformation', - 'metadata': [{'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'Analytics'}], - }, - { - 'id': '2', - 'componentId': 'keboola.snowflake-transformation', - 'metadata': [{'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'Analytics'}], - }, - ], - ['Analytics'], - ), - ], - ids=['no_folders', 'distinct_folders', 'deduplicated_folders'], -) -@pytest.mark.asyncio -async def test_get_config_folders( - folder_configs: list[dict[str, Any]], - expected_folders: list[str], -) -> None: - """Test get_config_folders when count >= 20 (search endpoint is called).""" - client = _make_client(_MANY_CONFIGS, folder_configs) - count, folders, lower_bound = await get_config_folders(client, 'keboola.snowflake-transformation') - assert count == len(_MANY_CONFIGS) - assert folders == expected_folders - assert lower_bound is False - client.storage_client.configuration_list.assert_called_once_with(component_id='keboola.snowflake-transformation') - client.storage_client.component_configurations_search.assert_called_once_with( - component_id='keboola.snowflake-transformation', - metadata_keys=[MetadataField.CONFIGURATION_FOLDER_NAME], - ) - - -@pytest.mark.parametrize( - ('folder', 'expected_saved', 'expect_call'), - [ - ('Analytics', 'Analytics', True), - (' Analytics ', 'Analytics', True), # whitespace stripped - ('', None, False), - (' ', None, False), # whitespace-only skipped - ], - ids=['normal', 'whitespace_stripped', 'empty', 'whitespace_only'], -) -@pytest.mark.asyncio -async def test_set_configuration_folder_metadata( - folder: str, - expected_saved: str | None, - expect_call: bool, -) -> None: - """Test set_configuration_folder_metadata: strips whitespace, skips empty, propagates errors.""" - client = _make_client([], []) - await set_configuration_folder_metadata(client, 'keboola.snowflake-transformation', 'cfg-1', folder) - if expect_call: - client.storage_client.configuration_metadata_update.assert_called_once_with( - component_id='keboola.snowflake-transformation', - configuration_id='cfg-1', - metadata={MetadataField.CONFIGURATION_FOLDER_NAME: expected_saved}, - ) - else: - client.storage_client.configuration_metadata_update.assert_not_called() - - -@pytest.mark.parametrize( - ('metadata', 'expected_delete_ids'), - [ - ([], []), - ([{'key': 'other.key', 'id': '99'}], []), - ([{'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'id': '1'}], ['1']), - ( - [ - {'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'id': '1'}, - {'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'id': '2'}, - ], - ['1', '2'], - ), - ([{'key': MetadataField.CONFIGURATION_FOLDER_NAME}], []), - ], - ids=['no_metadata', 'other_key', 'single_match', 'multiple_matches_all_deleted', 'missing_id_skipped'], -) -@pytest.mark.asyncio -async def test_clear_configuration_folder_metadata( - metadata: list[dict[str, Any]], - expected_delete_ids: list[str], -) -> None: - """Test clear_configuration_folder_metadata deletes all matching entries.""" - client = _make_client([], []) - client.storage_client.configuration_metadata_get = AsyncMock(return_value=metadata) - client.storage_client.configuration_metadata_delete = AsyncMock() - - await clear_configuration_folder_metadata(client, 'keboola.snowflake-transformation', 'cfg-1') - - assert client.storage_client.configuration_metadata_delete.call_count == len(expected_delete_ids) - for metadata_id in expected_delete_ids: - client.storage_client.configuration_metadata_delete.assert_any_call( - component_id='keboola.snowflake-transformation', - configuration_id='cfg-1', - metadata_id=metadata_id, - ) diff --git a/tests/tools/components/test_validation.py b/tests/tools/components/test_validation.py deleted file mode 100644 index e5859d9a0..000000000 --- a/tests/tools/components/test_validation.py +++ /dev/null @@ -1,870 +0,0 @@ -import copy -import json -import logging -from typing import Optional - -import jsonschema -import pytest - -from keboola_mcp_server.clients.client import CONDITIONAL_FLOW_COMPONENT_ID, ORCHESTRATOR_COMPONENT_ID -from keboola_mcp_server.clients.storage import ComponentAPIResponse, JsonDict -from keboola_mcp_server.tools import validation -from keboola_mcp_server.tools.components.model import Component - - -@pytest.mark.parametrize( - ('schema_name', 'expected_keywords'), - [ - ( - validation.ConfigurationSchemaResources.STORAGE, - ['type', 'properties', 'storage', 'input', 'output', 'tables', 'files', 'destination', 'source'], - ) - ], -) -def test_load_schema(schema_name, expected_keywords): - schema = validation._load_schema(schema_name) - assert schema is not None - for keyword in expected_keywords: - assert keyword in str(schema) - - -@pytest.mark.parametrize( - ('valid_storage_path'), - [ - # 1. Output table with delete_where using where_filters - ('tests/resources/storage/storage_valid_1.json'), - # 2. Minimal valid input and output tables - ('tests/resources/storage/storage_valid_2.json'), - # 3. Input and output files - ('tests/resources/storage/storage_valid_3.json'), - # 4. Input table with where_column and where_values, output table with schema - ('tests/resources/storage/storage_valid_4.json'), - # 5. Output table with unload_strategy='direct-grant' and no 'source' (data apps with Storage Access) - ('tests/resources/storage/storage_valid_5.json'), - ], -) -def test_validate_storage_valid(valid_storage_path: str): - with open(valid_storage_path, 'r') as f: - valid_storage = json.load(f) - # returns the same valid storage no exception is raised - assert validation.validate_storage_configuration_against_schema(valid_storage) == valid_storage - - -@pytest.mark.parametrize( - ('invalid_storage_path'), - [ - # 1. Input table missing required property (source or source_search) - # Each item in tables must have either source or source_search (enforced by oneOf). This object has neither. - ('tests/resources/storage/storage_invalid_1.json'), - # 2. The table_files item missing required destination (output table_files) - # Each item in table_files must have both source and destination. - ('tests/resources/storage/storage_invalid_2.json'), - # 3. Missing source in input, missing destination in output - ('tests/resources/storage/storage_invalid_3.json'), - # 4. Schema present with forbidden properties (output tables) - # If schema is present, columns (and several other properties) must not be present (enforced by allOf). - ('tests/resources/storage/storage_invalid_4.json'), - # 5. Output table missing required property (destination or source) - # Both destination and source are required for each output table. - ('tests/resources/storage/storage_invalid_5.json'), - # 6. The where_operator has an invalid value (input tables) - # where_operator must be either 'eq' or 'ne', not 'gt'. - ('tests/resources/storage/storage_invalid_6.json'), - # 7. The files item missing required source (output files) - # Each item in files must have a source property - ('tests/resources/storage/storage_invalid_7.json'), - ], -) -def test_validate_storage_invalid(invalid_storage_path: str): - """We expect the json will not be validated and raise a RecoverableValidationError""" - with open(invalid_storage_path, 'r') as f: - invalid_storage = json.load(f) - with pytest.raises(validation.RecoverableValidationError) as exc_info: - validation.validate_storage_configuration_against_schema( - invalid_storage, initial_message='This is a test message' - ) - err = exc_info.value - assert 'This is a test message' in str(err) - assert 'Failed validating' in str(err) - assert f'{json.dumps(invalid_storage, indent=2)}' not in str(err) - - -@pytest.mark.parametrize( - ('input_storage', 'output_storage'), - [ - ({'input': {}, 'output': {}}, {'input': {}, 'output': {}}), - ({'storage': {'input': {}, 'output': {}}}, {'storage': {'input': {}, 'output': {}}}), - ], -) -def test_validate_storage_output_format(input_storage, output_storage): - """Test that storage configuration validation preserves the input format - whether the input contains a 'storage' - key or not, the output will match the input structure exactly.""" - result = validation.validate_storage_configuration_against_schema(input_storage) - assert result == output_storage - - -@pytest.mark.parametrize( - ('input_parameters', 'output_parameters'), - [ - ({'a': 1}, {'a': 1}), - ({'parameters': {'a': 1, 'b': 2}}, {'parameters': {'a': 1, 'b': 2}}), - ], -) -def test_validate_parameters_output_format(input_parameters, output_parameters): - """Test that parameters configuration validation preserves the input format - whether the input contains a - 'parameters' key or not, the output will match the input structure exactly.""" - accepting_schema = {'type': 'object', 'additionalProperties': True} # accepts any json object - result = validation._validate_parameters_configuration_against_schema(input_parameters, accepting_schema) - assert result == output_parameters - - -@pytest.mark.parametrize( - ('valid_flow_path'), - [ - ('tests/resources/flow/flow_valid_1.json'), - ('tests/resources/flow/flow_valid_2.json'), - ('tests/resources/flow/flow_valid_3.json'), - ], -) -def test_validate_flow_valid(valid_flow_path: str): - with open(valid_flow_path, 'r') as f: - valid_flow = json.load(f) - assert ( - validation.validate_flow_configuration_against_schema(valid_flow, flow_type=ORCHESTRATOR_COMPONENT_ID) - == valid_flow - ) - - -@pytest.mark.parametrize( - 'invalid_flow_path', - [ - 'tests/resources/flow/flow_invalid_1.json', - 'tests/resources/flow/flow_invalid_2.json', - 'tests/resources/flow/flow_invalid_3.json', - 'tests/resources/flow/flow_invalid_4.json', - 'tests/resources/flow/flow_invalid_5.json', - 'tests/resources/flow/flow_invalid_6.json', - ], -) -def test_validate_flow_invalid(invalid_flow_path: str): - with open(invalid_flow_path, 'r') as f: - invalid_flow = json.load(f) - with pytest.raises(validation.RecoverableValidationError): - validation.validate_flow_configuration_against_schema(invalid_flow, flow_type=ORCHESTRATOR_COMPONENT_ID) - - -def test_validate_json_against_schema_invalid_schema(caplog): - """ - We expect passing when the schema is invalid since it is not an Agent error. - However, we expect logging the error. - """ - corrupted_schema = {'type': 'int', 'minimum': 5} - with caplog.at_level(logging.ERROR): - validation._validate_json_against_schema( - json_data={'foo': 1}, schema=corrupted_schema, initial_message='This is a test message' - ) - assert f'schema: {corrupted_schema}' in caplog.text - - -def test_recoverable_validation_error_str(): - err = jsonschema.ValidationError('Validation error', instance={'foo': 1}) - rve = validation.RecoverableValidationError.create_from_values( - err, - initial_message='Initial msg', - validation_context=validation.ValidationContext( - component_id='keboola.ex-test', - configuration_id='cfg-1', - configuration_row_id='row-1', - scope='parameters', - ), - ) - s = str(rve) - assert 'Validation error' in s - assert 'Initial msg' in s - assert ( - 'Validation component context: ' - 'component_id=keboola.ex-test, configuration_id=cfg-1, configuration_row_id=row-1, scope=parameters' in s - ) - assert '"foo": 1' not in s - - -ROOT_SCHEMA_PATH = 'tests/resources/parameters/root_parameters_schema.json' -ROW_SCHEMA_PATH = 'tests/resources/parameters/row_parameters_schema.json' - - -_MULTI_REQUIRED_SCHEMA: JsonDict = { - 'type': 'object', - 'required': ['api_key', 'endpoint', 'timeout'], - 'properties': { - 'api_key': {'type': 'string'}, - 'endpoint': {'type': 'string'}, - 'timeout': {'type': 'integer'}, - }, -} - - -@pytest.mark.parametrize( - ('schema_or_path', 'invalid_data', 'expected_in_str', 'not_expected_in_str', 'validation_context'), - [ - # Case 1: missing required property at root level - # Only the violated 'required' list should appear, not the full schema object - # The HINT must list ALL required fields so the agent can fix everything in one retry - # validation_context with scope='parameters' is required for the HINT to appear - ( - ROOT_SCHEMA_PATH, - {'qdrant_settings': {'url': 'http://localhost:6333', '#api_key': 'key'}}, - [ - "'embedding_settings' is a required property", - "Failed validating 'required' in schema", - '"embedding_settings"', # the required list value is shown - 'HINT: Ensure ALL of the following required fields are present in `parameters`', - '`embedding_settings`', # required field listed in the hint - 'get_components', # hint directs agent to look up the schema - ], - ['azure_settings', 'huggingface_settings', 'google_vertex_settings'], # full schema not dumped - validation.ValidationContext(component_id='keboola.ex-test', scope='parameters'), - ), - # Case 2: invalid enum value for provider_type - # Only the 'enum' list should appear at the precise schema path, not the full provider_type subschema - ( - ROOT_SCHEMA_PATH, - {'embedding_settings': {'provider_type': 'gpt-9000'}}, - [ - "'gpt-9000' is not one of", - "Failed validating 'enum' in schema['properties']['embedding_settings']['properties']['provider_type']" - "['enum']", - "On instance['embedding_settings']['provider_type']", - '"openai"', # enum values are shown - '"gpt-9000"', # the bad value is shown - ], - ['azure_settings', 'huggingface_settings', '"title"'], # full subschema properties not dumped - None, - ), - # Case 3: wrong type - batch_size must be integer, not string - # Only the 'type' constraint should appear at the precise schema path - ( - ROW_SCHEMA_PATH, - {'text_column': 'notes', 'advanced_options': {'batch_size': 'not-a-number'}}, - [ - "is not of type 'integer'", - "Failed validating 'type' in schema['properties']['advanced_options']['properties']['batch_size']" - "['type']", - "On instance['advanced_options']['batch_size']", - '"type": "integer"', # the type constraint value is shown - '"not-a-number"', # the bad value is shown - ], - ['enable_chunking', 'chunking_settings', '"title"'], # full subschema not dumped - None, - ), - # Case 4: minimum constraint violation - batch_size below minimum of 1 - # Only the 'minimum' constraint value should appear; HINT must NOT appear for non-required errors - ( - ROW_SCHEMA_PATH, - {'text_column': 'notes', 'advanced_options': {'batch_size': 0}}, - [ - '0 is less than the minimum of 1', - "Failed validating 'minimum' in schema['properties']['advanced_options']['properties']['batch_size']" - "['minimum']", - "On instance['advanced_options']['batch_size']", - '"minimum": 1', # the minimum value is shown - ], - ['enable_chunking', 'chunking_settings', '"title"', 'HINT:'], # full subschema not dumped; no hint - None, - ), - # Case 5: multiple required fields missing — HINT must list ALL of them - # Verifies the hint format when validator_value contains more than one field name - ( - _MULTI_REQUIRED_SCHEMA, - {}, # all three required fields missing - [ - 'HINT: Ensure ALL of the following required fields are present in `parameters`', - '`api_key`', - '`endpoint`', - '`timeout`', - 'get_components', - ], - [], - validation.ValidationContext(component_id='keboola.ex-test', scope='parameters'), - ), - # Case 6: required violation without parameters scope — HINT must NOT appear - # Verifies that hint is suppressed when scope != 'parameters' (e.g. storage or no context) - ( - _MULTI_REQUIRED_SCHEMA, - {}, - ['is a required property'], - ['HINT:'], - None, # no validation_context → no scope → hint suppressed - ), - ], -) -def test_recoverable_validation_error_compact_format( - schema_or_path: str | JsonDict, - invalid_data: JsonDict, - expected_in_str: list, - not_expected_in_str: list, - validation_context: validation.ValidationContext | None, -): - """Verify that RecoverableValidationError.__str__ shows only the violated schema constraint, - not the entire schema object, and that the required-field HINT only appears for parameters scope.""" - if isinstance(schema_or_path, str): - with open(schema_or_path) as f: - schema = json.load(f) - else: - schema = copy.deepcopy(schema_or_path) - - with pytest.raises(validation.RecoverableValidationError) as exc_info: - validation._validate_parameters_configuration_against_schema( - invalid_data, schema, validation_context=validation_context - ) - - err_str = str(exc_info.value) - for fragment in expected_in_str: - assert fragment in err_str, f'Expected {fragment!r} to be in error string:\n{err_str}' - for fragment in not_expected_in_str: - assert fragment not in err_str, f'Expected {fragment!r} NOT to be in error string:\n{err_str}' - - -@pytest.mark.parametrize( - ('input_schema', 'expected_schema'), - [ - # Case 1: required true -> remove the required field - ({'type': 'object', 'required': True}, {'type': 'object'}), - # Case 2: required false -> remove the required field - ({'type': 'object', 'required': False}, {'type': 'object'}), - # Case 3: required as list (should remain unchanged) - ({'type': 'object', 'required': ['foo', 'bar']}, {'type': 'object', 'required': ['foo', 'bar']}), - # Case 4: required missing (should not be added) - ({'type': 'object'}, {'type': 'object'}), - # Case 5: nested properties with required true/false - ( - { - 'type': 'object', - 'required': ['foo'], - 'properties': { - 'foo': { - 'type': 'string', - 'required': ['foo2'], - 'properties': {'foo2': {'type': 'string', 'required': True}}, - }, - 'bar': {'type': 'number', 'required': False}, - 'baz': {'type': 'boolean', 'required': ['baz']}, - }, - }, - { - 'type': 'object', - 'required': ['foo'], - 'properties': { - 'foo': { - 'type': 'string', - 'required': ['foo2'], - 'properties': {'foo2': {'type': 'string'}}, - }, - 'bar': {'type': 'number'}, - 'baz': {'type': 'boolean', 'required': ['baz']}, - }, - }, - ), - # Case 6: nested properties with required true/false - add if required and remove if Not - ( - { - 'type': 'object', - 'required': ['foo2'], - 'properties': { - 'foo': {'type': 'string', 'required': True}, - 'foo2': {'type': 'string', 'required': 'False'}, - }, - }, - { - 'type': 'object', - 'required': ['foo'], - 'properties': { - 'foo': {'type': 'string'}, - 'foo2': {'type': 'string'}, - }, - }, - ), - # Case 7: properties values are not a dict type (should return as it is) - ({'properties': {'a': 1}}, {'properties': {'a': 1}}), - # Case 8: properties are an empty list should convert to an empty dict - ({'properties': []}, {'properties': {}}), - # Case 9: required as string -> remove the required field - ({'type': 'object', 'required': 'yes'}, {'type': 'object'}), - # Case 10: required as int - remove the required field - ({'type': 'object', 'required': 1}, {'type': 'object'}), - # Case 11: empty schema should return an empty schema - ({}, {}), - # Case 12: empty enum stripping at top level - ({'type': 'string', 'enum': []}, {'type': 'string'}), - # Case 13: empty enum stripping inside properties - ( - {'type': 'object', 'properties': {'color': {'type': 'string', 'enum': []}}}, - {'type': 'object', 'properties': {'color': {'type': 'string'}}}, - ), - # Case 14: empty enum stripping inside items - ( - {'type': 'array', 'items': {'type': 'string', 'enum': []}}, - {'type': 'array', 'items': {'type': 'string'}}, - ), - # Case 15: empty enum stripping inside anyOf - ( - {'anyOf': [{'type': 'string', 'enum': []}, {'type': 'integer'}]}, - {'anyOf': [{'type': 'string'}, {'type': 'integer'}]}, - ), - # Case 16: empty enum stripping deeply nested (items -> properties) - ( - { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': {'status': {'type': 'string', 'enum': []}}, - }, - }, - { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': {'status': {'type': 'string'}}, - }, - }, - ), - # Case 17: non-empty enum should NOT be stripped - ( - {'type': 'string', 'enum': ['a', 'b']}, - {'type': 'string', 'enum': ['a', 'b']}, - ), - # Case 18: recursion into additionalProperties with empty enum and required normalization - ( - { - 'type': 'object', - 'additionalProperties': { - 'type': 'object', - 'properties': {'x': {'type': 'string', 'enum': [], 'required': True}}, - }, - }, - { - 'type': 'object', - 'additionalProperties': { - 'type': 'object', - 'required': ['x'], - 'properties': {'x': {'type': 'string'}}, - }, - }, - ), - # Case 19: recursion into if/then/else and not - ( - { - 'if': {'properties': {'a': {'type': 'string', 'enum': []}}}, - 'then': {'properties': {'b': {'type': 'string', 'required': True}}}, - 'else': {'properties': {'c': {'type': 'number', 'enum': []}}}, - 'not': {'type': 'object', 'properties': {'d': {'type': 'string', 'enum': []}}}, - }, - { - 'if': {'properties': {'a': {'type': 'string'}}}, - 'then': {'required': ['b'], 'properties': {'b': {'type': 'string'}}}, - 'else': {'properties': {'c': {'type': 'number'}}}, - 'not': {'type': 'object', 'properties': {'d': {'type': 'string'}}}, - }, - ), - # Case 20: recursion into definitions/$defs - ( - { - 'definitions': {'color': {'type': 'string', 'enum': []}}, - '$defs': {'size': {'type': 'integer', 'enum': []}}, - }, - { - 'definitions': {'color': {'type': 'string'}}, - '$defs': {'size': {'type': 'integer'}}, - }, - ), - # Case 21: recursion into patternProperties - ( - { - 'type': 'object', - 'patternProperties': {'^S_': {'type': 'string', 'enum': []}}, - }, - { - 'type': 'object', - 'patternProperties': {'^S_': {'type': 'string'}}, - }, - ), - ], -) -def test_normalize_schema(input_schema: JsonDict, expected_schema: JsonDict): - result = validation.KeboolaParametersValidator.sanitize_schema(input_schema) - assert result == expected_schema - - -def test_validate_with_empty_enum(): - """Functional test: data validates against a schema containing 'enum': [] after sanitization.""" - schema = { - 'type': 'object', - 'properties': { - 'color': {'type': 'string', 'enum': []}, - }, - } - data = {'color': 'red'} - # Should NOT raise - empty enum is stripped during sanitization - validation.KeboolaParametersValidator.validate(data, schema) - - -@pytest.mark.parametrize( - ('input_schema'), - [ - # case 1: properties are non-empty list -> fail - {'type': 'object', 'properties': [{'type': 'string'}]}, - # case 2: properties are not a dict -> fail - {'type': 'object', 'properties': 1}, - ], -) -def test_normalize_schema_invalid_parameters(input_schema: JsonDict): - with pytest.raises(jsonschema.SchemaError): - validation.KeboolaParametersValidator.sanitize_schema(input_schema) - - -@pytest.mark.parametrize( - ('schema_path', 'json_data'), - [ - # we pass the schema and json_data which are expected to be valid - ('tests/resources/parameters/root_parameters_schema.json', {'embedding_settings': {'provider_type': 'openai'}}), - ( - 'tests/resources/parameters/row_parameters_schema.json', - {'text_column': 'this is the only required field of this schema'}, - ), - ], -) -def test_schema_validation(caplog, schema_path: str, json_data: JsonDict): - """Testing the failure of the jsonschema.validate and the success of the KeboolaParametersValidator.validate""" - with open(schema_path, 'r') as f: - schema = json.load(f) - - with caplog.at_level(logging.ERROR): - # we expect the error logging when schema is invalid but not failure since it is not an Agent error - validation._validate_json_against_schema(json_data, schema, validate_fn=jsonschema.validate) - assert f'schema: {schema}' in caplog.text - - try: - validation._validate_json_against_schema( - json_data, schema, validate_fn=validation.KeboolaParametersValidator.validate - ) - except jsonschema.ValidationError: - pytest.fail('ValidationError was raised when it should not have been') - - -@pytest.mark.parametrize( - ('schema_path', 'data_path', 'valid'), - [ - # text_column is required and correctly set to "notes" (exists in columns). - # primary_key is required only when load_type = incremental_load, and it's provided ("email"). - # Chunking settings are only present when enable_chunking = true, which is respected. - ( - 'tests/resources/parameters/row_parameters_schema.json', - 'tests/resources/parameters/row_parameters_valid.json', - True, - ), - # Missing required field: text_column - # Missing required field: primary_key (required when load_type = incremental_load) - # Invalid batch_size: 0 (minimum allowed: 1) - # Invalid chunk_size: 9000 (maximum allowed: 8000) - # Invalid chunk_overlap: -10 (minimum allowed: 0) - ( - 'tests/resources/parameters/row_parameters_schema.json', - 'tests/resources/parameters/row_parameters_invalid.json', - False, - ), - ], -) -def test_validate_row_parameters(schema_path: str, data_path: str, valid: bool): - with open(schema_path, 'r') as f: - schema = json.load(f) - with open(data_path, 'r') as f: - data = json.load(f) - if valid: - try: - validation._validate_parameters_configuration_against_schema(data, schema) - except jsonschema.ValidationError: - pytest.fail('ValidationError was raised when it should not have been') - else: - with pytest.raises(jsonschema.ValidationError): - validation._validate_parameters_configuration_against_schema(data, schema) - - -@pytest.mark.parametrize( - ('schema_path', 'data_path', 'valid'), - [ - # embedding_settings is required. - # When provider_type is "openai", the openai_settings object must include model and #api_key. - ( - 'tests/resources/parameters/root_parameters_schema.json', - 'tests/resources/parameters/root_parameters_valid.json', - True, - ), - # "embedding_settings" is required at the top level. - # Even though qdrant_settings has all required fields, it doesn't satisfy the top-level schema. - ( - 'tests/resources/parameters/root_parameters_schema.json', - 'tests/resources/parameters/root_parameters_invalid.json', - False, - ), - ], -) -def test_validate_root_parameters(schema_path: str, data_path: str, valid: bool): - with open(schema_path, 'r') as f: - schema = json.load(f) - with open(data_path, 'r') as f: - data = json.load(f) - if valid: - try: - validation._validate_parameters_configuration_against_schema(data, schema) - except jsonschema.ValidationError: - pytest.fail('ValidationError was raised when it should not have been') - else: - with pytest.raises(jsonschema.ValidationError): - validation._validate_parameters_configuration_against_schema(data, schema) - - -@pytest.mark.parametrize( - ('input_storage', 'output_storage'), - [ - ({'input': {}, 'output': {}}, {'input': {}, 'output': {}}), - ({'storage': {'input': {}, 'output': {}}}, {'input': {}, 'output': {}}), - ({}, {}), # we expect passing when no storage is provided - (None, {}), # we expect passing when no storage is provided - ({'storage': None}, {}), # we expect passing when no storage is provided - ], -) -def test_validate_storage_configuration_output( - mock_component: dict, input_storage: Optional[JsonDict], output_storage: Optional[JsonDict] -): - """testing expected storage output for a given storage input""" - component_raw = mock_component.copy() - component_raw['type'] = 'extractor' # we need extractor to pass the validation for storage necessity - api_component = ComponentAPIResponse.model_validate(component_raw) - component = Component.from_api_response(api_component) - result = validation._validate_storage_configuration(input_storage, component) - expected = output_storage # we expect unwrapped structure - assert result == expected - - -@pytest.mark.parametrize( - ('is_writer_row_based', 'storage', 'is_storage_row_based', 'error_message'), - [ - # Non-row-based writer with input storage for root configuration - (False, {'storage': {'input': {'files': []}}}, False, None), - # Non-row-based writer without input storage for root configuration - ( - False, - {}, - False, - 'The "storage" must contain "input" mappings for the root configuration of the writer component', - ), - # Non-row-based writer with input storage for row configuration - (False, {'storage': {'input': {'files': []}}}, True, None), # should not fail, but log warning - # Row-based writer with input storage - (True, {'storage': {'input': {'files': []}}}, True, None), - # Row-based writer without input storage - ( - True, - {}, - True, - 'The "storage" must contain "input" mappings for the row configuration of the writer component', - ), - ], -) -def test_validate_storage_of_row_based_and_root_based_writers( - caplog, - mock_component: dict, - is_writer_row_based: bool, - storage: Optional[JsonDict], - is_storage_row_based: bool, - error_message: Optional[str], -): - """testing storage necessity validation""" - component_raw = mock_component.copy() - component_raw['type'] = 'writer' - component_raw['component_flags'] = ['genericDockerUI-rows'] if is_writer_row_based else [] - - api_component = ComponentAPIResponse.model_validate(component_raw) - component = Component.from_api_response(api_component) - if error_message is None: - if not is_writer_row_based and is_storage_row_based: - with caplog.at_level(logging.WARNING): - validation._validate_storage_configuration( - storage=storage, component=component, is_row_storage=is_storage_row_based - ) - assert 'Validating "storage" for row configuration of non-row-based writer' in caplog.text - else: - validation._validate_storage_configuration( - storage=storage, component=component, is_row_storage=is_storage_row_based - ) - else: - with pytest.raises(ValueError, match=error_message) as exception: - validation._validate_storage_configuration( - storage=storage, component=component, is_row_storage=is_storage_row_based - ) - assert component.component_id in str(exception.value) - - -@pytest.mark.parametrize( - ('storage', 'is_valid'), - [ - ({}, False), - ({'storage': None}, False), - ({'storage': {}}, False), - ({'storage': {'input': {}}}, False), - ({'storage': {'output': {}}}, False), - ({'storage': {'anything-else': {}}}, False), - ({'storage': {'input': {}, 'output': {}}}, False), # empty input or output is not allowed - ({'storage': {'input': {'tables': []}, 'output': {'tables': []}}}, True), - ({'input': {'tables': []}, 'output': {'tables': []}}, True), - ], -) -def test_validate_storage_of_sql_transformation(mock_component: dict, storage: Optional[JsonDict], is_valid: bool): - """testing storage necessity validation""" - component_raw = mock_component.copy() - component_raw['type'] = 'transformation' - # we test the validation for both SQL transformations - for transformation_id in [validation.BIGQUERY_TRANSFORMATION_ID, validation.SNOWFLAKE_TRANSFORMATION_ID]: - component_raw['id'] = transformation_id - api_component = ComponentAPIResponse.model_validate(component_raw) - component = Component.from_api_response(api_component) - if is_valid: - validation._validate_storage_configuration(storage=storage, component=component) - else: - with pytest.raises( - ValueError, - match='The "storage" must contain either "input" or "output" mappings in the configuration of the SQL ', - ) as exception: - validation._validate_storage_configuration(storage=storage, component=component) - assert f'{component.component_id}' in str(exception.value) - assert 'SQL transformation' in str(exception.value) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('input_parameters', 'output_parameters'), - [ - ({'a': 1}, {'a': 1}), - ({'parameters': {'a': 1, 'b': 2}}, {'a': 1, 'b': 2}), - ], -) -async def test_validate_root_parameters_configuration_output( - mock_component: dict, input_parameters: JsonDict, output_parameters: JsonDict -): - """testing returned format structures {...}""" - accepting_schema = {'type': 'object', 'additionalProperties': True} # accepts any json object - component_raw = mock_component.copy() - api_component = ComponentAPIResponse.model_validate(component_raw) - component = Component.from_api_response(api_component) - component.configuration_schema = accepting_schema - result = validation.validate_root_parameters_configuration(input_parameters, component) - expected = output_parameters # we expect unwrapped structure - assert result == expected - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('input_parameters', 'output_parameters'), - [ - ({'a': 1}, {'a': 1}), - ({'parameters': {'a': 1, 'b': 2}}, {'a': 1, 'b': 2}), - ], -) -async def test_validate_row_parameters_configuration_output( - mock_component: dict, input_parameters: JsonDict, output_parameters: JsonDict -): - """testing normalized and returned structures {parameters: {...}} vs {...}""" - accepting_schema = {'type': 'object', 'additionalProperties': True} # accepts any json object - component_raw = mock_component.copy() - api_component = ComponentAPIResponse.model_validate(component_raw) - component = Component.from_api_response(api_component) - component.configuration_row_schema = accepting_schema - result = validation.validate_row_parameters_configuration(input_parameters, component) - expected = output_parameters # we expect unwrapped structure - assert result == expected - - -@pytest.mark.asyncio -@pytest.mark.parametrize('input_schema', [None, {}]) -async def test_validate_parameters_configuration_no_schema(mock_component: dict, input_schema: Optional[JsonDict]): - """We expect passing the validation when no schema is provided""" - input_parameters: JsonDict = {'a': 1} - component_raw = mock_component.copy() - api_component = ComponentAPIResponse.model_validate(component_raw) - component = Component.from_api_response(api_component) - component.configuration_row_schema = input_schema - result = validation.validate_row_parameters_configuration(input_parameters, component) - expected = input_parameters # we expect unwrapped structure - assert result == expected - - -@pytest.mark.parametrize( - ('file_path', 'is_parameter_key_present', 'is_valid'), - [ - ('tests/resources/parameters/root_parameters_invalid.json', True, False), - ('tests/resources/parameters/root_parameters_invalid.json', False, False), - ('tests/resources/parameters/root_parameters_valid.json', True, True), - ('tests/resources/parameters/root_parameters_valid.json', False, True), - ], -) -def test_validate_parameters_root_real_scenario( - mock_component: dict, file_path: str, is_parameter_key_present: bool, is_valid: bool -): - """We test the validation of the root parameters configuration for a real scenario - regardless of the parameters key presence we expect the same output""" - with open(file_path, 'r') as f: - input_parameters = json.load(f) - assert 'parameters' not in input_parameters # we do not expect the parameters key in the input - with open('tests/resources/parameters/root_parameters_schema.json', 'r') as f: - input_schema = json.load(f) - - component_raw = mock_component.copy() - api_component = ComponentAPIResponse.model_validate(component_raw) - component = Component.from_api_response(api_component) - component.configuration_schema = input_schema - modified_input_parameters = {'parameters': input_parameters} if is_parameter_key_present else input_parameters - if is_valid: - ret_params = validation.validate_root_parameters_configuration(modified_input_parameters, component) - assert ret_params == input_parameters - else: - with pytest.raises(validation.RecoverableValidationError) as exception: - validation.validate_root_parameters_configuration(modified_input_parameters, component, 'test oops') - assert 'test oops' in str(exception.value) - - -def test_validate_conditional_flow_with_explicit_schema(): - """A conditional flow validates against an explicitly provided schema.""" - with open('tests/tools/flow/fixtures/conditional_flow_schema.json', 'r') as f: - schema = json.load(f) - valid_flow = { - 'phases': [ - { - 'id': 'p1', - 'name': 'Phase 1', - 'next': [{'id': 't1', 'name': 'End', 'goto': None}], - }, - ], - 'tasks': [ - { - 'id': 'task1', - 'name': 'Notify', - 'phase': 'p1', - 'task': { - 'type': 'notification', - 'title': 'Done', - 'recipients': [{'channel': 'email', 'address': 'ops@example.com'}], - }, - } - ], - } - result = validation.validate_flow_configuration_against_schema( - valid_flow, flow_type=CONDITIONAL_FLOW_COMPONENT_ID, schema=schema - ) - assert result == valid_flow - - -def test_validate_flow_conditional_without_schema_raises(): - """Conditional flow without an explicit schema is a programming error (no bundled fallback).""" - with pytest.raises(ValueError, match='No schema provided for flow type'): - validation.validate_flow_configuration_against_schema( - {'phases': [], 'tasks': []}, flow_type=CONDITIONAL_FLOW_COMPONENT_ID - ) diff --git a/tests/tools/flow/__init__.py b/tests/tools/flow/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/tools/flow/conftest.py b/tests/tools/flow/conftest.py deleted file mode 100644 index 4167f04ad..000000000 --- a/tests/tools/flow/conftest.py +++ /dev/null @@ -1,106 +0,0 @@ -import json -from pathlib import Path -from typing import Any, Dict, List - -import pytest - -from keboola_mcp_server.clients.storage import JsonDict - - -@pytest.fixture -def mock_project_id() -> str: - """Mocks a project id.""" - return '1' - - -@pytest.fixture -def mock_raw_flow_config() -> Dict[str, Any]: - """Mock raw flow configuration as returned by Keboola API.""" - return { - 'id': '21703284', - 'name': 'Test Flow', - 'description': 'Test flow description', - 'version': 1, - 'isDisabled': False, - 'isDeleted': False, - 'configuration': { - 'phases': [ - {'id': 1, 'name': 'Data Extraction', 'description': 'Extract data from sources', 'dependsOn': []}, - {'id': 2, 'name': 'Data Processing', 'description': 'Process extracted data', 'dependsOn': [1]}, - ], - 'tasks': [ - { - 'id': 20001, - 'name': 'Extract AWS S3', - 'phase': 1, - 'enabled': True, - 'continueOnFailure': False, - 'task': {'componentId': 'keboola.ex-aws-s3', 'configId': '12345', 'mode': 'run'}, - }, - { - 'id': 20002, - 'name': 'Process Data', - 'phase': 2, - 'enabled': True, - 'continueOnFailure': False, - 'task': {'componentId': 'keboola.snowflake-transformation', 'configId': '67890', 'mode': 'run'}, - }, - ], - }, - 'changeDescription': 'Initial creation', - 'metadata': [], - 'created': '2025-05-25T06:33:41+0200', - } - - -@pytest.fixture -def mock_empty_flow_config() -> Dict[str, Any]: - """Mock empty flow configuration.""" - return { - 'id': '21703285', - 'name': 'Empty Flow', - 'description': 'Empty test flow', - 'version': 1, - 'isDisabled': False, - 'isDeleted': False, - 'configuration': {'phases': [], 'tasks': []}, - 'changeDescription': None, - 'metadata': [], - 'created': '2025-05-25T07:00:00+0200', - } - - -@pytest.fixture -def sample_phases() -> List[Dict[str, Any]]: - """Sample phase definitions for testing.""" - return [ - {'name': 'Data Extraction', 'dependsOn': [], 'description': 'Extract data'}, - {'name': 'Data Processing', 'dependsOn': [1], 'description': 'Process data'}, - {'name': 'Data Output', 'dependsOn': [2], 'description': 'Output processed data'}, - ] - - -@pytest.fixture -def sample_tasks() -> List[Dict[str, Any]]: - """Sample task definitions for testing.""" - return [ - {'name': 'Extract from S3', 'phase': 1, 'task': {'componentId': 'keboola.ex-aws-s3', 'configId': '12345'}}, - { - 'name': 'Transform Data', - 'phase': 2, - 'task': {'componentId': 'keboola.snowflake-transformation', 'configId': '67890'}, - }, - { - 'name': 'Export to BigQuery', - 'phase': 3, - 'task': {'componentId': 'keboola.wr-google-bigquery-v2', 'configId': '11111'}, - }, - ] - - -@pytest.fixture -def conditional_flow_schema() -> JsonDict: - """Representative offline keboola.flow configuration schema for conditional-flow tests.""" - fixture_path = Path(__file__).parent / 'fixtures' / 'conditional_flow_schema.json' - with fixture_path.open('r', encoding='utf-8') as f: - return json.load(f) diff --git a/tests/tools/flow/fixtures/conditional_flow_schema.json b/tests/tools/flow/fixtures/conditional_flow_schema.json deleted file mode 100644 index 3c0013699..000000000 --- a/tests/tools/flow/fixtures/conditional_flow_schema.json +++ /dev/null @@ -1,480 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "type": "object", - "required": ["phases", "tasks"], - "description": "A Keboola Flow configuration that orchestrates the execution of components. Flows define how tasks are grouped into phases and executed sequentially or in parallel, with conditional transitions between phases.", - "properties": { - "phases": { - "type": "array", - "description": "Array of phases that group tasks and define execution order. Within each phase, tasks execute in this order: notification tasks first (sequentially), then variable tasks (sequentially), finally job tasks (in parallel). Phases themselves run sequentially based on dependencies and conditions. Phases cannot be empty and must have at least one enabled task.", - "items": { - "type": "object", - "required": ["id", "name"], - "description": "A phase groups related tasks and defines transitions to other phases. Each phase must have at least one active task (enabled: true).", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "retry": { - "$ref": "#/definitions/retryConfiguration", - "description": "Retry configuration that will be applied to all job tasks in this phase that don't have their own retry configuration. Only applies to job tasks, not notification or variable tasks." - }, - "next": { - "type": "array", - "description": "Array of conditional transitions to other phases. Conditions are evaluated after all tasks in the phase complete. Always include a default transition (without condition) as the last item to prevent execution errors.", - "items": { - "type": "object", - "required": ["id", "goto"], - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string", - "description": "Optional descriptive name for the transition - useful for debugging and monitoring" - }, - "condition": { - "$ref": "#/definitions/operatorCondition", - "description": "Condition that must be met for this transition to be taken. If omitted, this serves as the default transition." - }, - "goto": { - "type": ["string", "null"], - "description": "Target phase ID to transition to, or null to end the flow. When using conditional transitions, always include a default transition (without a condition) as the last item to prevent flow execution errors." - } - } - } - }, - "description": { - "type": "string" - } - } - } - }, - "tasks": { - "type": "array", - "description": "Array of tasks that perform the actual work. Tasks are executed within phases in this order: notification tasks first, then variable tasks, finally job tasks.", - "items": { - "type": "object", - "required": ["id", "name", "task", "phase"], - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "phase": { - "type": "string", - "description": "ID of the phase this task belongs to. Must reference an existing phase ID." - }, - "task": { - "type": "object", - "oneOf": [ - { - "required": ["type", "componentId", "mode"], - "anyOf": [ - { "required": ["configId"] }, - { "required": ["configData"] } - ], - "description": "Job task that executes a Keboola component configuration", - "properties": { - "type": { - "type": "string", - "enum": ["job"] - }, - "componentId": { - "type": "string", - "description": "ID of the Keboola component to execute (e.g., 'keboola.ex-db-snowflake')" - }, - "configId": { - "type": "string", - "description": "ID of the component configuration to run" - }, - "configData": { - "type": "object", - "description": "Inline component configuration parameters" - }, - "mode": { - "type": "string", - "enum": ["run"] - }, - "delay": { - "type": ["string", "number"], - "description": "Initial delay in seconds before starting the job" - }, - "retry": { - "$ref": "#/definitions/retryConfiguration", - "description": "Retry configuration specific to this job task. Takes precedence over phase-level retry configuration." - } - } - }, - { - "required": ["type", "recipients", "title"], - "description": "Notification task that sends messages via email or webhook", - "properties": { - "type": { - "type": "string", - "enum": ["notification"] - }, - "recipients": { - "type": "array", - "description": "List of notification recipients. Can mix email and webhook recipients.", - "items": { - "type": "object", - "required": ["channel", "address"], - "properties": { - "channel": { - "type": "string", - "enum": ["email", "webhook"], - "description": "Delivery channel: 'email' for email notifications, 'webhook' for HTTP POST notifications" - }, - "address": { - "type": "string", - "description": "Recipient address: email address for email channel, HTTP/HTTPS URL for webhook channel" - } - }, - "allOf": [ - { - "if": { - "properties": { - "channel": { - "const": "email" - } - } - }, - "then": { - "properties": { - "address": { - "format": "email" - } - } - } - }, - { - "if": { - "properties": { - "channel": { - "const": "webhook" - } - } - }, - "then": { - "properties": { - "address": { - "format": "uri" - } - } - } - } - ] - }, - "minItems": 1 - }, - "title": { - "type": "string" - }, - "message": { - "type": "string", - "description": "Optional message body. For email: defaults to empty string if not provided. For webhook: defaults to null if not provided." - } - } - }, - { - "required": ["type", "name"], - "description": "Variable task that defines a variable for use in conditions or job parameters", - "oneOf": [ - { - "required": ["value"], - "description": "Static variable with a fixed value", - "properties": { - "type": { - "type": "string", - "enum": ["variable"] - }, - "name": { - "type": "string", - "description": "Name of the variable. Will be accessible in conditions and passed to all job tasks via variableValuesData." - }, - "value": { - "type": "string" - } - } - }, - { - "required": ["source"], - "description": "Dynamic variable computed from other task/phase results", - "properties": { - "type": { - "type": "string", - "enum": ["variable"] - }, - "name": { - "type": "string", - "description": "Name of the variable. Will be accessible in conditions and passed to all job tasks via variableValuesData." - }, - "source": { - "$ref": "#/definitions/variableSourceObject", - "description": "Source definition for computing the variable value dynamically from task results, phase status, constants, or functions" - } - } - } - ] - } - ] - }, - "enabled": { - "type": "boolean", - "default": true, - "description": "Whether this task is enabled. Disabled tasks are skipped during execution. Defaults to true if not specified." - } - } - } - } - }, - "definitions": { - "conditionObject": { - "type": "object", - "oneOf": [ - { "$ref": "#/definitions/constantCondition" }, - { "$ref": "#/definitions/phaseCondition" }, - { "$ref": "#/definitions/taskCondition" }, - { "$ref": "#/definitions/variableCondition" }, - { "$ref": "#/definitions/operatorCondition" }, - { "$ref": "#/definitions/functionCondition" }, - { "$ref": "#/definitions/arrayCondition" } - ] - }, - "constantCondition": { - "type": "object", - "required": ["type", "value"], - "properties": { - "type": { - "type": "string", - "enum": ["const", "constant"] - }, - "value": { - "type": ["string", "number", "boolean", "array"] - } - } - }, - "phaseCondition": { - "type": "object", - "required": ["type", "phase", "value"], - "properties": { - "type": { - "type": "string", - "enum": ["phase"], - "description": "Returns phase-level metadata from a phase that has already completed." - }, - "phase": { - "type": "string", - "description": "ID of the phase to evaluate. The referenced phase must have already completed execution." - }, - "value": { - "type": "string", - "enum": ["phaseId", "status"], - "description": "Property to retrieve from the phase. 'phaseId' returns the phase ID, 'status' returns the phase execution status (success, user_error, application_error, terminated)." - } - } - }, - "taskCondition": { - "type": "object", - "required": ["type", "task", "value"], - "properties": { - "type": { - "type": "string", - "enum": ["task"], - "description": "Returns task-level metadata or associated job metadata from a task that has already completed." - }, - "task": { - "type": "string", - "description": "ID of the task to evaluate. The referenced task must have already completed execution. Set '*' when used with phase operators (ALL_TASKS_IN_PHASE, ANY_TASKS_IN_PHASE)" - }, - "value": { - "type": "string", - "enum": [ - "taskId", - "phaseId", - "status", - "job.id", - "job.componentId", - "job.configId", - "job.status", - "job.result", - "job.startTime", - "job.endTime", - "job.duration", - "job.result.output.tables", - "job.result.message" - ], - "description": "Property path to retrieve from the task context. Basic properties: 'taskId', 'phaseId', 'status' (success, user_error, application_error, terminated). Job properties available for job tasks: 'job.id', 'job.componentId', 'job.configId', 'job.status', 'job.result', 'job.startTime', 'job.endTime', 'job.duration' (in seconds), 'job.result.output.tables' (array), 'job.result.message' (string)." - } - } - }, - "variableCondition": { - "type": "object", - "required": ["type", "value"], - "properties": { - "type": { - "type": "string", - "enum": ["variable"] - }, - "value": { - "type": "string", - "description": "The name of the variable to evaluate. Must reference a variable defined earlier in the flow." - } - } - }, - "operatorCondition": { - "type": "object", - "required": ["type", "operator"], - "description": "A condition that applies logical or relational operators to other conditions and returns boolean result (true/false).", - "oneOf": [ - { - "required": ["operator", "operands"], - "properties": { - "type": { - "type": "string", - "enum": ["operator"] - }, - "operator": { - "type": "string", - "enum": ["AND", "OR", "EQUALS", "NOT_EQUALS", "GREATER_THAN", "LESS_THAN", "INCLUDES", "CONTAINS"], - "description": "Operator type: 'AND'/'OR' (logical, requires 1+ operands), 'EQUALS'/'NOT_EQUALS' (equality, requires 2 operands), 'GREATER_THAN'/'LESS_THAN' (comparison, requires 2 operands), 'INCLUDES' (checks if first operand is included in second operand array, requires 2 operands), 'CONTAINS' (checks if first operand string contains second operand string, case-insensitive, requires 2 operands)" - }, - "operands": { - "type": "array", - "description": "Array of conditions to apply the operator to. Number of required operands depends on operator type.", - "items": { - "$ref": "#/definitions/conditionObject" - } - } - } - }, - { - "required": ["operator", "phase", "operands"], - "description": "Phase-level operators that apply conditions to all or any tasks within a specific phase", - "properties": { - "type": { - "type": "string", - "enum": ["operator"] - }, - "operator": { - "type": "string", - "enum": ["ALL_TASKS_IN_PHASE", "ANY_TASKS_IN_PHASE"], - "description": "Phase operator type: 'ALL_TASKS_IN_PHASE' (condition must be true for all tasks in phase, requires 1 operand), 'ANY_TASKS_IN_PHASE' (condition must be true for at least one task in phase, requires 1 operand)" - }, - "phase": { - "type": "string", - "description": "ID of the phase to apply the condition to. Must reference an existing phase in the flow." - }, - "operands": { - "type": "array", - "description": "Array containing the condition to apply to each task in the phase. Use '*' as the task ID in task conditions.", - "items": { - "$ref": "#/definitions/operatorCondition" - } - } - } - } - ] - }, - "functionCondition": { - "type": "object", - "required": ["type", "function", "operands"], - "properties": { - "type": { - "type": "string", - "enum": ["function"] - }, - "function": { - "type": "string", - "description": "Function name. Supported values: 'COUNT' (returns the number of elements in its single array operand) and 'DATE' (returns the current date/time formatted via PHP DateTime::format; takes one operand that resolves to a format string such as 'Y', 'm', 'd', 'H', 'i', 's', or 'U'). The enum is intentionally omitted so unknown server-side values do not break flow listing; only the two values above are accepted at execution time." - }, - "operands": { - "type": "array", - "description": "Array of conditions that provide inputs to the function. Number of required operands depends on function type.", - "items": { - "$ref": "#/definitions/variableSourceObject" - } - } - } - }, - "variableSourceObject": { - "type": "object", - "description": "Source definition for computing variable values dynamically. Limited subset of condition types that can be used as variable sources (excludes logical operators).", - "oneOf": [ - { "$ref": "#/definitions/constantCondition" }, - { "$ref": "#/definitions/phaseCondition" }, - { "$ref": "#/definitions/taskCondition" }, - { "$ref": "#/definitions/variableCondition" }, - { "$ref": "#/definitions/functionCondition" }, - { "$ref": "#/definitions/arrayCondition" } - ] - }, - "arrayCondition": { - "type": "object", - "required": ["type", "operands"], - "description": "A condition that creates an array from multiple operands. Used primarily with the INCLUDES operator.", - "properties": { - "type": { - "type": "string", - "enum": ["array"] - }, - "operands": { - "type": "array", - "items": { - "$ref": "#/definitions/variableSourceObject" - } - } - } - }, - "retryConfiguration": { - "type": "object", - "description": "Configuration for automatic retry of failed job tasks. Can be set at phase level (applies to all job tasks in phase) or task level (overrides phase configuration).", - "properties": { - "retryOn": { - "type": "array", - "description": "Array of conditions that trigger a retry. Multiple conditions work as OR logic - at least one condition must be met to trigger a retry. If empty or missing, jobs will retry by default for any error.", - "items": { - "type": "object", - "required": ["type", "value"], - "properties": { - "type": { - "type": "string", - "enum": ["errorMessageContains", "errorMessageExact"], - "description": "Retry condition type: 'errorMessageContains' (retry if error message contains the value, case-insensitive), 'errorMessageExact' (retry if error message exactly matches the value)" - }, - "value": { - "type": "string" - } - } - } - }, - "strategy": { - "type": "string", - "enum": ["linear"], - "default": "linear" - }, - "strategyParams": { - "type": "object", - "properties": { - "maxRetries": { - "type": "integer", - "default": 3, - "description": "Maximum number of retry attempts (default: 3). This is the number of retries after the initial attempt. Set to 0 to disable retry while preserving retry configuration." - }, - "delay": { - "type": "integer", - "default": 10, - "description": "Delay in seconds between retry attempts" - } - } - } - } - } - } -} diff --git a/tests/tools/flow/test_model.py b/tests/tools/flow/test_model.py deleted file mode 100644 index 4a72da81e..000000000 --- a/tests/tools/flow/test_model.py +++ /dev/null @@ -1,282 +0,0 @@ -from typing import Any - -import pytest -from pydantic import ValidationError - -from keboola_mcp_server.clients.client import CONDITIONAL_FLOW_COMPONENT_ID, ORCHESTRATOR_COMPONENT_ID -from keboola_mcp_server.clients.storage import APIFlowResponse -from keboola_mcp_server.tools.flow.model import ( - ConditionalFlowConfiguration, - ConditionalFlowPhase, - ConditionalFlowTask, - ConditionalFlowTransition, - Flow, - FlowConfiguration, - FlowPhase, - FlowSummary, - FlowTask, - FunctionCondition, - JobTaskConfiguration, - VariableTaskConfiguration, -) - -# --- Test Model Parsing --- - - -class TestFlowModels: - """Test Flow models.""" - - def test_flow_from_api_response(self, mock_raw_flow_config: dict[str, Any]): - """Test Flow.from_api_response from a typical raw API response.""" - assert 'component_id' not in mock_raw_flow_config - api_model = APIFlowResponse.model_validate(mock_raw_flow_config) - flow = Flow.from_api_response(api_config=api_model, flow_component_id=ORCHESTRATOR_COMPONENT_ID) - assert flow.component_id == ORCHESTRATOR_COMPONENT_ID - assert flow.configuration_id == '21703284' - assert flow.name == 'Test Flow' - assert flow.description == 'Test flow description' - assert flow.version == 1 - assert flow.is_disabled is False - assert flow.is_deleted is False - config = flow.configuration - assert isinstance(config, FlowConfiguration) - assert len(config.phases) == 2 - assert len(config.tasks) == 2 - # Check phase and task structure - phase1 = config.phases[0] - assert isinstance(phase1, FlowPhase) - assert phase1.id == 1 - assert phase1.name == 'Data Extraction' - assert phase1.depends_on == [] - phase2 = config.phases[1] - assert phase2.id == 2 - assert phase2.depends_on == [1] - task1 = config.tasks[0] - assert isinstance(task1, FlowTask) - assert task1.id == 20001 - assert task1.name == 'Extract AWS S3' - assert task1.phase == 1 - assert task1.task['componentId'] == 'keboola.ex-aws-s3' - - def test_flow_summary_from_api_response(self, mock_raw_flow_config: dict[str, Any]): - """Test FlowSummary.from_api_response from a typical raw API response.""" - assert 'tasks_count' not in mock_raw_flow_config - assert 'phases_count' not in mock_raw_flow_config - api_model = APIFlowResponse.model_validate(mock_raw_flow_config) - flow_summary = FlowSummary.from_api_response(api_config=api_model, flow_component_id=ORCHESTRATOR_COMPONENT_ID) - assert flow_summary.component_id == ORCHESTRATOR_COMPONENT_ID - assert flow_summary.configuration_id == '21703284' - assert flow_summary.name == 'Test Flow' - assert flow_summary.description == 'Test flow description' - assert flow_summary.version == 1 - assert flow_summary.phases_count == 2 - assert flow_summary.tasks_count == 2 - assert flow_summary.is_disabled is False - assert flow_summary.is_deleted is False - - def test_empty_flow_from_api_response(self, mock_empty_flow_config: dict[str, Any]): - """Test Flow and FlowSummary from_api_response with an empty flow configuration.""" - assert 'component_id' not in mock_empty_flow_config - assert 'tasks_count' not in mock_empty_flow_config - assert 'phases_count' not in mock_empty_flow_config - api_model = APIFlowResponse.model_validate(mock_empty_flow_config) - flow = Flow.from_api_response(api_config=api_model, flow_component_id=ORCHESTRATOR_COMPONENT_ID) - flow_summary = FlowSummary.from_api_response(api_config=api_model, flow_component_id=ORCHESTRATOR_COMPONENT_ID) - assert len(flow.configuration.phases) == 0 - assert len(flow.configuration.tasks) == 0 - assert flow_summary.phases_count == 0 - assert flow_summary.tasks_count == 0 - - -class TestConditionalFlowPhase: - """Tests for conditional flow phase serialization helpers.""" - - def test_next_defaults_to_empty_list(self): - """Ensure default next is an empty list and serialized when requested.""" - phase = ConditionalFlowPhase(id='phase-1', name='Phase 1') - - assert phase.next == [] - - serialized = phase.model_dump() - assert 'next' in serialized - assert serialized['next'] == [] - - def test_model_dump_exclude_unset_omits_empty_next(self): - """When exclude_unset=True, empty next should be removed from payload.""" - phase = ConditionalFlowPhase(id='phase-1', name='Phase 1') - - serialized = phase.model_dump(exclude_unset=True) - - assert 'next' not in serialized - - def test_model_dump_keeps_non_empty_next(self): - """Non-empty next array should always be serialized.""" - transition = ConditionalFlowTransition(id='t1', name='Go to phase 2', goto='phase-2') - phase = ConditionalFlowPhase(id='phase-1', name='Phase 1', next=[transition]) - - serialized_default = phase.model_dump() - assert serialized_default['next'][0]['id'] == 't1' - - serialized_excluding_unset = phase.model_dump(exclude_unset=True) - assert serialized_excluding_unset['next'][0]['goto'] == 'phase-2' - - def test_model_dump_excludes_single_null_goto_transition(self): - """Single transition with goto=None should be excluded when exclude_unset=True to prevent UI damage.""" - transition = ConditionalFlowTransition(id='t1', name='Go to end', goto=None) - phase = ConditionalFlowPhase(id='phase-1', name='Phase 1', next=[transition]) - - # Without exclude_unset, the next array should be serialized - serialized_default = phase.model_dump() - assert 'next' in serialized_default - assert serialized_default['next'][0]['id'] == 't1' - assert serialized_default['next'][0]['goto'] is None - - # With exclude_unset=True, the next array should be excluded - serialized_excluding_unset = phase.model_dump(exclude_unset=True) - assert 'next' not in serialized_excluding_unset - - def test_model_dump_keeps_multiple_transitions_with_null_goto(self): - """Multiple transitions should be kept even if one has goto=None.""" - transition1 = ConditionalFlowTransition(id='t1', name='Go to phase 2', goto='phase-2') - transition2 = ConditionalFlowTransition(id='t2', name='Go to end', goto=None) - phase = ConditionalFlowPhase(id='phase-1', name='Phase 1', next=[transition1, transition2]) - - serialized_default = phase.model_dump() - assert 'next' in serialized_default - assert len(serialized_default['next']) == 2 - - serialized_excluding_unset = phase.model_dump(exclude_unset=True) - assert 'next' in serialized_excluding_unset - assert len(serialized_excluding_unset['next']) == 2 - assert serialized_excluding_unset['next'][0]['goto'] == 'phase-2' - assert serialized_excluding_unset['next'][1]['goto'] is None - - -class TestConditionalFlowValidationResilience: - """Regression tests for AI-3216 — `get_flows` should not crash on unknown variants. - - The bug: a real conditional flow on stack `com-keboola-gcp-europe-west3` contained a - `variable` task whose `source.function` was `'YEAR'`. The strict `Literal['COUNT','DATE']` - on `FunctionCondition.function` failed, and the undiscriminated `TaskConfiguration` union - surfaced 18 cascading validation errors that aborted the entire `get_flows` response. - """ - - @pytest.mark.parametrize('function_name', ['COUNT', 'DATE', 'YEAR', 'MONTH', 'DAY_OF_WEEK']) - def test_function_condition_accepts_arbitrary_function_names(self, function_name: str): - """The `function` field is now permissive — the backend evolves independently of MCP.""" - cond = FunctionCondition.model_validate( - {'type': 'function', 'function': function_name, 'operands': [{'type': 'const', 'value': 'U'}]} - ) - assert cond.function == function_name - - def test_variable_task_with_year_function_no_longer_raises(self): - """The exact shape from the Datadog alert in AI-3216 must parse without raising.""" - raw_task = { - 'id': 'task-1', - 'name': 'compute year', - 'phase': 'phase-1', - 'task': { - 'type': 'variable', - 'name': 'current_year', - 'source': { - 'type': 'function', - 'function': 'YEAR', - 'operands': [{'type': 'const', 'value': 'U'}], - }, - }, - } - task = ConditionalFlowTask.model_validate(raw_task) - assert isinstance(task.task, VariableTaskConfiguration) - assert isinstance(task.task.source, FunctionCondition) - assert task.task.source.function == 'YEAR' - - def test_task_configuration_uses_discriminator(self): - """Discriminator on `type` collapses the 18-error cascade to a single targeted error. - - Pre-fix: pydantic tried Job/Notification/Variable in turn and reported every literal - mismatch — 18 errors for one bad source value. Post-fix: pydantic dispatches by `type`, - so a `variable` task only triggers `VariableTaskConfiguration` validation. - """ - with pytest.raises(ValidationError) as exc_info: - ConditionalFlowTask.model_validate( - { - 'id': 'task-1', - 'name': 'broken', - 'phase': 'phase-1', - 'task': {'type': 'variable'}, # missing required `name` - } - ) - errors = exc_info.value.errors() - # All errors should be scoped to the matched variant only. - assert all('VariableTaskConfiguration' in str(e.get('loc', ())) or e['loc'][-1] == 'name' for e in errors) - # Confirm no cascading errors about Job/Notification literals. - assert not any('JobTaskConfiguration' in str(e.get('loc', ())) for e in errors) - assert not any('NotificationTaskConfiguration' in str(e.get('loc', ())) for e in errors) - - def test_unknown_task_type_falls_back_in_read_path(self, caplog: pytest.LogCaptureFixture): - """`Flow.from_api_response` must keep returning the flow even when a task type is unknown. - - Pre-fix: one unknown task type took down the entire `get_flows` response. Post-fix: - `_safe_validate` logs and falls back to `model_construct` for that task while keeping - the rest of the flow intact. - """ - raw = { - 'id': '99', - 'name': 'Flow with unknown task variant', - 'description': '', - 'version': 1, - 'isDisabled': False, - 'isDeleted': False, - 'configuration': { - 'phases': [{'id': 'p1', 'name': 'Phase 1', 'next': [{'id': 't1', 'goto': None}]}], - 'tasks': [ - { - 'id': 'task-good', - 'name': 'good', - 'phase': 'p1', - 'task': {'type': 'job', 'componentId': 'keboola.ex-aws-s3', 'mode': 'run'}, - }, - { - 'id': 'task-bad', - 'name': 'bad', - 'phase': 'p1', - 'task': {'type': 'unknown-future-variant', 'foo': 'bar'}, - }, - ], - }, - 'metadata': [], - 'created': '2026-01-01T00:00:00+0000', - } - api_model = APIFlowResponse.model_validate(raw) - with caplog.at_level('WARNING'): - flow = Flow.from_api_response(api_config=api_model, flow_component_id=CONDITIONAL_FLOW_COMPONENT_ID) - - assert isinstance(flow.configuration, ConditionalFlowConfiguration) - assert len(flow.configuration.tasks) == 2 - good, bad = flow.configuration.tasks - assert isinstance(good.task, JobTaskConfiguration) - # Bad task survives as raw passthrough; agent still sees the entry instead of nothing. - assert bad.id == 'task-bad' - assert any('failed strict validation' in m for m in caplog.messages) - - def test_unknown_task_type_still_strict_in_write_path(self): - """Write paths (`utils.get_flow_configuration`) must remain strict to reject agent garbage. - - The fallback is intentionally scoped to `Flow.from_api_response` (the READ path). Agents - constructing flows should still get loud failures so they can correct themselves. - """ - from keboola_mcp_server.tools.flow.utils import get_flow_configuration - - with pytest.raises(ValidationError): - get_flow_configuration( - phases=[{'id': 'p1', 'name': 'Phase 1', 'next': [{'id': 't1', 'goto': None}]}], - tasks=[ - { - 'id': 'task-bad', - 'name': 'bad', - 'phase': 'p1', - 'task': {'type': 'unknown-future-variant'}, - } - ], - flow_type=CONDITIONAL_FLOW_COMPONENT_ID, - ) diff --git a/tests/tools/flow/test_scheduler.py b/tests/tools/flow/test_scheduler.py deleted file mode 100644 index 0e2e3c73d..000000000 --- a/tests/tools/flow/test_scheduler.py +++ /dev/null @@ -1,116 +0,0 @@ -import re - -import pytest - -from keboola_mcp_server.tools.flow.scheduler import validate_cron_tab - - -class TestValidateCronTab: - """Test validate_cron_tab function.""" - - @pytest.mark.parametrize( - 'cron_tab', - [ - pytest.param(None, id='none_value'), - pytest.param('0 1,13 * * *', id='daily_at_1am_and_1pm'), - pytest.param('0 9 * * 1', id='weekly_monday_9am'), - pytest.param('0 10 1,20 * *', id='monthly_1st_and_20th_10am'), - pytest.param('0 11 1 1,8 *', id='yearly_jan_aug_1st_11am'), - pytest.param('0,15,30,45 * * * *', id='hourly_every_15_minutes'), - pytest.param('30 14 * * *', id='daily_2_30pm'), - pytest.param('0 0 * * *', id='midnight_daily'), - pytest.param('59 23 * * *', id='max_values'), - # L (last day of month) support - pytest.param('0 10 L * *', id='last_day_of_month_10am'), - pytest.param('0 0 L * *', id='last_day_of_month_midnight'), - pytest.param('0 10 L 1,6 *', id='last_day_jan_and_june'), - pytest.param('0 10 l * *', id='last_day_lowercase'), - ], - ) - def test_valid_cron_tab(self, cron_tab: str | None): - """Test valid cron tab expressions.""" - result = validate_cron_tab(cron_tab) - if cron_tab is None: - assert result is None - else: - assert result is None # Function returns None on success - - @pytest.mark.parametrize( - ('cron_tab', 'error_match'), - [ - pytest.param('', 'Cron expression must have exactly 5 parts', id='empty_string'), - pytest.param('0 8 * *', 'Cron expression must have exactly 5 parts', id='too_few_parts'), - pytest.param('0 8 * * * *', 'Cron expression must have exactly 5 parts', id='too_many_parts'), - pytest.param('0 8 * *', 'Cron expression must have exactly 5 parts', id='four_parts'), - pytest.param('0 8 * * * * *', 'Cron expression must have exactly 5 parts', id='seven_parts'), - pytest.param('60 8 * * *', 'Minutes of hour.*must be between 0 and 59', id='minutes_too_high'), - pytest.param('-1 8 * * *', 'Minutes of hour.*must be between 0 and 59', id='minutes_negative'), - pytest.param('abc 8 * * *', 'Cron expression must have only digits', id='minutes_non_digit'), - pytest.param('0 24 * * *', 'Hours of day.*must be between 0 and 23', id='hours_too_high'), - pytest.param('0 -1 * * *', 'Hours of day.*must be between 0 and 23', id='hours_negative'), - pytest.param('0 abc * * *', 'Cron expression must have only digits', id='hours_non_digit'), - pytest.param('0 8 0 * *', 'Days of month.*must be between 1 and 31', id='days_zero'), - pytest.param('0 8 32 * *', 'Days of month.*must be between 1 and 31', id='days_too_high'), - pytest.param('0 8 -1 * *', 'Days of month.*must be between 1 and 31', id='days_negative'), - pytest.param('0 8 abc * *', 'Cron expression must have only digits', id='days_non_digit'), - pytest.param('0 8 * 0 *', 'Months of year.*must be between 1 and 12', id='months_zero'), - pytest.param('0 8 * 13 *', 'Months of year.*must be between 1 and 12', id='months_too_high'), - pytest.param('0 8 * -1 *', 'Months of year.*must be between 1 and 12', id='months_negative'), - pytest.param('0 8 * abc *', 'Cron expression must have only digits', id='months_non_digit'), - pytest.param('0 8 * * 7', 'Days of week.*must be between 0=Sunday and 6=Saturday', id='weekdays_too_high'), - pytest.param('0 8 * * -1', 'Days of week.*must be between 0=Sunday and 6=Saturday', id='weekdays_negative'), - pytest.param('0 8 * * abc', 'Cron expression must have only digits', id='weekdays_non_digit'), - pytest.param('* 1,3 * *', 'Cron expression must have exactly 5 parts', id='missing_weekday'), - pytest.param( - '0 8 * 1,3 *', 'Months of year must be specified with days of month', id='months_without_days' - ), - pytest.param('0 * 1,3 * *', 'Days of month must be specified with hours of day', id='days_without_hours'), - pytest.param( - '* 8 * * *', 'Hours of day must be specified with minutes of hour', id='hours_without_minutes' - ), - pytest.param('* * * * 0', 'Days of week must be specified with hours of day', id='weekdays_without_hours'), - pytest.param('0 8 1 * 0', 'Days of week must not be specified with days of month', id='weekdays_with_days'), - pytest.param( - '0 8 1,3 1,3 0', 'Days of week must not be specified with days of month', id='weekdays_with_both' - ), - # L not allowed in fields other than day-of-month - pytest.param('L 8 * * *', 'Cron expression must have only digits', id='L_in_minutes'), - pytest.param('0 L * * *', 'Cron expression must have only digits', id='L_in_hours'), - pytest.param('0 8 * L *', 'Cron expression must have only digits', id='L_in_months'), - pytest.param('0 8 * * L', 'Cron expression must have only digits', id='L_in_weekdays'), - # L with weekdays not allowed - pytest.param('0 8 L * 0', 'Days of week must not be specified with days of month', id='L_with_weekdays'), - pytest.param( - '0 10 1,L * *', 'Day of month must use either `L` or numeric values, not both', id='L_with_days' - ), - ], - ) - def test_invalid_cron_tab(self, cron_tab: str, error_match: str): - """Test invalid cron tab expressions.""" - with pytest.raises(ValueError, match='Invalid cron tab expression') as exc_info: - validate_cron_tab(cron_tab) - error_message = str(exc_info.value) - # Check that the error message starts with "Invalid cron tab expression: " - assert error_message.startswith('Invalid cron tab expression: ') - # Check that the error message contains the specific error (using regex if needed) - assert re.search(error_match, error_message) is not None, f'Error message does not contain: {error_match}' - # Check that the error message includes the instructions - assert 'Cron Tab Expression should be in the format: `* * * * *`' in error_message - assert 'Field order:' in error_message - assert '1. Minute (0-59)' in error_message - assert '2. Hour (0-23)' in error_message - assert '3. Day of month (1-31, or L for last day of month)' in error_message - assert '4. Month (1-12)' in error_message - assert '5. Day of week (0-6, where 0 = Sunday)' in error_message - - def test_cron_tab_with_whitespace(self): - """Test that cron tab handles whitespace correctly.""" - # Should work with extra whitespace - validate_cron_tab(' 0 8 * * * ') - validate_cron_tab('0 8 * * *') - # Should fail with wrong number of parts after stripping - with pytest.raises(ValueError, match='Invalid cron tab expression') as exc_info: - validate_cron_tab(' 0 8 * * ') - error_message = str(exc_info.value) - assert 'Cron expression must have exactly 5 parts' in error_message - assert 'Cron Tab Expression should be in the format: `* * * * *`' in error_message diff --git a/tests/tools/flow/test_scheduler_model.py b/tests/tools/flow/test_scheduler_model.py deleted file mode 100644 index 74ba51102..000000000 --- a/tests/tools/flow/test_scheduler_model.py +++ /dev/null @@ -1,183 +0,0 @@ -from datetime import datetime - -import pytest - -from keboola_mcp_server.clients.scheduler import Schedule, ScheduleApiResponse, TargetConfiguration, TargetExecution -from keboola_mcp_server.tools.flow.scheduler_model import ScheduleDetail - - -class TestScheduleDetail: - """Test ScheduleDetail model and from_api_response method.""" - - def test_from_api_response_basic(self): - """Test ScheduleDetail.from_api_response with basic schedule data.""" - schedule = Schedule.model_construct(cron_tab='0 8 * * 1-5', timezone='UTC', state='enabled') - target = TargetConfiguration.model_construct( - component_id='keboola.flow', configuration_id='12345', mode='run', tag=None - ) - schedule_api = ScheduleApiResponse.model_construct( - id='100', - token_id='token123', - configuration_id='config123', - configuration_version_id='version1', - schedule=schedule, - target=target, - executions=[], - ) - - schedule_detail = ScheduleDetail.from_api_response(schedule_api) - - assert schedule_detail.schedule_id == 'config123' - assert schedule_detail.timezone == 'UTC' - assert schedule_detail.state == 'enabled' - assert schedule_detail.cron_tab == '0 8 * * 1-5' - assert schedule_detail.target_executions == [] - - def test_from_api_response_with_executions(self): - """Test ScheduleDetail.from_api_response with target executions.""" - schedule = Schedule.model_construct(cron_tab='15,45 1,13 * * 0', timezone='America/New_York', state='enabled') - target = TargetConfiguration.model_construct( - component_id='keboola.orchestrator', configuration_id='67890', mode='run', tag='v1.0' - ) - execution1 = TargetExecution.model_construct(job_id='job123', execution_time=datetime(2024, 1, 15, 10, 30, 0)) - execution2 = TargetExecution.model_construct(job_id='job456', execution_time=datetime(2024, 1, 16, 10, 30, 0)) - schedule_api = ScheduleApiResponse.model_construct( - id='200', - token_id='token456', - configuration_id='config456', - configuration_version_id='version2', - schedule=schedule, - target=target, - executions=[execution1, execution2], - ) - - schedule_detail = ScheduleDetail.from_api_response(schedule_api) - - assert schedule_detail.schedule_id == 'config456' - assert schedule_detail.timezone == 'America/New_York' - assert schedule_detail.state == 'enabled' - assert schedule_detail.cron_tab == '15,45 1,13 * * 0' - assert len(schedule_detail.target_executions) == 2 - assert schedule_detail.target_executions[0].job_id == 'job123' - assert schedule_detail.target_executions[1].job_id == 'job456' - - def test_from_api_response_disabled_schedule(self): - """Test ScheduleDetail.from_api_response with disabled schedule.""" - schedule = Schedule.model_construct(cron_tab='0 0 * * *', timezone='Europe/Prague', state='disabled') - target = TargetConfiguration.model_construct( - component_id='keboola.flow', configuration_id='99999', mode='run', tag=None - ) - schedule_api = ScheduleApiResponse.model_construct( - id='300', - token_id='token789', - configuration_id='config789', - configuration_version_id='version3', - schedule=schedule, - target=target, - executions=[], - ) - - schedule_detail = ScheduleDetail.from_api_response(schedule_api) - - assert schedule_detail.schedule_id == 'config789' - assert schedule_detail.timezone == 'Europe/Prague' - assert schedule_detail.state == 'disabled' - assert schedule_detail.cron_tab == '0 0 * * *' - assert schedule_detail.target_executions == [] - - def test_from_api_response_empty_executions(self): - """Test ScheduleDetail.from_api_response with empty executions list.""" - schedule = Schedule.model_construct(cron_tab='*/30 * * * *', timezone='Asia/Tokyo', state='enabled') - target = TargetConfiguration.model_construct( - component_id='keboola.orchestrator', configuration_id='11111', mode='run', tag=None - ) - schedule_api = ScheduleApiResponse.model_construct( - id='400', - token_id='token000', - configuration_id='config000', - configuration_version_id='version4', - schedule=schedule, - target=target, - executions=[], - ) - - schedule_detail = ScheduleDetail.from_api_response(schedule_api) - - assert schedule_detail.schedule_id == 'config000' - assert schedule_detail.timezone == 'Asia/Tokyo' - assert schedule_detail.state == 'enabled' - assert schedule_detail.cron_tab == '*/30 * * * *' - assert isinstance(schedule_detail.target_executions, list) - assert len(schedule_detail.target_executions) == 0 - - -class TestTargetExecution: - """Test TargetExecution model validation with missing or nullable fields.""" - - @pytest.mark.parametrize( - ('raw_execution', 'expected_job_id', 'expected_execution_time'), - [ - pytest.param( - {'jobId': '38562456', 'executionTime': '2026-02-11T10:10:07+00:00'}, - '38562456', - datetime.fromisoformat('2026-02-11T10:10:07+00:00'), - id='all_fields_present', - ), - pytest.param( - {}, - None, - None, - id='all_fields_missing', - ), - pytest.param( - {'jobId': '38562456'}, - '38562456', - None, - id='execution_time_missing', - ), - pytest.param( - {'executionTime': '2026-02-11T10:10:07+00:00'}, - None, - datetime.fromisoformat('2026-02-11T10:10:07+00:00'), - id='job_id_missing', - ), - pytest.param( - {'job-id': '38562456', 'execution-time': '2026-02-11T10:10:07+00:00'}, - '38562456', - datetime.fromisoformat('2026-02-11T10:10:07+00:00'), - id='kebab_case_keys', - ), - ], - ) - def test_target_execution_nullable_fields( - self, - raw_execution: dict, - expected_job_id: str | None, - expected_execution_time: datetime | None, - ): - """TargetExecution should not raise when API response is missing jobId or executionTime.""" - execution = TargetExecution.model_validate(raw_execution) - assert execution.job_id == expected_job_id - assert execution.execution_time == expected_execution_time - - def test_schedule_api_response_with_incomplete_executions(self): - """ScheduleApiResponse should not raise when executions have missing fields.""" - raw_response = { - 'id': '123', - 'tokenId': 'token-abc', - 'configurationId': 'config-456', - 'configurationVersionId': '1', - 'schedule': {'cronTab': '10 10 * * 2,3,4', 'timezone': 'UTC', 'state': 'enabled'}, - 'target': {'componentId': 'keboola.flow', 'configurationId': 'config-456', 'mode': 'run'}, - 'executions': [ - {'job_id': '38562456', 'executionTime': '2026-02-11T10:10:07+00:00'}, - {'jobId': '38487917', 'executionTime': '2026-02-10T10:10:05+00:00'}, - {}, - ], - } - schedule = ScheduleApiResponse.model_validate(raw_response) - assert len(schedule.executions) == 3 - assert schedule.executions[0].job_id == '38562456' # accepted via 'job_id' alias - assert schedule.executions[1].job_id == '38487917' - assert schedule.executions[2].job_id is None - assert schedule.executions[2].execution_time is None diff --git a/tests/tools/flow/test_tools.py b/tests/tools/flow/test_tools.py deleted file mode 100644 index 0f4dc7f2e..000000000 --- a/tests/tools/flow/test_tools.py +++ /dev/null @@ -1,1300 +0,0 @@ -"""Unit tests for Flow management tools.""" - -from typing import Any, Dict, List - -import httpx -import pytest -from mcp.server.fastmcp import Context -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.client import CONDITIONAL_FLOW_COMPONENT_ID, ORCHESTRATOR_COMPONENT_ID, KeboolaClient -from keboola_mcp_server.clients.storage import APIFlowResponse -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.flow.model import ( - ConditionalFlowConfiguration, - ConditionalFlowPhase, - ConditionalFlowTask, - Flow, - FlowConfiguration, - FlowPhase, - FlowSummary, - FlowTask, - GetFlowsDetailOutput, - GetFlowsListOutput, -) -from keboola_mcp_server.tools.flow.scheduler_model import SchedulesOutput -from keboola_mcp_server.tools.flow.tools import ( - FlowToolOutput, - create_conditional_flow, - create_flow, - get_flow_examples, - get_flow_schema, - get_flows, - modify_flow, - update_flow, -) - -# ============================================================================= -# FLOW DATA FIXTURES -# ============================================================================= - - -@pytest.fixture -def legacy_flow_phases() -> List[Dict[str, Any]]: - """Sample legacy flow phases.""" - return [ - {'id': 1, 'name': 'Data Extraction', 'description': 'Extract data from various sources', 'dependsOn': []}, - {'id': 2, 'name': 'Data Transformation', 'description': 'Transform and process data', 'dependsOn': [1]}, - {'id': 3, 'name': 'Data Loading', 'description': 'Load data to destination', 'dependsOn': [2]}, - ] - - -@pytest.fixture -def legacy_flow_tasks() -> List[Dict[str, Any]]: - """Sample legacy flow tasks.""" - return [ - { - 'id': 20001, - 'name': 'Extract from S3', - 'phase': 1, - 'enabled': True, - 'continueOnFailure': False, - 'task': {'componentId': 'keboola.ex-aws-s3', 'configId': '123456', 'mode': 'run'}, - }, - { - 'id': 20002, - 'name': 'Transform Data', - 'phase': 2, - 'enabled': True, - 'continueOnFailure': False, - 'task': {'componentId': 'keboola.snowflake-transformation', 'configId': '789012', 'mode': 'run'}, - }, - { - 'id': 20003, - 'name': 'Load to Warehouse', - 'phase': 3, - 'enabled': True, - 'continueOnFailure': False, - 'task': {'componentId': 'keboola.wr-snowflake', 'configId': '345678', 'mode': 'run'}, - }, - ] - - -@pytest.fixture -def mock_conditional_flow_phases() -> List[Dict[str, Any]]: - """Sample conditional flow phases with simple configuration.""" - return [ - { - 'id': 'phase1', - 'name': 'Simple Phase', - 'description': 'A simple conditional flow phase', - 'next': [{'id': 'transition1', 'name': 'Simple Transition', 'goto': None}], - } - ] - - -@pytest.fixture -def mock_conditional_flow_tasks() -> List[Dict[str, Any]]: - """Sample conditional flow tasks with simple configuration.""" - return [ - { - 'id': 'task1', - 'name': 'Simple Task', - 'phase': 'phase1', - 'enabled': True, - 'task': { - 'type': 'notification', - 'recipients': [{'channel': 'email', 'address': 'admin@company.com'}], - 'title': 'Simple Notification', - 'message': 'This is a simple notification task', - }, - } - ] - - -@pytest.fixture -def mock_conditional_flow( - mock_conditional_flow_phases: List[Dict[str, Any]], mock_conditional_flow_tasks: List[Dict[str, Any]] -) -> Dict[str, Any]: - """Mock conditional flow configuration response for get_flow endpoint.""" - return { - 'component_id': CONDITIONAL_FLOW_COMPONENT_ID, - 'configuration_id': 'conditional_flow_456', - 'name': 'Advanced Data Pipeline', - 'description': 'Advanced pipeline with conditional logic and error handling', - 'created': '2025-01-15T11:00:00Z', - 'updated': '2025-01-15T11:00:00Z', - 'creatorToken': {'id': 'test_token', 'description': 'Test token'}, - 'version': 1, - 'changeDescription': 'Initial creation', - 'isDisabled': False, - 'isDeleted': False, - 'configuration': {'phases': mock_conditional_flow_phases, 'tasks': mock_conditional_flow_tasks}, - 'rows': [], - 'metadata': [], - } - - -@pytest.fixture -def mock_conditional_flow_create_update( - mock_conditional_flow_phases: List[Dict[str, Any]], mock_conditional_flow_tasks: List[Dict[str, Any]] -) -> Dict[str, Any]: - """Mock conditional flow configuration response for create/update endpoints.""" - return { - 'id': 'conditional_flow_456', - 'name': 'Advanced Data Pipeline', - 'description': 'Advanced pipeline with conditional logic and error handling', - 'created': '2025-01-15T11:00:00Z', - 'creatorToken': {'id': 'test_token', 'description': 'Test token'}, - 'version': 1, - 'changeDescription': 'Initial creation', - 'isDisabled': False, - 'isDeleted': False, - 'configuration': {'phases': mock_conditional_flow_phases, 'tasks': mock_conditional_flow_tasks}, - 'state': {}, - 'currentVersion': {'version': 1}, - } - - -@pytest.fixture -def mock_legacy_flow_create_update( - legacy_flow_phases: list[dict[str, Any]], legacy_flow_tasks: list[dict[str, Any]] -) -> dict[str, Any]: - """Mock legacy flow configuration response for create/update endpoints.""" - return { - 'id': 'legacy_flow_123', - 'name': 'Legacy ETL Pipeline', - 'description': 'Traditional ETL pipeline using legacy flows', - 'created': '2025-01-15T10:30:00Z', - 'creatorToken': {'id': 'test_token', 'description': 'Test token'}, - 'version': 1, - 'changeDescription': 'Initial creation', - 'isDisabled': False, - 'isDeleted': False, - 'configuration': {'phases': legacy_flow_phases, 'tasks': legacy_flow_tasks}, - 'state': {}, - 'currentVersion': {'version': 1}, - } - - -@pytest.fixture -def mock_legacy_flow( - legacy_flow_phases: list[dict[str, Any]], legacy_flow_tasks: list[dict[str, Any]] -) -> dict[str, Any]: - """Mock legacy flow configuration response for get_flow endpoint.""" - return { - 'component_id': ORCHESTRATOR_COMPONENT_ID, - 'configuration_id': 'legacy_flow_123', - 'name': 'Legacy ETL Pipeline', - 'description': 'Traditional ETL pipeline using legacy flows', - 'created': '2025-01-15T10:30:00Z', - 'updated': '2025-01-15T10:30:00Z', - 'creatorToken': {'id': 'test_token', 'description': 'Test token'}, - 'version': 1, - 'changeDescription': 'Initial creation', - 'isDisabled': False, - 'isDeleted': False, - 'configuration': {'phases': legacy_flow_phases, 'tasks': legacy_flow_tasks}, - 'rows': [], - 'metadata': [], - } - - -# ============================================================================= -# CREATE_FLOW TOOL TESTS -# ============================================================================= - - -class TestCreateFlowTool: - """Tests for the create_flow tool.""" - - @pytest.mark.asyncio - async def test_create_legacy_flow( - self, - mocker: MockerFixture, - mcp_context_client: Context, - legacy_flow_phases: list[dict[str, Any]], - legacy_flow_tasks: list[dict[str, Any]], - mock_legacy_flow_create_update: dict[str, Any], - ): - """Should create a new legacy (orchestrator) flow with valid phases/tasks.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - mocker.patch.object( - keboola_client.storage_client, - 'configuration_create', - return_value=mock_legacy_flow_create_update, - ) - - result = await create_flow( - ctx=mcp_context_client, - name='Legacy ETL Pipeline', - description='Traditional ETL pipeline using legacy flows', - phases=legacy_flow_phases, - tasks=legacy_flow_tasks, - ) - - assert isinstance(result, FlowToolOutput) - assert result.success is True - assert result.configuration_id == mock_legacy_flow_create_update['id'] - assert result.component_id == 'keboola.orchestrator' - assert result.description == mock_legacy_flow_create_update['description'] - assert result.timestamp is not None - assert len(result.links) == 3 - assert result.version == mock_legacy_flow_create_update['version'] - - keboola_client.storage_client.configuration_create.assert_called_once() - - @pytest.mark.asyncio - async def test_create_conditional_flow( - self, - mocker: MockerFixture, - mcp_context_client: Context, - mock_conditional_flow_create_update: Dict[str, Any], - conditional_flow_schema: dict, - ): - """Test conditional flow creation.""" - component = mocker.Mock() - component.configuration_schema = conditional_flow_schema - mocker.patch( - 'keboola_mcp_server.tools.flow.utils.fetch_component', - mocker.AsyncMock(return_value=component), - ) - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.configuration_create = mocker.AsyncMock( - return_value=mock_conditional_flow_create_update - ) - - result = await create_conditional_flow( - ctx=mcp_context_client, - name='Advanced Data Pipeline', - description='Advanced pipeline with conditional logic and error handling', - phases=mock_conditional_flow_create_update['configuration']['phases'], - tasks=mock_conditional_flow_create_update['configuration']['tasks'], - ) - - assert isinstance(result, FlowToolOutput) - assert result.success is True - assert result.configuration_id == mock_conditional_flow_create_update['id'] - assert result.component_id == 'keboola.flow' - assert result.description == mock_conditional_flow_create_update['description'] - assert result.timestamp is not None - assert len(result.links) == 3 - assert result.version == mock_conditional_flow_create_update['version'] - - keboola_client.storage_client.configuration_create.assert_called_once() - - -# ============================================================================= -# UPDATE_FLOW TOOL TESTS -# ============================================================================= - - -class TestUpdateFlowTool: - """Tests for the update_flow tool.""" - - # TODO: The test_update_*() tests need to cover different variations of the tool's parameters - # and properly check that the original flow was correctly updated. - - @pytest.mark.asyncio - async def test_update_legacy_flow( - self, - mocker: MockerFixture, - mcp_context_client: Context, - legacy_flow_phases: List[Dict[str, Any]], - legacy_flow_tasks: List[Dict[str, Any]], - mock_legacy_flow_create_update: Dict[str, Any], - ): - """Test legacy flow update with new phases and tasks.""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = True - - async def mock_get_project_info(ctx): - return mock_project_info - - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', side_effect=mock_get_project_info) - - updated_config = mock_legacy_flow_create_update.copy() - updated_config['version'] = 2 - updated_config['description'] = 'Updated legacy ETL pipeline' - - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value={}) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=updated_config) - - result = await update_flow( - ctx=mcp_context_client, - configuration_id='legacy_flow_123', - flow_type=ORCHESTRATOR_COMPONENT_ID, - name='Updated Legacy ETL Pipeline', - description='Updated legacy ETL pipeline', - phases=legacy_flow_phases, - tasks=legacy_flow_tasks, - change_description='Added data validation phase and enhanced error handling', - ) - - assert isinstance(result, FlowToolOutput) - assert result.success is True - assert result.configuration_id == 'legacy_flow_123' - assert result.component_id == 'keboola.orchestrator' - assert result.description == 'Updated legacy ETL pipeline' - assert result.timestamp is not None - assert len(result.links) == 3 - assert result.version == updated_config['version'] - - keboola_client.storage_client.configuration_update.assert_called_once() - - @pytest.mark.asyncio - async def test_update_conditional_flow( - self, - mocker: MockerFixture, - mcp_context_client: Context, - mock_conditional_flow_create_update: Dict[str, Any], - conditional_flow_schema: dict, - ): - """Test conditional flow update with enhanced conditions.""" - component = mocker.Mock() - component.configuration_schema = conditional_flow_schema - mocker.patch( - 'keboola_mcp_server.tools.flow.utils.fetch_component', - mocker.AsyncMock(return_value=component), - ) - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = True - - async def mock_get_project_info(ctx): - return mock_project_info - - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', side_effect=mock_get_project_info) - - updated_config = mock_conditional_flow_create_update.copy() - updated_config['version'] = 2 - updated_config['name'] = 'Enhanced Advanced Data Pipeline' - updated_config['description'] = 'Enhanced pipeline with improved conditional logic' - - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value={}) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=updated_config) - - result = await update_flow( - ctx=mcp_context_client, - configuration_id='conditional_flow_456', - flow_type=CONDITIONAL_FLOW_COMPONENT_ID, - name='Enhanced Advanced Data Pipeline', - description='Enhanced pipeline with improved conditional logic', - phases=mock_conditional_flow_create_update['configuration']['phases'], - tasks=mock_conditional_flow_create_update['configuration']['tasks'], - change_description='Enhanced error handling and added notification phase', - ) - - assert isinstance(result, FlowToolOutput) - assert result.success is True - assert result.configuration_id == updated_config['id'] - assert result.component_id == 'keboola.flow' - assert result.description == updated_config['description'] - assert result.timestamp is not None - assert len(result.links) == 3 - assert result.version == updated_config['version'] - - keboola_client.storage_client.configuration_update.assert_called_once() - - @pytest.mark.asyncio - async def test_update_conditional_flow_fails_when_conditional_flows_disabled( - self, - mocker: MockerFixture, - mcp_context_client: Context, - ): - """Test that updating conditional flow fails when conditional flows are disabled.""" - # Mock project info with conditional flows disabled - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = False - mock_project_info.project_name = 'Test Project' - - async def mock_get_project_info(ctx): - return mock_project_info - - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', side_effect=mock_get_project_info) - - # Should raise ValueError with proper error message - with pytest.raises(ValueError, match='Conditional flows are not supported.') as exc_info: - await update_flow( - ctx=mcp_context_client, - configuration_id='test-config-id', - flow_type=CONDITIONAL_FLOW_COMPONENT_ID, - name='Updated Conditional Flow', - description='Updated description for conditional flow', - phases=[], - tasks=[], - change_description='Test update', - ) - - error_message = str(exc_info.value) - assert 'Conditional flows are not supported in this project' in error_message - assert 'Test Project' in error_message - assert 'conditional_flows=false' in error_message - assert 'enable them in your project settings' in error_message - - -# ============================================================================= -# GET_FLOWS TOOL TESTS -# ============================================================================= - - -class TestGetFlowsTool: - """Tests for the get_flows tool.""" - - @pytest.mark.asyncio - async def test_get_flows_with_legacy_flow_id( - self, - mocker: MockerFixture, - mcp_context_client: Context, - mock_legacy_flow: dict[str, Any], - legacy_flow_phases: list[dict[str, Any]], - legacy_flow_tasks: list[dict[str, Any]], - ): - """Should fall back to legacy flow when conditional flow is missing (404).""" - - async def mock_configuration_detail(component_id: str, configuration_id: str) -> dict[str, Any]: - if component_id == CONDITIONAL_FLOW_COMPONENT_ID: - response = mocker.Mock(status_code=404) - raise httpx.HTTPStatusError('404 Not Found', request=None, response=response) - if component_id == ORCHESTRATOR_COMPONENT_ID: - return mock_legacy_flow - raise ValueError(f'Unexpected component_id: {component_id}') - - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - mocker.patch.object(keboola_client.scheduler_client, 'list_schedules_by_config_id', return_value=[]) - mocker.patch.object( - keboola_client.storage_client, 'configuration_detail', side_effect=mock_configuration_detail - ) - - result = await get_flows( - ctx=mcp_context_client, - flow_ids=[mock_legacy_flow['configuration_id']], - ) - - # Get URL components from context - storage_api_url = keboola_client.storage_api_url - project_id = await keboola_client.storage_client.project_id() - base_url = f'{storage_api_url}/admin/projects/{project_id}' - - expected_flow = Flow( - component_id=ORCHESTRATOR_COMPONENT_ID, - configuration_id=mock_legacy_flow['configuration_id'], - name=mock_legacy_flow['name'], - description=mock_legacy_flow['description'], - version=mock_legacy_flow['version'], - is_disabled=mock_legacy_flow['isDisabled'], - is_deleted=mock_legacy_flow['isDeleted'], - configuration=FlowConfiguration( - phases=[FlowPhase.model_validate(p) for p in legacy_flow_phases], - tasks=[FlowTask.model_validate(t) for t in legacy_flow_tasks], - ), - change_description=mock_legacy_flow['changeDescription'], - configuration_metadata=mock_legacy_flow['metadata'], - created=mock_legacy_flow['created'], - updated=mock_legacy_flow['updated'], - links=[ - Link( - type='ui-detail', - title=f"Flow: {mock_legacy_flow['name']}", - url=f"{base_url}/flows/{mock_legacy_flow['configuration_id']}", - ), - Link(type='ui-dashboard', title='Flows in the project', url=f'{base_url}/flows'), - Link(type='docs', title='Documentation for Keboola Flows', url='https://help.keboola.com/flows/'), - ], - schedules=SchedulesOutput( - schedules=[], - n_schedules=0, - links=[ - Link( - type='ui-detail', - title='Schedules', - url=f"{base_url}/flows/{mock_legacy_flow['configuration_id']}/schedules", - ) - ], - ), - ) - - assert result == GetFlowsDetailOutput(flows=[expected_flow]) - - @pytest.mark.asyncio - async def test_get_flows_with_conditional_flow_id( - self, - mocker: MockerFixture, - mcp_context_client: Context, - mock_conditional_flow: Dict[str, Any], - mock_conditional_flow_phases: list[dict[str, Any]], - mock_conditional_flow_tasks: list[dict[str, Any]], - ): - """Test retrieving conditional flow details.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=mock_conditional_flow) - - result = await get_flows(ctx=mcp_context_client, flow_ids=[mock_conditional_flow['configuration_id']]) - - # Get URL components from context - storage_api_url = keboola_client.storage_api_url - project_id = await keboola_client.storage_client.project_id() - base_url = f'{storage_api_url}/admin/projects/{project_id}' - - expected_flow = Flow( - component_id=CONDITIONAL_FLOW_COMPONENT_ID, - configuration_id=mock_conditional_flow['configuration_id'], - name=mock_conditional_flow['name'], - description=mock_conditional_flow['description'], - version=mock_conditional_flow['version'], - is_disabled=mock_conditional_flow['isDisabled'], - is_deleted=mock_conditional_flow['isDeleted'], - configuration=ConditionalFlowConfiguration( - phases=[ConditionalFlowPhase.model_validate(p) for p in mock_conditional_flow_phases], - tasks=[ConditionalFlowTask.model_validate(t) for t in mock_conditional_flow_tasks], - ), - change_description=mock_conditional_flow['changeDescription'], - configuration_metadata=mock_conditional_flow['metadata'], - created=mock_conditional_flow['created'], - updated=mock_conditional_flow['updated'], - links=[ - Link( - type='ui-detail', - title=f"Flow: {mock_conditional_flow['name']}", - url=f"{base_url}/flows-v2/{mock_conditional_flow['configuration_id']}", - ), - Link(type='ui-dashboard', title='Conditional Flows in the project', url=f'{base_url}/flows-v2'), - Link(type='docs', title='Documentation for Keboola Flows', url='https://help.keboola.com/flows/'), - ], - schedules=SchedulesOutput( - schedules=[], - n_schedules=0, - links=[ - Link( - type='ui-detail', - title='Schedules', - url=f"{base_url}/flows-v2/{mock_conditional_flow['configuration_id']}/schedules", - ), - ], - ), - ) - - assert result == GetFlowsDetailOutput(flows=[expected_flow]) - - @pytest.mark.asyncio - async def test_get_flows_no_params( - self, - mocker: MockerFixture, - mcp_context_client: Context, - mock_legacy_flow: Dict[str, Any], - mock_conditional_flow: Dict[str, Any], - ): - """Test listing flows of both types.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - def mock_configuration_list(component_id): - if component_id == ORCHESTRATOR_COMPONENT_ID: - return [mock_legacy_flow] - elif component_id == CONDITIONAL_FLOW_COMPONENT_ID: - return [mock_conditional_flow] - return [] - - keboola_client.storage_client.configuration_list = mocker.AsyncMock(side_effect=mock_configuration_list) - - result = await get_flows(ctx=mcp_context_client) - - # Get URL components from context - storage_api_url = keboola_client.storage_api_url - project_id = await keboola_client.storage_client.project_id() - base_url = f'{storage_api_url}/admin/projects/{project_id}' - - expected_legacy_summary = FlowSummary( - component_id=ORCHESTRATOR_COMPONENT_ID, - configuration_id=mock_legacy_flow['configuration_id'], - name=mock_legacy_flow['name'], - description=mock_legacy_flow['description'], - version=mock_legacy_flow['version'], - is_disabled=mock_legacy_flow['isDisabled'], - is_deleted=mock_legacy_flow['isDeleted'], - phases_count=len(mock_legacy_flow['configuration']['phases']), - tasks_count=len(mock_legacy_flow['configuration']['tasks']), - created=mock_legacy_flow['created'], - updated=mock_legacy_flow['updated'], - ) - - expected_conditional_summary = FlowSummary( - component_id=CONDITIONAL_FLOW_COMPONENT_ID, - configuration_id=mock_conditional_flow['configuration_id'], - name=mock_conditional_flow['name'], - description=mock_conditional_flow['description'], - version=mock_conditional_flow['version'], - is_disabled=mock_conditional_flow['isDisabled'], - is_deleted=mock_conditional_flow['isDeleted'], - phases_count=len(mock_conditional_flow['configuration']['phases']), - tasks_count=len(mock_conditional_flow['configuration']['tasks']), - created=mock_conditional_flow['created'], - updated=mock_conditional_flow['updated'], - ) - - expected_links = [ - Link(type='ui-dashboard', title='Flows in the project', url=f'{base_url}/flows'), - Link(type='ui-dashboard', title='Conditional Flows in the project', url=f'{base_url}/flows-v2'), - ] - - # Note: flows are returned in FLOW_TYPES order (conditional flows first, then legacy) - assert result == GetFlowsListOutput( - flows=[expected_conditional_summary, expected_legacy_summary], - links=expected_links, - ) - assert keboola_client.storage_client.configuration_list.call_count == 2 - - @pytest.mark.asyncio - async def test_get_flows_specific_ids_mixed_types( - self, - mocker: MockerFixture, - mcp_context_client: Context, - mock_legacy_flow: Dict[str, Any], - mock_conditional_flow: Dict[str, Any], - legacy_flow_phases: list[dict[str, Any]], - legacy_flow_tasks: list[dict[str, Any]], - mock_conditional_flow_phases: list[dict[str, Any]], - mock_conditional_flow_tasks: list[dict[str, Any]], - ): - """Test retrieving specific flows by ID when they're different types.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - legacy_id = mock_legacy_flow['configuration_id'] - conditional_id = mock_conditional_flow['configuration_id'] - - def mock_configuration_detail(component_id, configuration_id): - if configuration_id == legacy_id and component_id == ORCHESTRATOR_COMPONENT_ID: - return mock_legacy_flow - elif configuration_id == conditional_id and component_id == CONDITIONAL_FLOW_COMPONENT_ID: - return mock_conditional_flow - raise Exception(f'Configuration {configuration_id} not found') - - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(side_effect=mock_configuration_detail) - mocker.patch.object(keboola_client.scheduler_client, 'list_schedules_by_config_id', return_value=[]) - - result = await get_flows(ctx=mcp_context_client, flow_ids=[legacy_id, conditional_id]) - - # Get URL components from context - storage_api_url = keboola_client.storage_api_url - project_id = await keboola_client.storage_client.project_id() - base_url = f'{storage_api_url}/admin/projects/{project_id}' - - expected_legacy_flow = Flow( - component_id=ORCHESTRATOR_COMPONENT_ID, - configuration_id=mock_legacy_flow['configuration_id'], - name=mock_legacy_flow['name'], - description=mock_legacy_flow['description'], - version=mock_legacy_flow['version'], - is_disabled=mock_legacy_flow['isDisabled'], - is_deleted=mock_legacy_flow['isDeleted'], - configuration=FlowConfiguration( - phases=[FlowPhase.model_validate(p) for p in legacy_flow_phases], - tasks=[FlowTask.model_validate(t) for t in legacy_flow_tasks], - ), - change_description=mock_legacy_flow['changeDescription'], - configuration_metadata=mock_legacy_flow['metadata'], - created=mock_legacy_flow['created'], - updated=mock_legacy_flow['updated'], - links=[ - Link( - type='ui-detail', - title=f"Flow: {mock_legacy_flow['name']}", - url=f"{base_url}/flows/{mock_legacy_flow['configuration_id']}", - ), - Link(type='ui-dashboard', title='Flows in the project', url=f'{base_url}/flows'), - Link(type='docs', title='Documentation for Keboola Flows', url='https://help.keboola.com/flows/'), - ], - schedules=SchedulesOutput( - schedules=[], - n_schedules=0, - links=[ - Link( - type='ui-detail', - title='Schedules', - url=f"{base_url}/flows/{mock_legacy_flow['configuration_id']}/schedules", - ), - ], - ), - ) - - expected_conditional_flow = Flow( - component_id=CONDITIONAL_FLOW_COMPONENT_ID, - configuration_id=mock_conditional_flow['configuration_id'], - name=mock_conditional_flow['name'], - description=mock_conditional_flow['description'], - version=mock_conditional_flow['version'], - is_disabled=mock_conditional_flow['isDisabled'], - is_deleted=mock_conditional_flow['isDeleted'], - configuration=ConditionalFlowConfiguration( - phases=[ConditionalFlowPhase.model_validate(p) for p in mock_conditional_flow_phases], - tasks=[ConditionalFlowTask.model_validate(t) for t in mock_conditional_flow_tasks], - ), - change_description=mock_conditional_flow['changeDescription'], - configuration_metadata=mock_conditional_flow['metadata'], - created=mock_conditional_flow['created'], - updated=mock_conditional_flow['updated'], - links=[ - Link( - type='ui-detail', - title=f"Flow: {mock_conditional_flow['name']}", - url=f"{base_url}/flows-v2/{mock_conditional_flow['configuration_id']}", - ), - Link(type='ui-dashboard', title='Conditional Flows in the project', url=f'{base_url}/flows-v2'), - Link(type='docs', title='Documentation for Keboola Flows', url='https://help.keboola.com/flows/'), - ], - schedules=SchedulesOutput( - schedules=[], - n_schedules=0, - links=[ - Link( - type='ui-detail', - title='Schedules', - url=f"{base_url}/flows-v2/{mock_conditional_flow['configuration_id']}/schedules", - ), - ], - ), - ) - - assert result == GetFlowsDetailOutput(flows=[expected_legacy_flow, expected_conditional_flow]) - # Since we look up for both types (conditional flows first) we expect the calls to be 2 and 1, respectfully - assert keboola_client.storage_client.configuration_detail.call_count == 3 - - -@pytest.mark.parametrize( - ('metadata', 'expected_folder'), - [ - ([{'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'ETL', 'provider': 'user'}], 'ETL'), - ([], ''), - ], - ids=['folder_in_metadata', 'no_metadata'], -) -def test_get_flows_includes_folder( - metadata: list[dict], - expected_folder: str, - legacy_flow_phases: list[dict[str, Any]], - legacy_flow_tasks: list[dict[str, Any]], -) -> None: - """FlowSummary.from_api_response extracts folder from metadata.""" - api_config = APIFlowResponse.model_validate( - { - 'id': 'flow_123', - 'name': 'My Flow', - 'version': 1, - 'configuration': {'phases': legacy_flow_phases, 'tasks': legacy_flow_tasks}, - 'metadata': metadata, - } - ) - summary = FlowSummary.from_api_response(api_config, ORCHESTRATOR_COMPONENT_ID) - assert summary.folder == expected_folder - - -# ============================================================================= -# GET_FLOW_SCHEMA TOOL TESTS -# ============================================================================= - - -class TestGetFlowSchemaTool: - """Tests for the get_flow_schema tool.""" - - @pytest.mark.asyncio - async def test_get_legacy_flow_schema_when_conditional_flows_disabled( - self, - mocker: MockerFixture, - mcp_context_client: Context, - ): - """Test getting schema for legacy flow type when conditional flows are disabled.""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = False - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', return_value=mock_project_info) - - result = await get_flow_schema(ctx=mcp_context_client, flow_type=ORCHESTRATOR_COMPONENT_ID) - - assert isinstance(result, str) - assert '```json' in result - assert 'dependsOn' in result - - @pytest.mark.asyncio - async def test_get_legacy_flow_schema_when_conditional_flows_enabled( - self, - mocker: MockerFixture, - mcp_context_client: Context, - ): - """Test getting schema for legacy flow type when conditional flows are enabled.""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = True - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', return_value=mock_project_info) - - result = await get_flow_schema(ctx=mcp_context_client, flow_type=ORCHESTRATOR_COMPONENT_ID) - - assert isinstance(result, str) - assert '```json' in result - assert 'dependsOn' in result - - @pytest.mark.asyncio - async def test_get_conditional_flow_schema_when_conditional_flows_enabled( - self, - mocker: MockerFixture, - mcp_context_client: Context, - conditional_flow_schema: dict, - ): - """Conditional schema is sourced live (mocked) when conditional flows are enabled.""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = True - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', return_value=mock_project_info) - - component = mocker.Mock() - component.configuration_schema = conditional_flow_schema - mocker.patch( - 'keboola_mcp_server.tools.flow.utils.fetch_component', - mocker.AsyncMock(return_value=component), - ) - - result = await get_flow_schema(ctx=mcp_context_client, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - - assert isinstance(result, str) - assert result.startswith('```json\n') - assert result.endswith('\n```') - assert 'next' in result - - @pytest.mark.asyncio - async def test_get_conditional_flow_schema_fails_when_conditional_flows_disabled( - self, - mocker: MockerFixture, - mcp_context_client: Context, - ): - """Test that requesting conditional flow schema fails when conditional flows are disabled.""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = False - mock_project_info.project_name = 'Test Project' - - async def mock_get_project_info(ctx): - return mock_project_info - - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', side_effect=mock_get_project_info) - - # Should raise ValueError with proper error message - with pytest.raises(ValueError, match='Conditional flows are not supported.') as exc_info: - await get_flow_schema(ctx=mcp_context_client, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - - error_message = str(exc_info.value) - assert 'Conditional flows are not supported in this project' in error_message - assert 'Test Project' in error_message - assert 'conditional_flows=false' in error_message - assert 'enable them in your project settings' in error_message - - -# ============================================================================= -# GET_FLOW_EXAMPLES TOOL TESTS -# ============================================================================= - - -class TestGetFlowExamplesTool: - """Tests for the get_flow_examples tool.""" - - @pytest.mark.asyncio - async def test_get_legacy_flow_examples( - self, - mocker: MockerFixture, - mcp_context_client: Context, - ): - """Test getting examples for legacy flow type.""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = True - - async def mock_get_project_info(ctx): - return mock_project_info - - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', side_effect=mock_get_project_info) - - # Mock the file path and content properly - using actual structure from the real file - mock_file_content = [ - ( - '{"tasks":[{"id":1,"name":"keboola.wr-google-bigquery-v2-28356142",' - '"task":{"mode":"run","configId":"28356142","componentId":"keboola.wr-google-bigquery-v2"},' - '"phase":1,"continueOnFailure":false,"enabled":true}],' - '"phases":[{"id":1,"name":"Scheduledconfiguration","dependsOn":[]}]}' - ), - ( - '{"phases":[{"id":59812,"name":"Extraction","dependsOn":[],' - '"description":"ExtractdatafromWhenIworkandPaychex"}],' - '"tasks":[{"id":36614,"name":"ex-generic-v2-34446855","phase":59812,' - '"task":{"componentId":"ex-generic-v2","configId":"34446855","mode":"run"},' - '"continueOnFailure":false,"enabled":false}]}' - ), - ] - - # Mock the file path resolution - mock_path = mocker.Mock() - mock_path.__truediv__ = mocker.Mock(return_value=mock_path) - mock_path.open = mocker.mock_open(read_data='\n'.join(mock_file_content)) - - # Mock the importlib.resources.files function - mocker.patch('importlib.resources.files', return_value=mock_path) - - result = await get_flow_examples(ctx=mcp_context_client, flow_type=ORCHESTRATOR_COMPONENT_ID) - - assert isinstance(result, str) - assert 'Flow Configuration Examples for `keboola.orchestrator`' in result - assert 'keboola.wr-google-bigquery-v2-28356142' in result - assert 'ex-generic-v2-34446855' in result - assert 'Scheduledconfiguration' in result - assert 'Extraction' in result - - @pytest.mark.asyncio - async def test_get_conditional_flow_examples( - self, - mocker: MockerFixture, - mcp_context_client: Context, - ): - """Test getting examples for conditional flow type.""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = True - - async def mock_get_project_info(ctx): - return mock_project_info - - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', side_effect=mock_get_project_info) - - # Mock the file path and content properly - using actual structure from the real file - mock_file_content = [ - ( - '{"tasks":[{"id":"40fef978-7092-4d79-a5b4-ea3fb2e38d03",' - '"name":"keboola.wr-azure-event-hub-92021091",' - '"phase":"6afbf55b-782c-47d7-bf70-f0ef1be6505b",' - '"task":{"type":"job","mode":"run","componentId":"keboola.wr-azure-event-hub","configId":"92021091"},' - '"enabled":true}],' - '"phases":[{"id":"7dd992b0-9ac5-495b-b277-d8bc0b7e15d5",' - '"name":"Phase1",' - '"next":[{"id":"a25a4e4a-3042-49a2-81d8-fb1103957ebe",' - '"goto":"6afbf55b-782c-47d7-bf70-f0ef1be6505b"}]}]}' - ), - ( - '{"tasks":[{"id":"6bcc72d8-d9a5-4708-b0bd-53c4f6e839f7",' - '"name":"keboola.python-transformation-v2-16550",' - '"phase":"78c07164-0d1c-41d6-ba48-b821e781d830",' - '"task":{"type":"job","mode":"run",' - '"componentId":"keboola.python-transformation-v2","configId":"16550"},' - '"enabled":true}],' - '"phases":[{"id":"78c07164-0d1c-41d6-ba48-b821e781d830",' - '"name":"Phase1",' - '"next":[{"id":"e5dc7c43-d311-4e90-a2ca-6cac8d2eb5f5",' - '"goto":"92649482-45d6-475d-aace-33466f37e381"}]}]}' - ), - ] - - # Mock the file path resolution - mock_path = mocker.Mock() - mock_path.__truediv__ = mocker.Mock(return_value=mock_path) - mock_path.open = mocker.mock_open(read_data='\n'.join(mock_file_content)) - - # Mock the importlib.resources.files function - mocker.patch('importlib.resources.files', return_value=mock_path) - - result = await get_flow_examples(ctx=mcp_context_client, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - - assert isinstance(result, str) - assert 'Flow Configuration Examples for `keboola.flow`' in result - assert 'keboola.wr-azure-event-hub-92021091' in result - assert 'keboola.python-transformation-v2-16550' in result - assert 'Phase1' in result - - @pytest.mark.asyncio - async def test_get_conditional_flow_examples_when_conditional_flows_disabled( - self, - mocker: MockerFixture, - mcp_context_client: Context, - ): - """Test that requesting conditional flow examples fails when conditional flows are disabled.""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = False - mock_project_info.project_name = 'Test Project' - - async def mock_get_project_info(ctx): - return mock_project_info - - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', side_effect=mock_get_project_info) - - # Mock the file path resolution (should not be called due to early failure) - mock_path = mocker.Mock() - mock_path.__truediv__ = mocker.Mock(return_value=mock_path) - mock_path.open = mocker.mock_open() - mocker.patch('importlib.resources.files', return_value=mock_path) - - # Should raise ValueError with proper error message - with pytest.raises(ValueError, match='Conditional flows are not supported.') as exc_info: - await get_flow_examples(ctx=mcp_context_client, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - error_message = str(exc_info.value) - assert 'Conditional flows are not supported in this project' in error_message - assert 'Test Project' in error_message - assert 'conditional_flows=false' in error_message - assert 'enable them in your project settings' in error_message - - -# ============================================================================= -# FOLDER METADATA TESTS -# ============================================================================= - - -@pytest.mark.parametrize( - ('folder', 'flow_count', 'flow_folders', 'expect_folder_metadata', 'expect_hint'), - [ - ('ETL', 0, [], True, False), - (' ETL ', 0, [], True, False), # whitespace stripped - ('', 5, [], False, False), - ('', 25, ['ETL'], False, True), - ('', 25, [], False, True), - ], - ids=[ - 'folder_provided', - 'folder_whitespace_stripped', - 'no_folder_few', - 'no_folder_many_with_folders', - 'no_folder_many_no_folders', - ], -) -@pytest.mark.asyncio -async def test_create_flow_folder( - mocker: MockerFixture, - mcp_context_client: Context, - mock_legacy_flow_create_update: dict[str, Any], - legacy_flow_phases: list[dict[str, Any]], - legacy_flow_tasks: list[dict[str, Any]], - folder: str, - flow_count: int, - flow_folders: list[str], - expect_folder_metadata: bool, - expect_hint: bool, -) -> None: - """Test folder metadata and change_summary hint for create_flow.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.configuration_create = mocker.AsyncMock(return_value=mock_legacy_flow_create_update) - mocker.patch( - 'keboola_mcp_server.tools.flow.tools.get_config_folders', - mocker.AsyncMock(return_value=(flow_count, flow_folders, False)), - ) - - result = await create_flow( - ctx=mcp_context_client, - name='ETL Pipeline', - description='desc', - phases=legacy_flow_phases, - tasks=legacy_flow_tasks, - folder=folder, - ) - - assert isinstance(result, FlowToolOutput) - metadata_calls = [ - call - for call in keboola_client.storage_client.configuration_metadata_update.call_args_list - if call.kwargs.get('metadata', {}).get(MetadataField.CONFIGURATION_FOLDER_NAME) - ] - if expect_folder_metadata: - assert len(metadata_calls) == 1 - assert metadata_calls[0].kwargs['metadata'] == {MetadataField.CONFIGURATION_FOLDER_NAME: folder.strip()} - else: - assert len(metadata_calls) == 0 - if expect_hint: - assert result.change_summary is not None - assert str(flow_count) in result.change_summary - else: - assert result.change_summary is None - - -@pytest.mark.parametrize( - ('folder', 'flow_count', 'flow_folders', 'expect_folder_metadata', 'expect_hint'), - [ - ('ETL', 0, [], True, False), - (' ETL ', 0, [], True, False), # whitespace stripped - ('', 5, [], False, False), - ('', 25, ['ETL'], False, True), - ('', 25, [], False, True), - ], - ids=[ - 'folder_provided', - 'folder_whitespace_stripped', - 'no_folder_few', - 'no_folder_many_with_folders', - 'no_folder_many_no_folders', - ], -) -@pytest.mark.asyncio -async def test_create_conditional_flow_folder( - mocker: MockerFixture, - mcp_context_client: Context, - mock_conditional_flow_create_update: dict[str, Any], - conditional_flow_schema: dict, - folder: str, - flow_count: int, - flow_folders: list[str], - expect_folder_metadata: bool, - expect_hint: bool, -) -> None: - """Test folder metadata and change_summary hint for create_conditional_flow.""" - component = mocker.Mock() - component.configuration_schema = conditional_flow_schema - mocker.patch( - 'keboola_mcp_server.tools.flow.utils.fetch_component', - mocker.AsyncMock(return_value=component), - ) - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.configuration_create = mocker.AsyncMock( - return_value=mock_conditional_flow_create_update - ) - mocker.patch( - 'keboola_mcp_server.tools.flow.tools.get_config_folders', - mocker.AsyncMock(return_value=(flow_count, flow_folders, False)), - ) - - result = await create_conditional_flow( - ctx=mcp_context_client, - name='Advanced Pipeline', - description='desc', - phases=mock_conditional_flow_create_update['configuration']['phases'], - tasks=mock_conditional_flow_create_update['configuration']['tasks'], - folder=folder, - ) - - assert isinstance(result, FlowToolOutput) - metadata_calls = [ - call - for call in keboola_client.storage_client.configuration_metadata_update.call_args_list - if call.kwargs.get('metadata', {}).get(MetadataField.CONFIGURATION_FOLDER_NAME) - ] - if expect_folder_metadata: - assert len(metadata_calls) == 1 - assert metadata_calls[0].kwargs['metadata'] == {MetadataField.CONFIGURATION_FOLDER_NAME: folder.strip()} - else: - assert len(metadata_calls) == 0 - if expect_hint: - assert result.change_summary is not None - assert str(flow_count) in result.change_summary - else: - assert result.change_summary is None - - -@pytest.mark.parametrize( - ('folder', 'flow_count', 'flow_folders', 'expect_folder_metadata', 'expect_folder_delete', 'expect_hint'), - [ - ('ETL', 0, [], True, False, False), - (' ETL ', 0, [], True, False, False), # whitespace stripped - (None, 5, [], False, False, False), - (None, 25, ['ETL'], False, False, True), - (None, 25, [], False, False, True), - ('', 5, [], False, True, False), # empty string → delete - ], - ids=[ - 'folder_provided', - 'folder_whitespace_stripped', - 'no_folder_few', - 'no_folder_many_with_folders', - 'no_folder_many_no_folders', - 'folder_empty_deletes', - ], -) -@pytest.mark.asyncio -async def test_modify_flow_folder( - mocker: MockerFixture, - mcp_context_client: Context, - mock_legacy_flow_create_update: dict[str, Any], - legacy_flow_phases: list[dict[str, Any]], - legacy_flow_tasks: list[dict[str, Any]], - folder: Any, - flow_count: int, - flow_folders: list[str], - expect_folder_metadata: bool, - expect_folder_delete: bool, - expect_hint: bool, -) -> None: - """Test folder metadata and change_summary hint for modify_flow.""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = True - mock_project_info.project_name = 'Test Project' - - async def mock_get_project_info(ctx): - return mock_project_info - - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', side_effect=mock_get_project_info) - - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=mock_legacy_flow_create_update) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value=mock_legacy_flow_create_update) - keboola_client.storage_client.configuration_metadata_get = mocker.AsyncMock( - return_value=[{'id': 'meta-1', 'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'OldFolder'}] - ) - keboola_client.storage_client.configuration_metadata_delete = mocker.AsyncMock() - mocker.patch( - 'keboola_mcp_server.tools.flow.tools.get_config_folders', - mocker.AsyncMock(return_value=(flow_count, flow_folders, False)), - ) - - result = await modify_flow( - ctx=mcp_context_client, - configuration_id=mock_legacy_flow_create_update['id'], - flow_type=ORCHESTRATOR_COMPONENT_ID, - change_description='test', - name='Updated Pipeline', - folder=folder, - ) - - assert isinstance(result, FlowToolOutput) - metadata_calls = [ - call - for call in keboola_client.storage_client.configuration_metadata_update.call_args_list - if call.kwargs.get('metadata', {}).get(MetadataField.CONFIGURATION_FOLDER_NAME) - ] - if expect_folder_metadata: - assert len(metadata_calls) == 1 - assert metadata_calls[0].kwargs['metadata'] == {MetadataField.CONFIGURATION_FOLDER_NAME: folder.strip()} - else: - assert len(metadata_calls) == 0 - if expect_folder_delete: - keboola_client.storage_client.configuration_metadata_delete.assert_called_once_with( - component_id=ORCHESTRATOR_COMPONENT_ID, - configuration_id=mock_legacy_flow_create_update['id'], - metadata_id='meta-1', - ) - else: - keboola_client.storage_client.configuration_metadata_delete.assert_not_called() - if expect_hint: - assert result.change_summary is not None - assert str(flow_count) in result.change_summary - else: - assert result.change_summary is None - - -@pytest.mark.asyncio -async def test_modify_flow_folder_only( - mocker: MockerFixture, - mcp_context_client: Context, - mock_legacy_flow_create_update: dict[str, Any], -) -> None: - """Test folder metadata is set when folder is the only change (no config fields updated).""" - mock_project_info = mocker.Mock() - mock_project_info.conditional_flows = True - mock_project_info.project_name = 'Test Project' - mocker.patch('keboola_mcp_server.tools.flow.tools.get_project_info', side_effect=lambda ctx: mock_project_info) - - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock(return_value=mock_legacy_flow_create_update) - - result = await modify_flow( - ctx=mcp_context_client, - configuration_id=mock_legacy_flow_create_update['id'], - flow_type=ORCHESTRATOR_COMPONENT_ID, - change_description='assign folder', - folder='ETL', - ) - - assert isinstance(result, FlowToolOutput) - assert result.success is True - # configuration_update must NOT be called — no config changes - keboola_client.storage_client.configuration_update.assert_not_called() - # folder metadata MUST be set - metadata_calls = [ - call - for call in keboola_client.storage_client.configuration_metadata_update.call_args_list - if call.kwargs.get('metadata', {}).get(MetadataField.CONFIGURATION_FOLDER_NAME) - ] - assert len(metadata_calls) == 1 - assert metadata_calls[0].kwargs['metadata'] == {MetadataField.CONFIGURATION_FOLDER_NAME: 'ETL'} diff --git a/tests/tools/flow/test_utils.py b/tests/tools/flow/test_utils.py deleted file mode 100644 index 3f83b94de..000000000 --- a/tests/tools/flow/test_utils.py +++ /dev/null @@ -1,779 +0,0 @@ -from typing import Any - -import pytest -from httpx import ConnectError, HTTPStatusError, Request, Response - -from keboola_mcp_server.clients.client import CONDITIONAL_FLOW_COMPONENT_ID, ORCHESTRATOR_COMPONENT_ID -from keboola_mcp_server.clients.storage import APIFlowResponse -from keboola_mcp_server.tools.flow.model import Flow -from keboola_mcp_server.tools.flow.utils import ( - _check_legacy_circular_dependencies, - _reachable_ids, - ensure_legacy_phase_ids, - ensure_legacy_task_ids, - get_flow_configuration, - resolve_flow_schema, - validate_flow_structure, -) - - -def _notification_task(task_id: str, phase_id: str) -> dict[str, Any]: - """Create a minimal notification task for conditional flow tests.""" - return { - 'id': task_id, - 'name': f'Task {task_id}', - 'phase': phase_id, - 'task': { - 'type': 'notification', - 'title': 'Notify', - 'message': 'Done', - 'recipients': [{'channel': 'email', 'address': 'ops@example.com'}], - }, - } - - -# --- Test Helper Functions --- - - -class TestFlowHelpers: - """Test helper functions for flow processing.""" - - def test_ensure_phase_ids_with_missing_ids(self): - """Test phase ID generation when IDs are missing.""" - phases = [{'name': 'Phase 1'}, {'name': 'Phase 2', 'dependsOn': [1]}, {'id': 5, 'name': 'Phase 5'}] - - processed_phases = ensure_legacy_phase_ids(phases) - - assert len(processed_phases) == 3 - assert processed_phases[0].id == 1 - assert processed_phases[0].name == 'Phase 1' - assert processed_phases[1].id == 2 - assert processed_phases[1].name == 'Phase 2' - assert processed_phases[2].id == 5 - assert processed_phases[2].name == 'Phase 5' - - def test_ensure_phase_ids_with_existing_ids(self): - """Test phase processing when IDs already exist.""" - phases = [ - {'id': 10, 'name': 'Custom Phase 1'}, - {'id': 'string-id', 'name': 'Custom Phase 2', 'dependsOn': [10]}, - ] - - processed_phases = ensure_legacy_phase_ids(phases) - - assert len(processed_phases) == 2 - assert processed_phases[0].id == 10 - assert processed_phases[1].id == 'string-id' - assert processed_phases[1].depends_on == [10] - - def test_ensure_task_ids_with_missing_ids(self): - """Test task ID generation using 20001+ pattern.""" - tasks = [ - {'name': 'Task 1', 'phase': 1, 'task': {'componentId': 'comp1'}}, - {'name': 'Task 2', 'phase': 2, 'task': {'componentId': 'comp2'}}, - {'id': 30000, 'name': 'Task 3', 'phase': 3, 'task': {'componentId': 'comp3'}}, - ] - - processed_tasks = ensure_legacy_task_ids(tasks) - - assert len(processed_tasks) == 3 - assert processed_tasks[0].id == 20001 - assert processed_tasks[1].id == 20002 - assert processed_tasks[2].id == 30000 - - def test_ensure_task_ids_adds_default_mode(self): - """Test that default mode 'run' is added to tasks.""" - tasks = [ - {'name': 'Task 1', 'phase': 1, 'task': {'componentId': 'comp1'}}, - {'name': 'Task 2', 'phase': 1, 'task': {'componentId': 'comp2', 'mode': 'debug'}}, - ] - - processed_tasks = ensure_legacy_task_ids(tasks) - - assert processed_tasks[0].task['mode'] == 'run' # Default added - assert processed_tasks[1].task['mode'] == 'debug' # Existing preserved - - def test_ensure_task_ids_validates_required_fields(self): - """Test validation of required task fields.""" - with pytest.raises(ValueError, match="missing 'task' configuration"): - ensure_legacy_task_ids([{'name': 'Bad Task', 'phase': 1}]) - - with pytest.raises(ValueError, match='missing componentId'): - ensure_legacy_task_ids([{'name': 'Bad Task', 'phase': 1, 'task': {}}]) - - def test_validate_flow_structure_success(self, sample_phases, sample_tasks): - """Test successful flow structure validation.""" - flow_configuration = get_flow_configuration(sample_phases, sample_tasks, ORCHESTRATOR_COMPONENT_ID) - validate_flow_structure(flow_configuration, flow_type=ORCHESTRATOR_COMPONENT_ID) - - def test_validate_flow_structure_invalid_phase_dependency(self): - """Test validation failure for invalid phase dependencies.""" - flow_configuration = get_flow_configuration( - phases=[{'id': 1, 'name': 'Phase 1', 'dependsOn': [999]}], tasks=[], flow_type=ORCHESTRATOR_COMPONENT_ID - ) - - with pytest.raises(ValueError, match='depends on non-existent phase 999'): - validate_flow_structure(flow_configuration, flow_type=ORCHESTRATOR_COMPONENT_ID) - - def test_validate_flow_structure_invalid_task_phase(self): - """Test validation failure for task referencing non-existent phase.""" - flow_configuration = get_flow_configuration( - phases=[{'id': 1, 'name': 'Phase 1'}], - tasks=[{'name': 'Bad Task', 'phase': 999, 'task': {'componentId': 'comp1'}}], - flow_type=ORCHESTRATOR_COMPONENT_ID, - ) - - with pytest.raises(ValueError, match='references non-existent phase 999'): - validate_flow_structure(flow_configuration, flow_type=ORCHESTRATOR_COMPONENT_ID) - - -# --- Test Circular Dependency Detection --- - - -class TestCircularDependencies: - """Test circular dependency detection.""" - - @pytest.mark.parametrize( - 'phases', - [ - pytest.param( - [ - {'id': 1, 'name': 'Phase 1'}, - {'id': 2, 'name': 'Phase 2', 'dependsOn': [1]}, - {'id': 3, 'name': 'Phase 3', 'dependsOn': [2]}, - ], - id='no_circular_dependencies', - ), - pytest.param( - [ - {'id': 1, 'name': 'Phase 1'}, - {'id': 2, 'name': 'Phase 2'}, - {'id': 3, 'name': 'Phase 3', 'dependsOn': [1, 2]}, - {'id': 4, 'name': 'Phase 4', 'dependsOn': [3]}, - {'id': 5, 'name': 'Phase 5', 'dependsOn': [1]}, - ], - id='complex_valid_dependencies', - ), - ], - ) - def test_no_circular_dependency_cases(self, phases: list[dict[str, Any]]): - """Test cases where no circular dependencies should be detected.""" - phases = ensure_legacy_phase_ids(phases) - ret = _check_legacy_circular_dependencies(phases) - assert ret is None - - @pytest.mark.parametrize( - 'phases', - [ - pytest.param( - [{'id': 1, 'name': 'Phase 1', 'dependsOn': [2]}, {'id': 2, 'name': 'Phase 2', 'dependsOn': [1]}], - id='direct_circular_dependency', - ), - pytest.param( - [ - {'id': 1, 'name': 'Phase 1', 'dependsOn': [3]}, - {'id': 2, 'name': 'Phase 2', 'dependsOn': [1]}, - {'id': 3, 'name': 'Phase 3', 'dependsOn': [2]}, - ], - id='indirect_circular_dependency', - ), - pytest.param([{'id': 1, 'name': 'Phase 1', 'dependsOn': [1]}], id='self_referencing_dependency'), - ], - ) - def test_circular_dependency_errors(self, phases: list[dict[str, Any]]): - """Test detection of direct, indirect, and self-referencing circular dependencies.""" - phases = ensure_legacy_phase_ids(phases) - - with pytest.raises(ValueError, match='Circular dependency detected'): - _check_legacy_circular_dependencies(phases) - - -# --- Test Edge Cases --- - - -class TestFlowEdgeCases: - """Test edge cases and error conditions.""" - - def test_phase_validation_with_missing_name(self): - """Test phase validation when required name field is missing.""" - invalid_phases = [{'name': 'Valid Phase'}, {}] - - processed_phases = ensure_legacy_phase_ids(invalid_phases) - assert len(processed_phases) == 2 - assert processed_phases[1].name == 'Phase 2' - - def test_task_validation_with_missing_name(self): - """Test task validation when required name field is missing.""" - invalid_tasks = [{}] - - with pytest.raises(ValueError, match="missing 'task' configuration"): - ensure_legacy_task_ids(invalid_tasks) - - def test_empty_flow_validation(self): - """Test validation of completely empty flow.""" - flow_configuration = get_flow_configuration([], [], ORCHESTRATOR_COMPONENT_ID) - ret = validate_flow_structure(flow_configuration, flow_type=ORCHESTRATOR_COMPONENT_ID) - assert ret is None - - -class TestFlowConfigurationBuilder: - """Test flow configuration builder helper.""" - - def test_get_flow_configuration_legacy_generates_ids_and_aliases(self): - """Legacy builder should sanitize IDs and serialize aliases.""" - flow_configuration = get_flow_configuration( - phases=[{'name': 'Generated Phase', 'depends_on': []}], - tasks=[{'name': 'Legacy Task', 'phase': 1, 'task': {'componentId': 'keboola.component'}}], - flow_type=ORCHESTRATOR_COMPONENT_ID, - ) - - phase = flow_configuration['phases'][0] - task = flow_configuration['tasks'][0] - - assert phase['id'] == 1 - assert 'dependsOn' in phase - assert 'depends_on' not in phase - assert task['id'] == 20001 - assert task['task']['mode'] == 'run' - assert 'continueOnFailure' in task - - def test_get_flow_configuration_conditional_excludes_unset_fields(self): - """Conditional builder should drop unset optional fields including single goto=null transitions.""" - flow_configuration = get_flow_configuration( - phases=[ - { - 'id': 'phase1', - 'name': 'Start', - 'next': [{'id': 'transition1', 'goto': None}], - } - ], - tasks=[_notification_task('task1', 'phase1')], - flow_type=CONDITIONAL_FLOW_COMPONENT_ID, - ) - - phase = flow_configuration['phases'][0] - task = flow_configuration['tasks'][0] - - assert 'description' not in phase - # Single transition with goto=None should be dropped entirely - assert 'next' not in phase - assert 'enabled' not in task - - -class TestConditionalFlowValidation: - """Test validation logic for conditional flows.""" - - @pytest.mark.parametrize( - 'phases', - [ - pytest.param( - [ - {'id': 'phase1', 'name': 'Start', 'next': [{'id': 't1', 'goto': 'phase2'}]}, - {'id': 'phase2', 'name': 'End', 'next': [{'id': 't2', 'goto': None}]}, - ], - id='simple-start-end', - ), - pytest.param( - [ - {'id': 'phase1', 'name': 'Phase 1', 'next': [{'id': 't1', 'goto': 'phase2'}]}, - { - 'id': 'phase2', - 'name': 'Phase 2', - 'next': [{'id': 't2', 'goto': 'phase3'}, {'id': 't3', 'goto': 'phase4'}], - }, - {'id': 'phase3', 'name': 'Phase 3', 'next': [{'id': 't4', 'goto': None}]}, - {'id': 'phase4', 'name': 'Phase 4', 'next': [{'id': 't5', 'goto': None}]}, - ], - id='complex-branched', - ), - ], - ) - def test_validate_conditional_flow_valid_cases(self, phases: list[dict[str, Any]]): - """Test valid conditional flow dependency structures (includes both simple and complex cases).""" - tasks = [_notification_task(f'task{i}', phase['id']) for i, phase in enumerate(phases)] - # Should not raise any errors - ret = validate_flow_structure({'phases': phases, 'tasks': tasks}, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - assert ret is None - - @pytest.mark.parametrize( - ('phases', 'task_specs', 'error_match'), - [ - pytest.param( - [ - {'id': 'phase1', 'name': 'Start', 'next': [{'id': 't1', 'goto': 'phase2'}]}, - {'id': 'phase1', 'name': 'Duplicate', 'next': [{'id': 't2', 'goto': None}]}, - ], - [('task1', 'phase1'), ('task2', 'phase1')], - 'duplicate phase IDs', - id='duplicate_phase_ids', - ), - pytest.param( - [{'id': 'phase1', 'name': 'Start', 'next': [{'id': 't1', 'goto': None}]}], - [('task1', 'phase1'), ('task1', 'phase1')], - 'duplicate task IDs', - id='duplicate_task_ids', - ), - pytest.param( - [{'id': 'phase1', 'name': 'Start', 'next': [{'id': 't1', 'goto': None}]}], - [('task1', 'missing-phase')], - 'references non-existent phase', - id='task_references_missing_phase', - ), - pytest.param( - [{'id': 'phase1', 'name': 'Start', 'next': [{'id': 't1', 'goto': 'ghost-phase'}]}], - [('task1', 'phase1')], - 'references non-existent phase', - id='transition_references_missing_phase', - ), - pytest.param( - [ - {'id': 'phase0', 'name': 'Start', 'next': [{'id': 't0', 'goto': 'phase1'}]}, - {'id': 'phase1', 'name': 'Loop', 'next': [{'id': 't1', 'goto': 'phase2'}]}, - {'id': 'phase2', 'name': 'Loop Again', 'next': [{'id': 't2', 'goto': 'phase1'}]}, - ], - [('task1', 'phase1'), ('task2', 'phase2')], - 'has no ending phases', - id='requires_ending_phase', - ), - pytest.param( - [ - {'id': 'phase1', 'name': 'One', 'next': [{'id': 't1', 'goto': 'phase2'}]}, - { - 'id': 'phase2', - 'name': 'Two', - 'next': [{'id': 't2', 'goto': 'phase1'}, {'id': 't3', 'goto': None}], - }, - ], - [('task1', 'phase1'), ('task2', 'phase2')], - 'has no entry phase', - id='requires_entry_phase', - ), - pytest.param( - [ - {'id': 'phase1', 'name': 'Entry A', 'next': [{'id': 't1', 'goto': None}]}, - {'id': 'phase2', 'name': 'Entry B', 'next': [{'id': 't2', 'goto': None}]}, - ], - [('task1', 'phase1'), ('task2', 'phase2')], - 'multiple entry phases', - id='single_entry_required', - ), - pytest.param( - [ - {'id': 'phase1', 'name': 'Start', 'next': [{'id': 't1', 'goto': 'phase2'}]}, - {'id': 'phase2', 'name': 'End', 'next': [{'id': 't2', 'goto': None}]}, - {'id': 'phase3', 'name': 'Isolated', 'next': [{'id': 't3', 'goto': 'phase4'}]}, - {'id': 'phase4', 'name': 'Isolated', 'next': [{'id': 't4', 'goto': 'phase3'}]}, - ], - [('task1', 'phase1'), ('task2', 'phase2'), ('task3', 'phase3')], - 'not reachable', - id='all_phases_reachable', - ), - pytest.param( - [ - {'id': 'phase0', 'name': 'Phase 0', 'next': [{'id': 't0', 'goto': 'phase1'}]}, - {'id': 'phase1', 'name': 'Phase 1', 'next': [{'id': 't1', 'goto': 'phase2'}]}, - { - 'id': 'phase2', - 'name': 'Phase 2', - 'next': [{'id': 't2', 'goto': 'phase1'}, {'id': 't3', 'goto': None}], - }, - ], - [('task1', 'phase1'), ('task2', 'phase2')], - 'Circular dependency detected', - id='circular_dependency', - ), - pytest.param( - [ - {'id': 'phase0', 'name': 'Phase 0', 'next': [{'id': 't0', 'goto': 'phase1'}]}, - {'id': 'phase1', 'name': 'Phase 1', 'next': [{'id': 't1', 'goto': 'phase2'}]}, - {'id': 'phase2', 'name': 'Phase 2', 'next': [{'id': 't2', 'goto': 'phase3'}]}, - { - 'id': 'phase3', - 'name': 'Phase 3', - 'next': [{'id': 't3', 'goto': 'phase1'}, {'id': 't4', 'goto': None}], - }, - ], - [('task1', 'phase1'), ('task2', 'phase2'), ('task3', 'phase3')], - 'Circular dependency detected', - id='indirect_circular_dependency', - ), - pytest.param( - [ - {'id': 'phase0', 'name': 'Phase 0', 'next': [{'id': 't0', 'goto': 'phase1'}]}, - { - 'id': 'phase1', - 'name': 'Phase 1', - 'next': [{'id': 't1', 'goto': 'phase1'}, {'id': 't2', 'goto': None}], - }, - ], - [('task1', 'phase1')], - 'Circular dependency detected', - id='self_referencing_dependency', - ), - ], - ) - def test_validate_conditional_flow_error_cases( - self, phases: list[dict[str, Any]], task_specs: list[tuple[str, str]], error_match: str - ): - """Parametrize conditional flow error cases to avoid repetitive tests.""" - tasks = [_notification_task(task_id, phase_id) for task_id, phase_id in task_specs] - - with pytest.raises(ValueError, match=error_match): - validate_flow_structure({'phases': phases, 'tasks': tasks}, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - - -class TestReachableIds: - """Test _reachable_ids function for finding reachable phases in a graph.""" - - @pytest.mark.parametrize( - ( - 'start_id', - 'edges', - 'initial_visited', - 'expected', - 'expected_visited', - ), - [ - pytest.param( - 'A', - {}, - set(), - {'A'}, - {'A'}, - id='empty_graph_single_node', - ), - pytest.param( - 'A', - {'A': set()}, - set(), - {'A'}, - {'A'}, - id='single_node_no_outgoing_edges', - ), - pytest.param( - 'A', - {'B': {'C'}, 'C': set()}, - set(), - {'A'}, - {'A'}, - id='start_node_not_in_edges', - ), - pytest.param( - 'A', - {'A': {'A'}}, - set(), - {'A'}, - {'A'}, - id='single_node_self_loop', - ), - pytest.param( - 'A', - {'A': {'B'}, 'B': {'C'}, 'C': set()}, - set(), - {'A', 'B', 'C'}, - {'A', 'B', 'C'}, - id='linear_chain', - ), - pytest.param( - 'A', - {'A': {'B', 'C'}, 'B': set(), 'C': set()}, - set(), - {'A', 'B', 'C'}, - {'A', 'B', 'C'}, - id='branching_structure', - ), - pytest.param( - 'A', - {'A': {'B'}, 'B': {'C'}, 'C': {'A'}}, - set(), - {'A', 'B', 'C'}, - {'A', 'B', 'C'}, - id='cycle_handling', - ), - pytest.param( - 'A', - {'A': {'B'}, 'B': set(), 'C': {'D'}, 'D': set()}, - set(), - {'A', 'B'}, - {'A', 'B'}, - id='disconnected_graph', - ), - pytest.param( - 'A', - { - 'A': {'B', 'C'}, - 'B': {'D'}, - 'C': {'D', 'E'}, - 'D': {'F'}, - 'E': {'F'}, - 'F': set(), - }, - set(), - {'A', 'B', 'C', 'D', 'E', 'F'}, - {'A', 'B', 'C', 'D', 'E', 'F'}, - id='complex_graph_with_branches_and_merges', - ), - pytest.param( - 'A', - {'A': {'B', 'C', 'D'}, 'B': set(), 'C': set(), 'D': set()}, - set(), - {'A', 'B', 'C', 'D'}, - {'A', 'B', 'C', 'D'}, - id='node_with_multiple_outgoing_edges', - ), - pytest.param( - 'A', - {'A': {'B'}, 'B': {'C'}, 'C': {'B', 'D'}, 'D': {'A'}}, - set(), - {'A', 'B', 'C', 'D'}, - {'A', 'B', 'C', 'D'}, - id='nested_cycles', - ), - pytest.param( - 'A', - {'A': {'B', 'C'}, 'B': {'D'}, 'C': {'D'}, 'D': set()}, - {'C', 'D'}, - {'A', 'B', 'C', 'D'}, - {'A', 'B', 'C', 'D'}, - id='partial_visited_set', - ), - pytest.param( - 'A', - {'A': {'B'}, 'B': {'A', 'C'}, 'C': set()}, - set('B'), - {'A', 'B'}, - {'A', 'B'}, - id='visited_nodes_are_not_revisited', - ), - ], - ) - def test_reachable_ids( - self, - start_id: str, - edges: dict[str, set[str]], - initial_visited: set[str], - expected: set[str], - expected_visited: set[str], - ): - """Parametrized coverage for _reachable_ids scenarios.""" - visited = set(initial_visited) - result = _reachable_ids(start_id, edges, visited) - - assert result == expected - assert visited == expected_visited - - -class TestResolveFlowSchema: - """Tests for resolve_flow_schema.""" - - @pytest.mark.asyncio - async def test_returns_live_schema_for_conditional(self, mocker, conditional_flow_schema): - client = mocker.Mock() - client.get_cached_flow_schema.return_value = None - component = mocker.Mock() - component.configuration_schema = conditional_flow_schema - mocker.patch( - 'keboola_mcp_server.tools.flow.utils.fetch_component', - mocker.AsyncMock(return_value=component), - ) - - result = await resolve_flow_schema(client, CONDITIONAL_FLOW_COMPONENT_ID) - - assert result == conditional_flow_schema - client.cache_flow_schema.assert_called_once_with(CONDITIONAL_FLOW_COMPONENT_ID, conditional_flow_schema) - - @pytest.mark.asyncio - async def test_uses_cache_and_does_not_refetch(self, mocker, conditional_flow_schema): - client = mocker.Mock() - client.get_cached_flow_schema.return_value = conditional_flow_schema - fetch = mocker.patch( - 'keboola_mcp_server.tools.flow.utils.fetch_component', - mocker.AsyncMock(), - ) - - result = await resolve_flow_schema(client, CONDITIONAL_FLOW_COMPONENT_ID) - - assert result == conditional_flow_schema - fetch.assert_not_called() - - @pytest.mark.asyncio - async def test_raises_on_empty_schema(self, mocker): - client = mocker.Mock() - client.get_cached_flow_schema.return_value = None - component = mocker.Mock() - component.configuration_schema = None - mocker.patch( - 'keboola_mcp_server.tools.flow.utils.fetch_component', - mocker.AsyncMock(return_value=component), - ) - - with pytest.raises(ValueError, match='Could not retrieve the conditional flow'): - await resolve_flow_schema(client, CONDITIONAL_FLOW_COMPONENT_ID) - - @pytest.mark.asyncio - @pytest.mark.parametrize( - 'error', - [ - # non-404 HTTP status error re-raised by fetch_component - HTTPStatusError('boom', request=Request('GET', 'https://ai.keboola.com'), response=Response(500)), - # transport/network error - ConnectError('connection refused', request=Request('GET', 'https://ai.keboola.com')), - # unexpected non-HTTP failure (e.g. a malformed AI Service payload failing model validation) - RuntimeError('unexpected AI Service payload'), - ], - ids=['http_status_error', 'network_error', 'unexpected_error'], - ) - async def test_raises_recoverable_error_on_fetch_failure(self, mocker, error): - client = mocker.Mock() - client.get_cached_flow_schema.return_value = None - mocker.patch( - 'keboola_mcp_server.tools.flow.utils.fetch_component', - mocker.AsyncMock(side_effect=error), - ) - - with pytest.raises(ValueError, match='Could not retrieve the conditional flow'): - await resolve_flow_schema(client, CONDITIONAL_FLOW_COMPONENT_ID) - - @pytest.mark.asyncio - async def test_returns_bundled_schema_for_legacy(self, mocker): - client = mocker.Mock() - fetch = mocker.patch( - 'keboola_mcp_server.tools.flow.utils.fetch_component', - mocker.AsyncMock(), - ) - - result = await resolve_flow_schema(client, ORCHESTRATOR_COMPONENT_ID) - - assert result['properties']['phases']['items']['properties'] # bundled legacy schema - assert 'dependsOn' in result['properties']['phases']['items']['properties'] - fetch.assert_not_called() - - -# --- Conditional flow variables (variableOverrides + JMESPath) round-trip --- - -# JMESPath expression over a prior job's result; not one of the legacy enumerated `value` paths. -_JMESPATH_VALUE = "sum(job.result.output.tables[].metrics[?name=='importedRowsCount'][].value)" - - -def _variables_flow() -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """A conditional flow that sets a flow variable from a JMESPath over a job result, consumes it via - `variableOverrides`, and branches on a JMESPath-driven condition (mirrors a real CF-variables flow).""" - phases = [ - { - 'id': 'extract', - 'name': 'Extract', - 'next': [ - { - 'id': 'cond', - 'name': 'Has rows', - 'condition': { - 'type': 'operator', - 'operator': 'GREATER_THAN', - 'operands': [ - {'type': 'task', 'task': 'extract_task', 'value': _JMESPATH_VALUE}, - {'type': 'const', 'value': 0}, - ], - }, - 'goto': 'transform', - }, - {'id': 'cond_default', 'goto': 'transform'}, - ], - }, - {'id': 'transform', 'name': 'Transform', 'next': [{'id': 'end', 'goto': None}]}, - ] - tasks = [ - { - 'id': 'extract_task', - 'name': 'Extract', - 'phase': 'extract', - 'task': {'type': 'job', 'mode': 'run', 'componentId': 'keboola.ex-google-drive', 'configId': '123'}, - }, - { - 'id': 'set_var', - 'name': 'importedRowsSum', - 'phase': 'transform', - 'task': { - 'type': 'variable', - 'name': 'importedRowsSum', - 'source': {'type': 'task', 'task': 'extract_task', 'value': _JMESPATH_VALUE}, - }, - }, - { - 'id': 'use_var', - 'name': 'Transform', - 'phase': 'transform', - 'task': { - 'type': 'job', - 'mode': 'run', - 'componentId': 'keboola.snowflake-transformation', - 'configId': 'abc', - 'variableOverrides': ['importedRowsSum'], - }, - }, - ] - return phases, tasks - - -class TestConditionalFlowVariablesRoundTrip: - """get_flow_configuration must faithfully carry the CF-variables fields the live schema allows.""" - - def test_variable_overrides_preserved_on_job_task(self): - phases, tasks = _variables_flow() - cfg = get_flow_configuration(phases=phases, tasks=tasks, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - job_task = next(t for t in cfg['tasks'] if t['id'] == 'use_var')['task'] - assert job_task['variableOverrides'] == ['importedRowsSum'] - - def test_jmespath_value_preserved_in_variable_source(self): - phases, tasks = _variables_flow() - cfg = get_flow_configuration(phases=phases, tasks=tasks, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - var_task = next(t for t in cfg['tasks'] if t['id'] == 'set_var')['task'] - assert var_task['source']['value'] == _JMESPATH_VALUE - - def test_jmespath_value_preserved_in_phase_condition(self): - phases, tasks = _variables_flow() - cfg = get_flow_configuration(phases=phases, tasks=tasks, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - condition = cfg['phases'][0]['next'][0]['condition'] - assert condition['operands'][0]['value'] == _JMESPATH_VALUE - - def test_validate_flow_structure_accepts_variables_flow(self): - phases, tasks = _variables_flow() - cfg = get_flow_configuration(phases=phases, tasks=tasks, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - # Should not raise. - validate_flow_structure(cfg, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - - def test_unknown_future_job_field_is_preserved(self): - """extra='allow' keeps fields the live schema may add instead of silently dropping them.""" - phases, tasks = _variables_flow() - tasks[0]['task']['someFutureField'] = {'k': 'v'} - cfg = get_flow_configuration(phases=phases, tasks=tasks, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - extract_task = next(t for t in cfg['tasks'] if t['id'] == 'extract_task')['task'] - assert extract_task['someFutureField'] == {'k': 'v'} - - def test_unknown_future_condition_field_is_preserved(self): - """extra='allow' applies to condition nodes too (not just task configs) via BaseExtraModel.""" - phases, tasks = _variables_flow() - phases[0]['next'][0]['condition']['someFutureField'] = 'keep-me' - cfg = get_flow_configuration(phases=phases, tasks=tasks, flow_type=CONDITIONAL_FLOW_COMPONENT_ID) - condition = cfg['phases'][0]['next'][0]['condition'] - assert condition['someFutureField'] == 'keep-me' - - def test_read_path_preserves_variables_via_from_api_response(self): - """The READ/display path (Flow.from_api_response, used by get_flows) must also carry the - variables fields — it routes through the same conditional-flow models as the write path.""" - phases, tasks = _variables_flow() - api_config = APIFlowResponse.model_validate( - { - 'id': 'flow-1', - 'name': 'Variables flow', - 'version': 1, - 'configuration': {'phases': phases, 'tasks': tasks}, - } - ) - - flow = Flow.from_api_response(api_config=api_config, flow_component_id=CONDITIONAL_FLOW_COMPONENT_ID) - - job_task = next(t for t in flow.configuration.tasks if t.id == 'use_var') - assert job_task.task.variable_overrides == ['importedRowsSum'] - var_task = next(t for t in flow.configuration.tasks if t.id == 'set_var') - assert var_task.task.source.value == _JMESPATH_VALUE - assert flow.configuration.phases[0].next[0].condition.operands[0].value == _JMESPATH_VALUE diff --git a/tests/tools/semantic/test_service.py b/tests/tools/semantic/test_service.py deleted file mode 100644 index 7882df70a..000000000 --- a/tests/tools/semantic/test_service.py +++ /dev/null @@ -1,1093 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping, Sequence - -import pytest - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.metastore import MetastoreObject -from keboola_mcp_server.tools.semantic.model import SemanticObjectType -from keboola_mcp_server.tools.semantic.service import ( - SemanticServiceDataTypeGroup, - SemanticValidationServiceOutput, - _constraint_is_relevant, - _extract_join_columns, - _extract_metric_column, - _matches_sql, - _to_semantic_service_data, - detect_used_objects_from_context, - evaluate_constraints_from_context, - search_semantic_context, - validate_semantic_query_with_used_objects, -) - - -def _metastore_object( - object_type: SemanticObjectType, - object_id: str, - *, - name: str, - attributes: Mapping[str, object] | None = None, -) -> MetastoreObject: - return MetastoreObject.model_validate( - { - 'type': object_type.value, - 'id': object_id, - 'attributes': dict(attributes or {}), - 'meta': {'name': name}, - } - ) - - -def _group_objects( - result: SemanticValidationServiceOutput, -) -> dict[SemanticObjectType, SemanticServiceDataTypeGroup]: - return {group.object_type: group for group in result.used_object_groups} - - -def _service_group( - object_type: SemanticObjectType, - objects: Sequence[MetastoreObject], -) -> SemanticServiceDataTypeGroup: - return SemanticServiceDataTypeGroup( - object_type=object_type, - objects=[_to_semantic_service_data(object_type, item) for item in objects], - ) - - -def _detect_context( - *, - datasets: Sequence[MetastoreObject] = (), - metrics: Sequence[MetastoreObject] = (), - relationships: Sequence[MetastoreObject] = (), -) -> dict[SemanticObjectType, SemanticServiceDataTypeGroup]: - context_by_type: dict[SemanticObjectType, SemanticServiceDataTypeGroup] = {} - if datasets: - context_by_type[SemanticObjectType.SEMANTIC_DATASET] = _service_group( - SemanticObjectType.SEMANTIC_DATASET, - datasets, - ) - if metrics: - context_by_type[SemanticObjectType.SEMANTIC_METRIC] = _service_group( - SemanticObjectType.SEMANTIC_METRIC, - metrics, - ) - if relationships: - context_by_type[SemanticObjectType.SEMANTIC_RELATIONSHIP] = _service_group( - SemanticObjectType.SEMANTIC_RELATIONSHIP, - relationships, - ) - return context_by_type - - -def _build_metastore_objects( - object_type: SemanticObjectType, - specs: Sequence[tuple[str, str, Mapping[str, object]]], -) -> list[MetastoreObject]: - return [ - _metastore_object( - object_type, - object_id, - name=name, - attributes=attributes, - ) - for object_id, name, attributes in specs - ] - - -def _evaluate_context( - *, - model_specs: Sequence[tuple[str, str, Mapping[str, object]]] = (), - constraint_specs: Sequence[tuple[str, str, Mapping[str, object]]] = (), -) -> dict[SemanticObjectType, SemanticServiceDataTypeGroup]: - context_by_type: dict[SemanticObjectType, SemanticServiceDataTypeGroup] = {} - if model_specs: - context_by_type[SemanticObjectType.SEMANTIC_MODEL] = _service_group( - SemanticObjectType.SEMANTIC_MODEL, - _build_metastore_objects(SemanticObjectType.SEMANTIC_MODEL, model_specs), - ) - if constraint_specs: - context_by_type[SemanticObjectType.SEMANTIC_CONSTRAINT] = _service_group( - SemanticObjectType.SEMANTIC_CONSTRAINT, - _build_metastore_objects(SemanticObjectType.SEMANTIC_CONSTRAINT, constraint_specs), - ) - return context_by_type - - -def _used_object_groups( - *, - dataset_specs: Sequence[tuple[str, str, Mapping[str, object]]] = (), - metric_specs: Sequence[tuple[str, str, Mapping[str, object]]] = (), - relationship_specs: Sequence[tuple[str, str, Mapping[str, object]]] = (), -) -> dict[SemanticObjectType, SemanticServiceDataTypeGroup]: - used_groups: dict[SemanticObjectType, SemanticServiceDataTypeGroup] = {} - if dataset_specs: - used_groups[SemanticObjectType.SEMANTIC_DATASET] = _service_group( - SemanticObjectType.SEMANTIC_DATASET, - _build_metastore_objects(SemanticObjectType.SEMANTIC_DATASET, dataset_specs), - ) - if metric_specs: - used_groups[SemanticObjectType.SEMANTIC_METRIC] = _service_group( - SemanticObjectType.SEMANTIC_METRIC, - _build_metastore_objects(SemanticObjectType.SEMANTIC_METRIC, metric_specs), - ) - if relationship_specs: - used_groups[SemanticObjectType.SEMANTIC_RELATIONSHIP] = _service_group( - SemanticObjectType.SEMANTIC_RELATIONSHIP, - _build_metastore_objects(SemanticObjectType.SEMANTIC_RELATIONSHIP, relationship_specs), - ) - return used_groups - - -@pytest.fixture -def semantic_api_objects() -> dict[SemanticObjectType, list[MetastoreObject]]: - model_id = 'model-1' - orders_table_id = 'in.c-main.orders' - customers_table_id = 'in.c-main.customers' - - return { - SemanticObjectType.SEMANTIC_MODEL: [ - _metastore_object( - SemanticObjectType.SEMANTIC_MODEL, - model_id, - name='Revenue Semantic Model', - attributes={ - 'name': 'Revenue Semantic Model', - 'description': 'Semantic model for revenue analytics', - 'sql_dialect': 'snowflake', - }, - ) - ], - SemanticObjectType.SEMANTIC_DATASET: [ - _metastore_object( - SemanticObjectType.SEMANTIC_DATASET, - 'dataset-orders', - name='Orders', - attributes={ - 'name': 'Orders', - 'tableId': orders_table_id, - 'fqn': 'analytics.orders', - 'description': 'Fact table with order level data', - 'modelUUID': model_id, - }, - ), - _metastore_object( - SemanticObjectType.SEMANTIC_DATASET, - 'dataset-customers', - name='Customers', - attributes={ - 'name': 'Customers', - 'tableId': customers_table_id, - 'fqn': 'analytics.customers', - 'description': 'Customer dimension', - 'modelUUID': model_id, - }, - ), - ], - SemanticObjectType.SEMANTIC_METRIC: [ - _metastore_object( - SemanticObjectType.SEMANTIC_METRIC, - 'metric-revenue', - name='Revenue', - attributes={ - 'name': 'Revenue', - 'sql': 'SUM(order_amount)', - 'dataset': orders_table_id, - 'description': 'Total revenue', - 'modelUUID': model_id, - }, - ), - _metastore_object( - SemanticObjectType.SEMANTIC_METRIC, - 'metric-order-count', - name='Order Count', - attributes={ - 'name': 'Order Count', - 'sql': 'COUNT(*)', - 'dataset': orders_table_id, - 'description': 'Count of orders', - 'modelUUID': model_id, - }, - ), - ], - SemanticObjectType.SEMANTIC_RELATIONSHIP: [ - _metastore_object( - SemanticObjectType.SEMANTIC_RELATIONSHIP, - 'relationship-orders-customers', - name='Orders to Customers', - attributes={ - 'name': 'Orders to Customers', - 'from': orders_table_id, - 'to': customers_table_id, - 'type': 'many_to_one', - 'on': 'orders.customer_id = customers.id', - 'modelUUID': model_id, - }, - ) - ], - SemanticObjectType.SEMANTIC_CONSTRAINT: [ - _metastore_object( - SemanticObjectType.SEMANTIC_CONSTRAINT, - 'constraint-composition', - name='Revenue requires order count', - attributes={ - 'name': 'Revenue requires order count', - 'constraintType': 'composition', - 'severity': 'warning', - 'metrics': ['Revenue', 'Order Count'], - 'modelUUID': model_id, - }, - ), - _metastore_object( - SemanticObjectType.SEMANTIC_CONSTRAINT, - 'constraint-exclusion', - name='Orders and Customers combination', - attributes={ - 'name': 'Orders and Customers combination', - 'constraintType': 'exclusion', - 'severity': 'error', - 'datasets': [orders_table_id, customers_table_id], - 'modelUUID': model_id, - }, - ), - _metastore_object( - SemanticObjectType.SEMANTIC_CONSTRAINT, - 'constraint-pre-query', - name='Revenue freshness', - attributes={ - 'name': 'Revenue freshness', - 'constraintType': 'conditional', - 'severity': 'warning', - 'datasets': [orders_table_id], - 'modelUUID': model_id, - 'errorMessage': 'Revenue must be checked against fresh source data.', - 'remediation': 'Compare the report with the operational source before sharing it.', - 'ai': {'preQueryCheck': True}, - 'validationQuery': {'default': 'SELECT 1'}, - }, - ), - _metastore_object( - SemanticObjectType.SEMANTIC_CONSTRAINT, - 'constraint-post-query', - name='Revenue threshold', - attributes={ - 'name': 'Revenue threshold', - 'constraintType': 'inequality', - 'severity': 'warning', - 'metrics': ['Revenue'], - 'modelUUID': model_id, - 'validationQuery': {'snowflake': 'SELECT * FROM revenue_threshold_check'}, - }, - ), - ], - SemanticObjectType.SEMANTIC_GLOSSARY: [ - _metastore_object( - SemanticObjectType.SEMANTIC_GLOSSARY, - 'glossary-revenue', - name='Revenue glossary', - attributes={ - 'term': 'Revenue', - 'definition': 'Revenue recognized from completed orders', - 'modelUUID': model_id, - }, - ) - ], - } - - -@pytest.fixture -def mock_semantic_api( - keboola_client: KeboolaClient, - semantic_api_objects: dict[SemanticObjectType, list[MetastoreObject]], -) -> dict[SemanticObjectType, list[MetastoreObject]]: - async def list_objects_side_effect( - object_type: SemanticObjectType | str, - *, - limit: int | None = None, - offset: int | None = None, - **_: object, - ) -> list[MetastoreObject]: - semantic_type = object_type if isinstance(object_type, SemanticObjectType) else SemanticObjectType(object_type) - items = semantic_api_objects.get(semantic_type, []) - start = offset or 0 - if limit is None: - return items[start:] - return items[start : start + limit] - - keboola_client.metastore_client.list_objects.side_effect = list_objects_side_effect - return semantic_api_objects - - -@pytest.mark.asyncio -async def test_validate_semantic_query_detects_used_objects_and_relevant_validations( - keboola_client: KeboolaClient, - mock_semantic_api: dict[SemanticObjectType, list[MetastoreObject]], -) -> None: - result = await validate_semantic_query_with_used_objects( - keboola_client, - ( - 'SELECT SUM(order_amount) AS revenue ' - 'FROM analytics.orders orders ' - 'JOIN analytics.customers customers ON orders.customer_id = customers.id' - ), - ['model-1'], - ) - - assert result.valid is False - assert result.matched_relationships == ['Orders to Customers'] - - groups = _group_objects(result) - assert [dataset.id for dataset in groups[SemanticObjectType.SEMANTIC_DATASET].objects] == [ - 'dataset-orders', - 'dataset-customers', - ] - assert [metric.id for metric in groups[SemanticObjectType.SEMANTIC_METRIC].objects] == ['metric-revenue'] - assert [relationship.id for relationship in groups[SemanticObjectType.SEMANTIC_RELATIONSHIP].objects] == [ - 'relationship-orders-customers' - ] - - findings_by_id = {finding.constraint_id: finding for finding in result.violations + result.post_execution_checks} - - composition_finding = findings_by_id['constraint-composition'] - assert composition_finding.status == 'missing_metrics' - assert 'Order Count' in composition_finding.message - - exclusion_finding = findings_by_id['constraint-exclusion'] - assert exclusion_finding.status == 'excluded_combination' - assert exclusion_finding.severity == 'error' - - pre_query_finding = findings_by_id['constraint-pre-query'] - assert pre_query_finding.status == 'pre_query_check' - assert pre_query_finding.validation_query == 'SELECT 1' - assert 'Revenue must be checked against fresh source data.' in pre_query_finding.message - assert 'Compare the report with the operational source before sharing it.' in pre_query_finding.message - - post_query_finding = findings_by_id['constraint-post-query'] - assert post_query_finding.status == 'post_query_check' - assert post_query_finding.validation_query == 'SELECT * FROM revenue_threshold_check' - - -@pytest.mark.asyncio -async def test_validate_semantic_query_with_used_objects_uses_only_provided_scope( - keboola_client: KeboolaClient, - mock_semantic_api: dict[SemanticObjectType, list[MetastoreObject]], -) -> None: - result = await validate_semantic_query_with_used_objects( - keboola_client, - 'SELECT * FROM analytics.orders', - ['model-1'], - used_object_groups=[ - _service_group( - SemanticObjectType.SEMANTIC_DATASET, [mock_semantic_api[SemanticObjectType.SEMANTIC_DATASET][0]] - ), - _service_group( - SemanticObjectType.SEMANTIC_METRIC, [mock_semantic_api[SemanticObjectType.SEMANTIC_METRIC][0]] - ), - ], - ) - - assert result.valid is True - assert result.matched_relationships == [] - - groups = _group_objects(result) - assert [dataset.id for dataset in groups[SemanticObjectType.SEMANTIC_DATASET].objects] == ['dataset-orders'] - assert [metric.id for metric in groups[SemanticObjectType.SEMANTIC_METRIC].objects] == ['metric-revenue'] - assert SemanticObjectType.SEMANTIC_RELATIONSHIP not in groups - - findings_by_id = {finding.constraint_id: finding for finding in result.violations + result.post_execution_checks} - assert findings_by_id['constraint-pre-query'].status == 'pre_query_check' - assert findings_by_id['constraint-post-query'].status == 'post_query_check' - assert 'constraint-exclusion' not in findings_by_id - - -@pytest.mark.parametrize( - ('sql_query', 'candidate', 'expected'), - [ - ('SELECT revenue FROM analytics.orders', 'revenue', True), - ('SELECT Revenue FROM analytics.orders', 'revenue', True), - ('SELECT order_count FROM analytics.orders', 'order', False), - ('SELECT preorders FROM analytics.orders', 'order', False), - ('SELECT customer_id FROM analytics.orders', 'customer_id', True), - ('SELECT customer_id_2 FROM analytics.orders', 'customer_id', False), - ('SELECT analytics.orders.id FROM analytics.orders', 'analytics.orders', True), - ('SELECT analytics.orders_backup.id FROM analytics.orders_backup', 'analytics.orders', True), - ('SELECT SUM(order_amount) FROM analytics.orders', 'SUM(order_amount)', True), - ('SELECT SUM(other_amount) FROM analytics.orders', 'SUM(order_amount)', False), - ('SELECT * FROM analytics.orders', '', False), - ('SELECT * FROM analytics.orders', ' ', False), - ], -) -def test_matches_sql_handles_identifiers_and_substrings( - sql_query: str, - candidate: str, - expected: bool, -) -> None: - assert _matches_sql(sql_query, candidate) is expected - - -@pytest.mark.parametrize( - ('sql', 'expected'), - [ - # Simple double-quoted column. - ('SUM("REVENUE_YTD")', 'REVENUE_YTD'), - # Unquoted column. - ('AVG(margin_pct)', 'margin_pct'), - # SUM without quotes, uppercase. - ('SUM(AMOUNT)', 'AMOUNT'), - # COUNT(*) has no column name. - ('COUNT(*)', None), - # Complex expression — no match. - ('SUM(CASE WHEN x = 1 THEN amount ELSE 0 END)', None), - # Multi-argument — no match. - ('COALESCE(a, b)', None), - ], -) -def test_extract_metric_column(sql: str, expected: str | None) -> None: - assert _extract_metric_column(sql) == expected - - -@pytest.mark.parametrize( - ('on_clause', 'expected'), - [ - # Standard Snowflake-style uppercase ON clause with alias prefixes. - ( - 'fact.FK_BUSINESS_SUBUNIT = dim.PK_BUSINESS_SUBUNIT', - ['FK_BUSINESS_SUBUNIT', 'PK_BUSINESS_SUBUNIT'], - ), - # Multi-condition clause. - ( - 'fact.FK_BUSINESS_SUBUNIT = dim.FK_BUSINESS_SUBUNIT AND fact.CODE_FIN_STAT = dim.CODE_FIN_STAT', - ['FK_BUSINESS_SUBUNIT', 'CODE_FIN_STAT'], - ), - # Function call + string literal — LEFT and AVG (keyword) are filtered; string literal stripped. - ( - 'fact.DIM_CURRENCY = dim.CURRENCY_FROM AND LEFT(dim.CODE_PERIOD_VALUE, 6) = fact.CODE_PERIOD_VALUE ' - "AND dim.RATE_TYPE = 'AVG'", - ['DIM_CURRENCY', 'CURRENCY_FROM', 'CODE_PERIOD_VALUE', 'RATE_TYPE'], - ), - # All-lowercase on-clause returns empty list (triggers full-string fallback). - ('orders.customer_id = customers.id', []), - # Short tokens (< 3 chars after first letter) are excluded. - ('fact.FK = dim.PK', []), - ], -) -def test_extract_join_columns(on_clause: str, expected: list[str]) -> None: - assert _extract_join_columns(on_clause) == expected - - -@pytest.mark.parametrize( - ('constraint_attributes', 'used_metric_names', 'used_dataset_ids', 'expected'), - [ - ({'metrics': ['Revenue']}, {'Revenue'}, set(), True), - ({'metrics': ['Revenue']}, {'Order Count'}, set(), False), - ({'datasets': ['in.c-main.orders']}, set(), {'in.c-main.orders'}, True), - ({'datasets': ['in.c-main.orders']}, set(), {'in.c-main.customers'}, False), - ({'metrics': [' Revenue ', ' ']}, {'Revenue'}, set(), True), - ({'datasets': [' in.c-main.orders ', ' ']}, set(), {'in.c-main.orders'}, True), - ({'metrics': ['Revenue'], 'datasets': ['in.c-main.orders']}, set(), {'in.c-main.orders'}, True), - ({'metrics': ['Revenue'], 'datasets': ['in.c-main.orders']}, {'Revenue'}, set(), True), - ({'metrics': ['Revenue'], 'datasets': ['in.c-main.orders']}, {'Other'}, {'other'}, False), - ({}, set(), set(), True), - ({'metrics': [' '], 'datasets': [' ']}, set(), set(), True), - ], -) -def test_constraint_is_relevant_edge_cases( - constraint_attributes: Mapping[str, object], - used_metric_names: set[str], - used_dataset_ids: set[str], - expected: bool, -) -> None: - constraint = _service_group( - SemanticObjectType.SEMANTIC_CONSTRAINT, - [ - _metastore_object( - SemanticObjectType.SEMANTIC_CONSTRAINT, - 'constraint-test', - name='Constraint Test', - attributes={ - 'name': 'Constraint Test', - 'modelUUID': 'model-1', - **constraint_attributes, - }, - ) - ], - ).objects[0] - - assert _constraint_is_relevant(constraint, used_metric_names, used_dataset_ids) is expected - - -@pytest.mark.parametrize( - ('sql_query', 'dataset_specs', 'metric_specs', 'relationship_specs', 'expected_group_ids'), - [ - # Nothing from the semantic context should be detected when the SQL does not reference any object. - ( - 'SELECT 1', - [ - ( - 'dataset-orders', - 'Orders', - { - 'name': 'Orders', - 'tableId': 'in.c-main.orders', - 'fqn': 'analytics.orders', - 'modelUUID': 'model-1', - }, - ) - ], - [ - ( - 'metric-revenue', - 'Revenue', - { - 'name': 'Revenue', - 'sql': 'SUM(order_amount)', - 'dataset': 'in.c-main.orders', - 'modelUUID': 'model-1', - }, - ) - ], - [ - ( - 'relationship-orders-customers', - 'Orders to Customers', - { - 'name': 'Orders to Customers', - 'from': 'in.c-main.orders', - 'to': 'in.c-main.customers', - 'on': 'orders.customer_id = customers.id', - 'modelUUID': 'model-1', - }, - ) - ], - {}, - ), - # Metrics are only considered after their source dataset was detected, so this metric must be skipped. - ( - 'SELECT SUM(order_amount) FROM analytics.orders', - [ - ( - 'dataset-orders', - 'Orders', - { - 'name': 'Orders', - 'tableId': 'in.c-main.orders', - 'fqn': 'analytics.orders', - 'modelUUID': 'model-1', - }, - ) - ], - [ - ( - 'metric-revenue', - 'Revenue', - { - 'name': 'Revenue', - 'sql': 'SUM(order_amount)', - 'dataset': 'in.c-main.other', - 'modelUUID': 'model-1', - }, - ) - ], - [], - { - SemanticObjectType.SEMANTIC_DATASET: ['dataset-orders'], - }, - ), - # Both datasets are present, but the join predicate differs from the relationship definition, so no - # relationship should be reported. - ( - ( - 'SELECT * FROM analytics.orders orders ' - 'JOIN analytics.customers customers ON orders.account_id = customers.id' - ), - [ - ( - 'dataset-orders', - 'Orders', - { - 'name': 'Orders', - 'tableId': 'in.c-main.orders', - 'fqn': 'analytics.orders', - 'modelUUID': 'model-1', - }, - ), - ( - 'dataset-customers', - 'Customers', - { - 'name': 'Customers', - 'tableId': 'in.c-main.customers', - 'fqn': 'analytics.customers', - 'modelUUID': 'model-1', - }, - ), - ], - [], - [ - ( - 'relationship-orders-customers', - 'Orders to Customers', - { - 'name': 'Orders to Customers', - 'from': 'in.c-main.orders', - 'to': 'in.c-main.customers', - 'on': 'orders.customer_id = customers.id', - 'modelUUID': 'model-1', - }, - ) - ], - { - SemanticObjectType.SEMANTIC_DATASET: ['dataset-orders', 'dataset-customers'], - }, - ), - # Metric SQL uses a quoted column (SUM("REVENUE_YTD")); the query writes the column with a - # table-alias prefix (ep."REVENUE_YTD"). The old full-string match failed; the new - # column-extraction path should detect the metric. - ( - 'SELECT SUM(ep."REVENUE_YTD") FROM "DB"."schema"."FACT_PERFORMANCE" ep', - [ - ( - 'dataset-fact', - 'Fact Performance', - { - 'name': 'Fact Performance', - 'tableId': 'out.c-main.FACT_PERFORMANCE', - 'fqn': '"DB"."schema"."FACT_PERFORMANCE"', - 'modelUUID': 'model-1', - }, - ) - ], - [ - ( - 'metric-revenue-ytd', - 'Revenue YTD', - { - 'name': 'Revenue YTD', - 'sql': 'SUM("REVENUE_YTD")', - 'dataset': 'out.c-main.FACT_PERFORMANCE', - 'modelUUID': 'model-1', - }, - ) - ], - [], - { - SemanticObjectType.SEMANTIC_DATASET: ['dataset-fact'], - SemanticObjectType.SEMANTIC_METRIC: ['metric-revenue-ytd'], - }, - ), - # Relationship ON clause uses uppercase Snowflake-style column names with template aliases - # (fact./dim.); the SQL uses different aliases (o./c.). Column-extraction should detect - # the relationship because all column names are present in the SQL. - ( - ( - 'SELECT * FROM "DB"."s"."ORDERS" o ' - 'JOIN "DB"."s"."CUSTOMERS" c ON o."FK_CUSTOMER_ID" = c."PK_CUSTOMER_ID"' - ), - [ - ( - 'dataset-orders', - 'Orders', - { - 'name': 'Orders', - 'tableId': 'out.c-main.ORDERS', - 'fqn': '"DB"."s"."ORDERS"', - 'modelUUID': 'model-1', - }, - ), - ( - 'dataset-customers', - 'Customers', - { - 'name': 'Customers', - 'tableId': 'out.c-main.CUSTOMERS', - 'fqn': '"DB"."s"."CUSTOMERS"', - 'modelUUID': 'model-1', - }, - ), - ], - [], - [ - ( - 'rel-orders-customers', - 'Orders to Customers', - { - 'name': 'Orders to Customers', - 'from': 'out.c-main.ORDERS', - 'to': 'out.c-main.CUSTOMERS', - 'on': 'fact.FK_CUSTOMER_ID = dim.PK_CUSTOMER_ID', - 'modelUUID': 'model-1', - }, - ) - ], - { - SemanticObjectType.SEMANTIC_DATASET: ['dataset-orders', 'dataset-customers'], - SemanticObjectType.SEMANTIC_RELATIONSHIP: ['rel-orders-customers'], - }, - ), - # Relationship with uppercase ON clause columns — but the actual SQL uses DIFFERENT columns - # in the join (FK_ORDER_ID instead of FK_CUSTOMER_ID). Should NOT detect the relationship. - ( - ('SELECT * FROM "DB"."s"."ORDERS" o ' 'JOIN "DB"."s"."CUSTOMERS" c ON o."FK_ORDER_ID" = c."PK_ORDER_ID"'), - [ - ( - 'dataset-orders', - 'Orders', - { - 'name': 'Orders', - 'tableId': 'out.c-main.ORDERS', - 'fqn': '"DB"."s"."ORDERS"', - 'modelUUID': 'model-1', - }, - ), - ( - 'dataset-customers', - 'Customers', - { - 'name': 'Customers', - 'tableId': 'out.c-main.CUSTOMERS', - 'fqn': '"DB"."s"."CUSTOMERS"', - 'modelUUID': 'model-1', - }, - ), - ], - [], - [ - ( - 'rel-orders-customers', - 'Orders to Customers', - { - 'name': 'Orders to Customers', - 'from': 'out.c-main.ORDERS', - 'to': 'out.c-main.CUSTOMERS', - 'on': 'fact.FK_CUSTOMER_ID = dim.PK_CUSTOMER_ID', - 'modelUUID': 'model-1', - }, - ) - ], - { - SemanticObjectType.SEMANTIC_DATASET: ['dataset-orders', 'dataset-customers'], - }, - ), - ], -) -def test_detect_used_objects_from_context_edge_cases( - sql_query: str, - dataset_specs: Sequence[tuple[str, str, Mapping[str, object]]], - metric_specs: Sequence[tuple[str, str, Mapping[str, object]]], - relationship_specs: Sequence[tuple[str, str, Mapping[str, object]]], - expected_group_ids: dict[SemanticObjectType, list[str]], -) -> None: - context_by_type = _detect_context( - datasets=_build_metastore_objects(SemanticObjectType.SEMANTIC_DATASET, dataset_specs), - metrics=_build_metastore_objects(SemanticObjectType.SEMANTIC_METRIC, metric_specs), - relationships=_build_metastore_objects(SemanticObjectType.SEMANTIC_RELATIONSHIP, relationship_specs), - ) - result = detect_used_objects_from_context(sql_query, context_by_type) - - assert { - object_type: [item.id for item in group.objects] for object_type, group in result.items() - } == expected_group_ids - - -@pytest.mark.parametrize( - ( - 'model_specs', - 'constraint_specs', - 'used_dataset_specs', - 'used_metric_specs', - 'used_relationship_specs', - 'expected_valid', - 'expected_violation_statuses', - 'expected_post_check_statuses', - 'expected_matched_relationships', - 'expected_post_check_queries', - 'expected_post_check_severities', - ), - [ - # No constraints means the output should stay valid and contain no findings. - ( - [('model-1', 'Model', {'name': 'Model', 'sql_dialect': 'snowflake'})], - [], - [ - ( - 'dataset-orders', - 'Orders', - { - 'name': 'Orders', - 'tableId': 'in.c-main.orders', - 'fqn': 'analytics.orders', - 'modelUUID': 'model-1', - }, - ) - ], - [], - [], - True, - [], - [], - [], - [], - [], - ), - # The constraint references a different metric than the one used by the query, so it is irrelevant. - ( - [('model-1', 'Model', {})], - [ - ( - 'constraint-irrelevant', - 'Irrelevant Constraint', - { - 'name': 'Irrelevant Constraint', - 'constraintType': 'inequality', - 'metrics': ['Revenue'], - 'modelUUID': 'model-1', - }, - ) - ], - [], - [ - ( - 'metric-orders', - 'Orders', - {'name': 'Orders', 'dataset': 'in.c-main.orders', 'modelUUID': 'model-1'}, - ) - ], - [], - True, - [], - [], - [], - [], - [], - ), - # Unknown constraint types without a validation query are ignored even when they match the used dataset. - ( - [('model-1', 'Model', {})], - [ - ( - 'constraint-unknown', - 'Unknown Constraint', - { - 'name': 'Unknown Constraint', - 'constraintType': 'custom', - 'datasets': ['in.c-main.orders'], - 'modelUUID': 'model-1', - }, - ) - ], - [ - ( - 'dataset-orders', - 'Orders', - {'name': 'Orders', 'tableId': 'in.c-main.orders', 'modelUUID': 'model-1'}, - ) - ], - [], - [], - True, - [], - [], - [], - [], - [], - ), - # A pre-query check with error severity should make the validation fail immediately. - ( - [('model-1', 'Model', {})], - [ - ( - 'constraint-pre-query-error', - 'Pre Query Error', - { - 'name': 'Pre Query Error', - 'constraintType': 'conditional', - 'severity': 'error', - 'datasets': ['in.c-main.orders'], - 'ai': {'preQueryCheck': True}, - 'modelUUID': 'model-1', - }, - ) - ], - [ - ( - 'dataset-orders', - 'Orders', - {'name': 'Orders', 'tableId': 'in.c-main.orders', 'modelUUID': 'model-1'}, - ) - ], - [], - [], - False, - ['pre_query_check'], - [], - [], - [], - [], - ), - # Relationship names should fall back to object IDs when no display name is available, and constraints - # without severity should default to "error". - ( - [('model-1', 'Model', {'sql_dialect': 'bigquery'})], - [ - ( - 'constraint-defaults', - 'Constraint Defaults', - { - 'name': 'Constraint Defaults', - 'constraintType': 'range', - 'datasets': ['in.c-main.orders'], - 'validationQuery': {'default': 'SELECT default_check'}, - 'modelUUID': 'model-1', - }, - ) - ], - [ - ( - 'dataset-orders', - 'Orders', - {'name': 'Orders', 'tableId': 'in.c-main.orders', 'modelUUID': 'model-1'}, - ) - ], - [], - [ - ( - 'relationship-1', - '', - { - 'from': 'in.c-main.orders', - 'to': 'in.c-main.customers', - 'modelUUID': 'model-1', - }, - ) - ], - True, - [], - ['post_query_check'], - ['relationship-1'], - ['SELECT default_check'], - ['error'], - ), - ], -) -def test_evaluate_constraints_from_context_edge_cases( - model_specs: Sequence[tuple[str, str, Mapping[str, object]]], - constraint_specs: Sequence[tuple[str, str, Mapping[str, object]]], - used_dataset_specs: Sequence[tuple[str, str, Mapping[str, object]]], - used_metric_specs: Sequence[tuple[str, str, Mapping[str, object]]], - used_relationship_specs: Sequence[tuple[str, str, Mapping[str, object]]], - expected_valid: bool, - expected_violation_statuses: list[str], - expected_post_check_statuses: list[str], - expected_matched_relationships: list[str], - expected_post_check_queries: list[str], - expected_post_check_severities: list[str], -) -> None: - context_by_type = _evaluate_context( - model_specs=model_specs, - constraint_specs=constraint_specs, - ) - used_object_groups_by_type = _used_object_groups( - dataset_specs=used_dataset_specs, - metric_specs=used_metric_specs, - relationship_specs=used_relationship_specs, - ) - - result = evaluate_constraints_from_context(context_by_type, used_object_groups_by_type) - - assert result.valid is expected_valid - assert [finding.status for finding in result.violations] == expected_violation_statuses - assert [finding.status for finding in result.post_execution_checks] == expected_post_check_statuses - assert result.matched_relationships == expected_matched_relationships - assert [finding.validation_query for finding in result.post_execution_checks] == expected_post_check_queries - assert [finding.severity for finding in result.post_execution_checks] == expected_post_check_severities - - -@pytest.mark.parametrize( - ('sql_query', 'semantic_model_ids', 'message'), - [ - (' ', ['model-1'], 'sql_query must not be empty.'), - ('SELECT 1', [], 'At least one semantic_model_id must be provided.'), - ], -) -@pytest.mark.asyncio -async def test_validate_semantic_query_requires_non_empty_inputs( - keboola_client: KeboolaClient, - sql_query: str, - semantic_model_ids: list[str], - message: str, -) -> None: - with pytest.raises(ValueError, match=message): - await validate_semantic_query_with_used_objects(keboola_client, sql_query, semantic_model_ids) - - -@pytest.mark.parametrize( - ('patterns', 'semantic_types', 'case_sensitive', 'expected_ids', 'expected_paths'), - [ - ( - ['orders'], - [SemanticObjectType.SEMANTIC_DATASET], - False, - ['dataset-orders'], - ['fqn', 'meta.name', 'name', 'tableId'], - ), - ( - ['SUM\\(ORDER_AMOUNT\\)'], - [SemanticObjectType.SEMANTIC_METRIC], - False, - ['metric-revenue'], - ['sql'], - ), - ( - ['customer_id'], - [SemanticObjectType.SEMANTIC_RELATIONSHIP], - False, - ['relationship-orders-customers'], - ['on'], - ), - ( - ['revenue'], - [SemanticObjectType.SEMANTIC_GLOSSARY], - True, - [], - [], - ), - ], -) -@pytest.mark.asyncio -async def test_search_semantic_context_returns_expected_matches( - keboola_client: KeboolaClient, - mock_semantic_api: dict[SemanticObjectType, list[MetastoreObject]], - patterns: list[str], - semantic_types: list[SemanticObjectType], - case_sensitive: bool, - expected_ids: list[str], - expected_paths: list[str], -) -> None: - hits = await search_semantic_context( - keboola_client, - patterns, - semantic_types=semantic_types, - case_sensitive=case_sensitive, - ) - - assert [hit.object.id for hit in hits] == expected_ids - assert [hit.matched_paths for hit in hits] == ([expected_paths] if expected_paths else []) - - -@pytest.mark.parametrize( - ('patterns', 'max_results', 'message'), - [ - ([], 10, 'At least one regex pattern must be provided.'), - ([' '], 10, 'At least one regex pattern must be provided.'), - (['orders'], 0, 'max_results must be a positive integer.'), - ], -) -@pytest.mark.asyncio -async def test_search_semantic_context_validates_inputs( - keboola_client: KeboolaClient, - patterns: Sequence[str], - max_results: int, - message: str, -) -> None: - with pytest.raises(ValueError, match=message): - await search_semantic_context(keboola_client, patterns, max_results=max_results) diff --git a/tests/tools/storage/__init__.py b/tests/tools/storage/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/tools/storage/test_tools.py b/tests/tools/storage/test_tools.py deleted file mode 100644 index c02896697..000000000 --- a/tests/tools/storage/test_tools.py +++ /dev/null @@ -1,2041 +0,0 @@ -import json -from typing import Any, Mapping, Sequence -from unittest.mock import AsyncMock, call - -import httpx -import pytest -from fastmcp import Client, FastMCP -from mcp.server.fastmcp import Context -from mcp.types import TextContent -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import Config, MetadataField, ServerRuntimeInfo -from keboola_mcp_server.links import Link, ProjectLinksManager -from keboola_mcp_server.server import create_server -from keboola_mcp_server.tools.storage.tools import ( - BucketCounts, - BucketDetail, - DescriptionUpdate, - GetBucketsOutput, - GetTablesOutput, - TableColumnInfo, - TableDetail, - TableSummary, - UpdateDescriptionsOutput, - get_buckets, - get_tables, - update_descriptions, -) -from keboola_mcp_server.utils import parse_iso_timestamp -from keboola_mcp_server.workspace import DbColumnInfo, DbTableInfo, TableFqn, WorkspaceManager - - -def _get_sapi_tables(details: bool | None = None) -> list[dict[str, Any]]: - tables = [ - # users table in c-foo bucket in the production branch - { - 'uri': 'https://connection.keboola.com/v2/storage/tables/in.c-foo.users', - 'id': 'in.c-foo.users', - 'name': 'users', - 'displayName': 'All system users.', - 'transactional': False, - 'primaryKey': ['user_id'], - 'indexType': None, - 'indexKey': [], - 'distributionType': None, - 'distributionKey': [], - 'syntheticPrimaryKeyEnabled': False, - 'created': '2025-08-17T07:39:18+0200', - 'lastImportDate': '2025-08-20T19:11:52+0200', - 'lastChangeDate': '2025-08-20T19:11:52+0200', - 'rowsCount': 10, - 'dataSizeBytes': 10240, - 'isAlias': False, - 'isAliasable': True, - 'isTyped': False, - 'tableType': 'table', - 'path': '/users', - 'attributes': [], - 'metadata': [], - 'columns': ['user_id', 'name', 'surname'], - 'columnMetadata': { - 'user_id': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'INT'}, - ], - 'name': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'VARCHAR'}, - {'id': '1234', 'key': 'KBC.description', 'value': 'Name of the user.'}, - ], - 'surname': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'VARCHAR'}, - ], - }, - 'bucket': {'id': 'in.c-foo', 'name': 'c-foo'}, - }, - # emails table in c-foo bucket in the production branch - { - 'uri': 'https://connection.keboola.com/v2/storage/tables/in.c-foo.emails', - 'id': 'in.c-foo.emails', - 'name': 'emails', - 'displayName': 'All user emails.', - 'transactional': False, - 'primaryKey': ['email_id'], - 'indexType': None, - 'indexKey': [], - 'distributionType': None, - 'distributionKey': [], - 'syntheticPrimaryKeyEnabled': False, - 'created': '2025-08-17T07:39:18+0200', - 'lastImportDate': '2025-08-20T19:11:52+0200', - 'lastChangeDate': '2025-08-20T19:11:52+0200', - 'rowsCount': 33, - 'dataSizeBytes': 332211, - 'isAlias': False, - 'isAliasable': True, - 'isTyped': False, - 'tableType': 'table', - 'path': '/emails', - 'attributes': [], - 'metadata': [], - 'columns': ['email_id', 'address', 'user_id'], - 'columnMetadata': { - 'email_id': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'INT'}, - ], - 'address': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'VARCHAR'}, - {'id': '1234', 'key': 'KBC.description', 'value': 'Email address. 1'}, - ], - 'user_id': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'INT'}, - ], - }, - 'bucket': {'id': 'in.c-foo', 'name': 'c-foo'}, - }, - # emails table in c-foo bucket in the dev branch - { - 'uri': 'https://connection.keboola.com/v2/storage/tables/in.c-1246948-foo.emails', - 'id': 'in.c-1246948-foo.emails', - 'name': 'emails', - 'displayName': 'All user emails.', - 'transactional': False, - 'primaryKey': ['email_id'], - 'indexType': None, - 'indexKey': [], - 'distributionType': None, - 'distributionKey': [], - 'syntheticPrimaryKeyEnabled': False, - 'created': '2025-08-21T01:02:03+0400', - 'lastImportDate': '2025-08-21T01:02:03+0400', - 'lastChangeDate': '2025-08-21T01:02:03+0400', - 'rowsCount': 22, - 'dataSizeBytes': 2211, - 'isAlias': False, - 'isAliasable': True, - 'isTyped': False, - 'tableType': 'table', - 'path': '/emails', - 'attributes': [], - 'metadata': [{'id': '1726664231', 'key': 'KBC.createdBy.branch.id', 'value': '1246948'}], - 'columns': ['email_id', 'address', 'user_id'], - 'columnMetadata': { - 'email_id': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'INT'}, - ], - 'address': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'VARCHAR'}, - {'id': '1234', 'key': 'KBC.description', 'value': 'Email address. 2'}, - ], - 'user_id': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'INT'}, - ], - }, - 'bucket': {'id': 'in.c-1246948-foo', 'name': 'c-1246948-foo'}, - }, - # assets table in c-foo bucket in the dev branch - { - 'uri': 'https://connection.keboola.com/v2/storage/tables/in.c-1246948-foo.assets', - 'id': 'in.c-1246948-foo.assets', - 'name': 'assets', - 'displayName': 'Company assets.', - 'transactional': False, - 'primaryKey': ['asset_id'], - 'indexType': None, - 'indexKey': [], - 'distributionType': None, - 'distributionKey': [], - 'syntheticPrimaryKeyEnabled': False, - 'created': '2025-08-22T11:22:33+0200', - 'lastImportDate': '2025-08-22T11:22:33+0200', - 'lastChangeDate': '2025-08-22T11:22:33+0200', - 'rowsCount': 123, - 'dataSizeBytes': 123456, - 'isAlias': False, - 'isAliasable': True, - 'isTyped': False, - 'tableType': 'table', - 'path': '/assets', - 'attributes': [], - 'metadata': [{'id': '1726664231', 'key': 'KBC.createdBy.branch.id', 'value': '1246948'}], - 'columns': ['asset_id', 'name', 'value'], - 'columnMetadata': { - 'asset_id': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'INT'}, - ], - 'name': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'VARCHAR'}, - ], - 'value': [ - {'id': '1234', 'key': 'KBC.datatype.type', 'value': 'INT'}, - {'id': '1234', 'key': 'KBC.datatype.nullable', 'value': '1'}, - ], - }, - 'bucket': {'id': 'in.c-1246948-foo', 'name': 'c-1246948-foo'}, - 'sourceTable': { - 'project': { - 'name': 'Source Project', - 'id': '1234', - } - }, - }, - ] - if not details: - for t in tables: - t.pop('columns') - t.pop('columnMetadata') - t.pop('bucket') - return tables - - -def _bucket_table_list_side_effect(bid: str, *, include: list[str], **kwargs: Any) -> list[dict[str, Any]]: - prefix = f'{bid}.' - return [table for table in _get_sapi_tables() if table['id'].startswith(prefix)] - - -def _table_detail_side_effect(tid: str, **kwargs: Any) -> JsonDict: - for table in _get_sapi_tables(details=True): - if table['id'] == tid: - return table - - raise httpx.HTTPStatusError( - message=f'Table not found: {tid}', request=AsyncMock(), response=httpx.Response(status_code=404) - ) - - -def _get_sapi_buckets() -> list[dict[str, Any]]: - return [ - # foo bucket in the production branch - { - 'uri': 'https://connection.keboola.com/v2/storage/buckets/in.c-foo', - 'id': 'in.c-foo', - 'name': 'c-foo', - 'displayName': 'foo', - 'idBranch': 792027, - 'stage': 'in', - 'description': 'The foo bucket.', - 'tables': 'https://connection.keboola.com/v2/storage/buckets/in.c-foo', - 'created': '2025-07-03T11:02:54+0200', - 'lastChangeDate': '2025-08-17T07:37:42+0200', - 'updated': None, - 'isReadOnly': False, - 'dataSizeBytes': 1024, - 'rowsCount': 5, - 'isMaintenance': False, - 'backend': 'snowflake', - 'sharing': None, - 'hasExternalSchema': False, - 'databaseName': '', - 'path': 'in.c-foo', - 'isSnowflakeSharedDatabase': False, - 'color': None, - 'owner': None, - 'metadata': [], - }, - # foo bucket in the dev branch - { - 'uri': 'https://connection.keboola.com/v2/storage/buckets/in.c-1246948-foo', - 'id': 'in.c-1246948-foo', - 'name': 'c-1246948-foo', - 'displayName': '1246948-foo', - 'idBranch': 792027, - 'stage': 'in', - 'description': 'The dev branch foo bucket.', - 'tables': 'https://connection.keboola.com/v2/storage/buckets/in.c-1246948-foo', - 'created': '2025-08-17T07:39:14+0200', - 'lastChangeDate': '2025-08-17T07:39:26+0200', - 'updated': None, - 'isReadOnly': False, - 'dataSizeBytes': 4608, - 'rowsCount': 14, - 'isMaintenance': False, - 'backend': 'snowflake', - 'sharing': None, - 'hasExternalSchema': False, - 'databaseName': '', - 'path': 'in.c-1246948-foo', - 'isSnowflakeSharedDatabase': False, - 'color': None, - 'owner': None, - 'metadata': [ - {'id': '1726664228', 'key': 'KBC.createdBy.branch.id', 'value': '1246948'}, - ], - }, - # bar bucket in the production branch - { - 'uri': 'https://connection.keboola.com/v2/storage/buckets/in.c-bar', - 'id': 'out.c-bar', - 'name': 'c-bar', - 'displayName': 'bar', - 'idBranch': 792027, - 'stage': 'out', - 'description': 'Sample of Restaurant Reviews', - 'tables': 'https://connection.keboola.com/v2/storage/buckets/in.c-bar', - 'created': '2024-04-03T14:11:53+0200', - 'lastChangeDate': None, - 'updated': None, - 'isReadOnly': True, - 'dataSizeBytes': 2048, - 'rowsCount': 3, - 'isMaintenance': False, - 'backend': 'snowflake', - 'sharing': None, - 'hasExternalSchema': False, - 'databaseName': '', - 'path': 'out.c-bar', - 'isSnowflakeSharedDatabase': False, - 'color': None, - 'owner': None, - 'sourceBucket': { - 'id': 'out.c-bar', - 'name': 'c-bar', - 'displayName': 'bar', - 'stage': 'out', - 'description': 'Sample of Restaurant Reviews', - 'sharing': 'organization', - 'created': '2017-04-07T14:15:24+0200', - 'lastChangeDate': '2017-04-07T14:20:36+0200', - 'dataSizeBytes': 900096, - 'rowsCount': 2239, - 'backend': 'snowflake', - 'hasExternalSchema': False, - 'databaseName': '', - 'path': 'out.c-bar', - 'project': {'id': 1234, 'name': 'A demo project'}, - 'tables': [ - { - 'id': 'in.c-bar.restaurants', - 'name': 'restaurants', - 'displayName': 'restaurants', - 'path': '/406653-restaurants', - }, - {'id': 'in.c-bar.reviews', 'name': 'reviews', 'displayName': 'reviews', 'path': '/406653-reviews'}, - ], - 'color': None, - 'sharingParameters': [], - 'sharedBy': {'id': None, 'name': None, 'date': ''}, - 'owner': None, - }, - 'metadata': [], - }, - # baz bucket in the dev branch - { - 'uri': 'https://connection.keboola.com/v2/storage/buckets/in.c-1246948-baz', - 'id': 'in.c-1246948-baz', - 'name': 'c-1246948-baz', - 'displayName': '1246948-baz', - 'idBranch': 792027, - 'stage': 'in', - 'description': 'The dev branch baz bucket.', - 'tables': 'https://connection.keboola.com/v2/storage/buckets/in.c-1246948-baz', - 'created': '2025-01-02T03:04:05+0600', - 'lastChangeDate': '2025-01-02T03:04:55+0600', - 'updated': None, - 'isReadOnly': False, - 'dataSizeBytes': 987654321, - 'rowsCount': 123, - 'isMaintenance': False, - 'backend': 'snowflake', - 'sharing': None, - 'hasExternalSchema': False, - 'databaseName': '', - 'path': 'in.c-1246948-baz', - 'isSnowflakeSharedDatabase': False, - 'color': None, - 'owner': None, - 'metadata': [ - {'id': '1726664228', 'key': 'KBC.createdBy.branch.id', 'value': '1246948'}, - ], - }, - ] - - -def _bucket_detail_side_effect(bid: str, **kwargs: Any) -> JsonDict: - for bucket in _get_sapi_buckets(): - if bucket['id'] == bid: - return bucket - - raise httpx.HTTPStatusError( - message=f'Bucket not found: {bid}', request=AsyncMock(), response=httpx.Response(status_code=404) - ) - - -@pytest.fixture -def mock_update_bucket_description_response() -> Sequence[Mapping[str, Any]]: - """Mock valid response list for updating a bucket description.""" - return [ - { - 'id': '999', - 'key': MetadataField.DESCRIPTION, - 'value': 'Updated bucket description', - 'provider': 'user', - 'timestamp': '2024-01-01T00:00:00Z', - } - ] - - -@pytest.fixture -def mock_update_table_description_response() -> Mapping[str, Any]: - """Mock valid response from the Keboola API for table description update.""" - return { - 'metadata': [ - { - 'id': '1724427984', - 'key': 'KBC.description', - 'value': 'Updated table description', - 'provider': 'user', - 'timestamp': '2024-01-01T00:00:00Z', - } - ], - 'columnsMetadata': { - 'text': [ - { - 'id': '1725066342', - 'key': 'KBC.description', - 'value': 'Updated column description', - 'provider': 'user', - 'timestamp': '2024-01-01T00:00:00Z', - } - ] - }, - } - - -@pytest.fixture -def mock_update_column_description_response() -> Mapping[str, Any]: - """Mock valid response from the Keboola API for column description update.""" - return { - 'metadata': [ - { - 'id': '1724427984', - 'key': 'KBC.description', - 'value': 'Updated table description', - 'provider': 'user', - 'timestamp': '2024-01-01T00:00:00Z', - } - ], - 'columnsMetadata': { - 'column_name': [ - { - 'id': '1725066342', - 'key': 'KBC.description', - 'value': 'Updated column description', - 'provider': 'user', - 'timestamp': '2024-01-01T00:00:00Z', - } - ] - }, - } - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('branch_id', 'bucket_id', 'expected_bucket'), - [ - ( - None, - 'in.c-foo', - BucketDetail( - id='in.c-foo', - name='c-foo', - display_name='foo', - description='The foo bucket.', - stage='in', - created='2025-07-03T11:02:54+0200', - updated='2025-08-17T07:37:42+0200', - data_size_bytes=1024, - links=[ - Link( - type='ui-detail', - title='Bucket: c-foo', - url='https://connection.test.keboola.com/admin/projects/69420/storage/in.c-foo', - ), - ], - ), - ), - ( - '1246948', - 'in.c-foo', - BucketDetail( - # all fields come from the prod bucket except for data_size_bytes - id='in.c-foo', - name='c-foo', - display_name='foo', - description='The foo bucket.', - stage='in', - created='2025-07-03T11:02:54+0200', - updated='2025-08-17T07:37:42+0200', - data_size_bytes=4608 + 1024, - links=[ - Link( - type='ui-detail', - title='Bucket: c-foo', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948' - '/storage/in.c-1246948-foo', - ), - ], - ), - ), - ( - None, - 'out.c-bar', - BucketDetail( - id='out.c-bar', - name='c-bar', - display_name='bar', - description='Sample of Restaurant Reviews', - stage='out', - created='2024-04-03T14:11:53+0200', - updated=None, - data_size_bytes=2048, - links=[ - Link( - type='ui-detail', - title='Bucket: c-bar', - url='https://connection.test.keboola.com/admin/projects/69420/storage/out.c-bar', - ), - ], - source_project='A demo project (ID: 1234)', - ), - ), - ( - '1246948', # no in.c-bar on this branch - 'out.c-bar', - BucketDetail( - id='out.c-bar', - name='c-bar', - display_name='bar', - description='Sample of Restaurant Reviews', - stage='out', - created='2024-04-03T14:11:53+0200', - updated=None, - data_size_bytes=2048, - links=[ - Link( - type='ui-detail', - title='Bucket: c-bar', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948/storage/out.c-bar', - ), - ], - source_project='A demo project (ID: 1234)', - ), - ), - ( - '1246948', - 'in.c-baz', - BucketDetail( - id='in.c-baz', - name='c-1246948-baz', - display_name='1246948-baz', - description='The dev branch baz bucket.', - stage='in', - created='2025-01-02T03:04:05+0600', - updated='2025-01-02T03:04:55+0600', - data_size_bytes=987654321, - links=[ - Link( - type='ui-detail', - title='Bucket: c-1246948-baz', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948' - '/storage/in.c-1246948-baz', - ), - ], - ), - ), - (None, 'in.c-not-existing', None), - ], -) -async def test_get_bucket( - branch_id: str | None, - bucket_id: str, - expected_bucket: BucketDetail | None, - mocker: MockerFixture, - mcp_context_client: Context, -): - """Test get_bucket tool.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = branch_id - keboola_client.has_feature = mocker.AsyncMock(return_value=False) - keboola_client.storage_client.bucket_detail = mocker.AsyncMock(side_effect=_bucket_detail_side_effect) - - result = await get_buckets(mcp_context_client, [bucket_id]) - - if branch_id: - keboola_client.storage_client.bucket_detail.assert_has_calls( - [ - call(bucket_id, branch_id='default'), - call(bucket_id.replace('c-', f'c-{branch_id}-'), branch_id='default'), - ] - ) - dashboard_url = f'https://connection.test.keboola.com/admin/projects/69420/branch/{branch_id}/storage' - else: - keboola_client.storage_client.bucket_detail.assert_called_once_with(bucket_id, branch_id='default') - dashboard_url = 'https://connection.test.keboola.com/admin/projects/69420/storage' - - assert isinstance(result, GetBucketsOutput) - if expected_bucket is not None: - expected_result = GetBucketsOutput( - buckets=[expected_bucket], - links=[Link(type='ui-dashboard', title='Buckets in the project', url=dashboard_url)], - ).pack_links() - assert result == expected_result - else: - expectd_result = GetBucketsOutput( - buckets=[], - buckets_not_found=[bucket_id], - links=[Link(type='ui-dashboard', title='Buckets in the project', url=dashboard_url)], - ) - assert result == expectd_result - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('branch_id', 'expected_buckets'), - [ - ( - None, # production branch - [ - BucketDetail( - id='in.c-foo', - name='c-foo', - display_name='foo', - description='The foo bucket.', - stage='in', - created='2025-07-03T11:02:54+0200', - updated='2025-08-17T07:37:42+0200', - data_size_bytes=1024, - ), - BucketDetail( - id='out.c-bar', - name='c-bar', - display_name='bar', - description='Sample of Restaurant Reviews', - stage='out', - created='2024-04-03T14:11:53+0200', - updated=None, - data_size_bytes=2048, - source_project='A demo project (ID: 1234)', - ), - ], - ), - ( - '1246948', # development branch - [ - BucketDetail( - id='in.c-foo', - name='c-foo', - display_name='foo', - description='The foo bucket.', - stage='in', - created='2025-07-03T11:02:54+0200', - updated='2025-08-17T07:37:42+0200', - data_size_bytes=4608 + 1024, - ), - BucketDetail( - id='out.c-bar', - name='c-bar', - display_name='bar', - description='Sample of Restaurant Reviews', - stage='out', - created='2024-04-03T14:11:53+0200', - updated=None, - data_size_bytes=2048, - source_project='A demo project (ID: 1234)', - ), - BucketDetail( - id='in.c-baz', - name='c-1246948-baz', - display_name='1246948-baz', - description='The dev branch baz bucket.', - stage='in', - created='2025-01-02T03:04:05+0600', - updated='2025-01-02T03:04:55+0600', - data_size_bytes=987654321, - ), - ], - ), - ], -) -async def test_get_buckets( - branch_id: str | None, expected_buckets: list[BucketDetail], mocker: MockerFixture, mcp_context_client: Context -) -> None: - """Test the get_buckets tool.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = branch_id - keboola_client.has_feature = mocker.AsyncMock(return_value=False) - keboola_client.storage_client.bucket_list = mocker.AsyncMock(return_value=_get_sapi_buckets()) - - result = await get_buckets(mcp_context_client) - - assert isinstance(result, GetBucketsOutput) - assert result.buckets == expected_buckets - assert result.bucket_counts.total_buckets == len(expected_buckets) - - # Count expected buckets by stage - expected_input_count = sum(1 for bucket in expected_buckets if bucket.stage == 'in') - expected_output_count = sum(1 for bucket in expected_buckets if bucket.stage == 'out') - - assert result.bucket_counts.input_buckets == expected_input_count - assert result.bucket_counts.output_buckets == expected_output_count - keboola_client.storage_client.bucket_list.assert_called_once() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('branch_id', 'table_id', 'expected_table'), - [ - ( - None, - 'in.c-foo.users', - TableDetail( - id='in.c-foo.users', - name='users', - display_name='All system users.', - primary_key=['user_id'], - created='2025-08-17T07:39:18+0200', - updated='2025-08-20T19:11:52+0200', - rows_count=10, - data_size_bytes=10240, - columns=[ - TableColumnInfo( - name='user_id', - quoted_name='#user_id#', - database_native_type='INT', - nullable=False, - description=None, - ), - TableColumnInfo( - name='name', - quoted_name='#name#', - database_native_type='VARCHAR', - nullable=False, - description='Name of the user.', - ), - TableColumnInfo( - name='surname', - quoted_name='#surname#', - database_native_type='VARCHAR', - nullable=False, - description=None, - ), - ], - fully_qualified_name='#SAPI_TEST#.#in.c-foo#.#users#', - links=[ - Link( - type='ui-detail', - title='Table: users', - url='https://connection.test.keboola.com/admin/projects/69420/storage/in.c-foo/table/users', - ), - ], - ), - ), - ( - '1246948', - 'in.c-foo.users', - TableDetail( - id='in.c-foo.users', - name='users', - display_name='All system users.', - primary_key=['user_id'], - created='2025-08-17T07:39:18+0200', - updated='2025-08-20T19:11:52+0200', - rows_count=10, - data_size_bytes=10240, - columns=[ - TableColumnInfo( - name='user_id', - quoted_name='#user_id#', - database_native_type='INT', - nullable=False, - description=None, - ), - TableColumnInfo( - name='name', - quoted_name='#name#', - database_native_type='VARCHAR', - nullable=False, - description='Name of the user.', - ), - TableColumnInfo( - name='surname', - quoted_name='#surname#', - database_native_type='VARCHAR', - nullable=False, - description=None, - ), - ], - fully_qualified_name='#SAPI_TEST#.#in.c-foo#.#users#', - links=[ - Link( - type='ui-detail', - title='Table: users', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948/storage/in.c-foo' - '/table/users', - ), - ], - ), - ), - ( - None, - 'in.c-foo.emails', - TableDetail( - id='in.c-foo.emails', - name='emails', - display_name='All user emails.', - primary_key=['email_id'], - created='2025-08-17T07:39:18+0200', - updated='2025-08-20T19:11:52+0200', - rows_count=33, - data_size_bytes=332211, - columns=[ - TableColumnInfo( - name='email_id', quoted_name='#email_id#', database_native_type='INT', nullable=False - ), - TableColumnInfo( - name='address', - quoted_name='#address#', - database_native_type='VARCHAR', - nullable=False, - description='Email address. 1', - ), - TableColumnInfo( - name='user_id', quoted_name='#user_id#', database_native_type='INT', nullable=False - ), - ], - fully_qualified_name='#SAPI_TEST#.#in.c-foo#.#emails#', - links=[ - Link( - type='ui-detail', - title='Table: emails', - url='https://connection.test.keboola.com/admin/projects/69420/storage/in.c-foo/table/emails', - ), - ], - ), - ), - ( - '1246948', - 'in.c-foo.emails', - TableDetail( - id='in.c-foo.emails', - name='emails', - display_name='All user emails.', - primary_key=['email_id'], - created='2025-08-21T01:02:03+0400', - updated='2025-08-21T01:02:03+0400', - rows_count=22, - data_size_bytes=2211, - columns=[ - TableColumnInfo( - name='email_id', quoted_name='#email_id#', database_native_type='INT', nullable=False - ), - TableColumnInfo( - name='address', - quoted_name='#address#', - database_native_type='VARCHAR', - nullable=False, - description='Email address. 2', - ), - TableColumnInfo( - name='user_id', quoted_name='#user_id#', database_native_type='INT', nullable=False - ), - ], - fully_qualified_name='#SAPI_TEST#.#in.c-1246948-foo#.#emails#', - links=[ - Link( - type='ui-detail', - title='Table: emails', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948' - '/storage/in.c-1246948-foo/table/emails', - ), - ], - ), - ), - (None, 'in.c-1246948-foo.assets', None), - ( - '1246948', - 'in.c-1246948-foo.emails', - TableDetail( - id='in.c-foo.emails', - name='emails', - display_name='All user emails.', - primary_key=['email_id'], - created='2025-08-21T01:02:03+0400', - updated='2025-08-21T01:02:03+0400', - rows_count=22, - data_size_bytes=2211, - columns=[ - TableColumnInfo( - name='email_id', quoted_name='#email_id#', database_native_type='INT', nullable=False - ), - TableColumnInfo( - name='address', - quoted_name='#address#', - database_native_type='VARCHAR', - nullable=False, - description='Email address. 2', - ), - TableColumnInfo( - name='user_id', quoted_name='#user_id#', database_native_type='INT', nullable=False - ), - ], - fully_qualified_name='#SAPI_TEST#.#in.c-1246948-foo#.#emails#', - links=[ - Link( - type='ui-detail', - title='Table: emails', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948' - '/storage/in.c-1246948-foo/table/emails', - ), - ], - ), - ), - ( - '1246948', - 'in.c-foo.assets', - TableDetail( - id='in.c-foo.assets', - name='assets', - display_name='Company assets.', - primary_key=['asset_id'], - created='2025-08-22T11:22:33+0200', - updated='2025-08-22T11:22:33+0200', - rows_count=123, - data_size_bytes=123456, - columns=[ - TableColumnInfo( - name='asset_id', quoted_name='#asset_id#', database_native_type='INT', nullable=False - ), - TableColumnInfo(name='name', quoted_name='#name#', database_native_type='VARCHAR', nullable=False), - TableColumnInfo(name='value', quoted_name='#value#', database_native_type='INT', nullable=True), - ], - fully_qualified_name='#SAPI_TEST#.#in.c-1246948-foo#.#assets#', - links=[ - Link( - type='ui-detail', - title='Table: assets', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948' - '/storage/in.c-1246948-foo/table/assets', - ), - ], - source_project='Source Project (ID: 1234)', - ), - ), - ], -) -async def test_get_table( - branch_id: str | None, - table_id: str, - expected_table: TableDetail | None, - mocker: MockerFixture, - mcp_context_client: Context, -) -> None: - """Test get_table tool.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = branch_id - keboola_client.has_feature = mocker.AsyncMock(return_value=False) - keboola_client.storage_client.bucket_detail = mocker.AsyncMock(side_effect=_bucket_detail_side_effect) - keboola_client.storage_client.table_detail = mocker.AsyncMock(side_effect=_table_detail_side_effect) - - workspace_manager = WorkspaceManager.from_state(mcp_context_client.session.state) - workspace_manager.get_table_info = mocker.AsyncMock( - side_effect=lambda sapi_table: DbTableInfo( - id=sapi_table['id'], - fqn=TableFqn( - db_name='SAPI_TEST', - schema_name=sapi_table['bucket']['id'], - table_name=sapi_table['id'].rsplit('.')[-1], - quote_char='#', - ), - columns={}, - ) - ) - workspace_manager.get_quoted_name = mocker.AsyncMock(side_effect=lambda name: f'#{name}#') - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='test-sql-dialect') - - result = await get_tables(mcp_context_client, table_ids=[table_id]) - assert isinstance(result, GetTablesOutput) - - if branch_id: - keboola_client.storage_client.table_detail.assert_has_calls( - [ - call(table_id, branch_id='default'), - call( - table_id.replace('c-', f'c-{branch_id}-') if f'c-{branch_id}-' not in table_id else table_id, - branch_id='default', - ), - ] - ) - dashboard_url = f'https://connection.test.keboola.com/admin/projects/69420/branch/{branch_id}/storage' - else: - keboola_client.storage_client.table_detail.assert_called_once_with(table_id, branch_id='default') - dashboard_url = 'https://connection.test.keboola.com/admin/projects/69420/storage' - - if expected_table: - expected_result = GetTablesOutput( - tables=[expected_table], - links=[Link(type='ui-dashboard', title='Buckets in the project', url=dashboard_url)], - ).pack_links() - assert result == expected_result - workspace_manager.get_sql_dialect.assert_called_once() - workspace_manager.get_table_info.assert_called_once() - workspace_manager.get_quoted_name.assert_has_calls([call(col_info.name) for col_info in expected_table.columns]) - - else: - expected_result = GetTablesOutput( - tables=[], - tables_not_found=[table_id], - links=[Link(type='ui-dashboard', title='Buckets in the project', url=dashboard_url)], - ).pack_links() - assert result == expected_result - workspace_manager.get_sql_dialect.assert_not_called() - workspace_manager.get_table_info.assert_not_called() - workspace_manager.get_quoted_name.assert_not_called() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('branch_id', 'bucket_id', 'expected_tables'), - [ - ( - None, - 'in.c-foo', - [ - TableSummary( - id='in.c-foo.users', - name='users', - display_name='All system users.', - primary_key=['user_id'], - created='2025-08-17T07:39:18+0200', - updated='2025-08-20T19:11:52+0200', - rows_count=10, - data_size_bytes=10240, - links=[ - Link( - type='ui-detail', - title='Table: users', - url='https://connection.test.keboola.com/admin/projects/69420/storage/in.c-foo/table/users', - ) - ], - ), - TableSummary( - id='in.c-foo.emails', - name='emails', - display_name='All user emails.', - primary_key=['email_id'], - created='2025-08-17T07:39:18+0200', - updated='2025-08-20T19:11:52+0200', - rows_count=33, - data_size_bytes=332211, - links=[ - Link( - type='ui-detail', - title='Table: emails', - url='https://connection.test.keboola.com/admin/projects/69420' - '/storage/in.c-foo/table/emails', - ) - ], - ), - ], - ), - ( - '1246948', # development branch - 'in.c-foo', - [ - TableSummary( - id='in.c-foo.users', - name='users', - display_name='All system users.', - primary_key=['user_id'], - created='2025-08-17T07:39:18+0200', - updated='2025-08-20T19:11:52+0200', - rows_count=10, - data_size_bytes=10240, - links=[ - Link( - type='ui-detail', - title='Table: users', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948' - '/storage/in.c-foo/table/users', - ) - ], - ), - # in.c-foo.emails comes from in.c-1246948-foo bucket - TableSummary( - id='in.c-foo.emails', - name='emails', - display_name='All user emails.', - primary_key=['email_id'], - created='2025-08-21T01:02:03+0400', - updated='2025-08-21T01:02:03+0400', - rows_count=22, - data_size_bytes=2211, - links=[ - Link( - type='ui-detail', - title='Table: emails', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948' - '/storage/in.c-1246948-foo/table/emails', - ) - ], - ), - TableSummary( - id='in.c-foo.assets', - name='assets', - display_name='Company assets.', - primary_key=['asset_id'], - created='2025-08-22T11:22:33+0200', - updated='2025-08-22T11:22:33+0200', - rows_count=123, - data_size_bytes=123456, - source_project='Source Project (ID: 1234)', - links=[ - Link( - type='ui-detail', - title='Table: assets', - url='https://connection.test.keboola.com/admin/projects/69420/branch/1246948' - '/storage/in.c-1246948-foo/table/assets', - ) - ], - ), - ], - ), - ], -) -async def test_get_tables( - branch_id: str | None, - bucket_id: str, - expected_tables: list[TableSummary], - mocker: MockerFixture, - mcp_context_client: Context, -) -> None: - """Test get_tables tool.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = branch_id - keboola_client.has_feature = mocker.AsyncMock(return_value=False) - keboola_client.storage_client.bucket_detail = mocker.AsyncMock(side_effect=_bucket_detail_side_effect) - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(side_effect=_bucket_table_list_side_effect) - links_manager = await ProjectLinksManager.from_client(keboola_client) - - result = await get_tables(mcp_context_client, [bucket_id]) - assert isinstance(result, GetTablesOutput) - - expected_result = GetTablesOutput( - tables=expected_tables, links=[links_manager.get_bucket_dashboard_link()] - ).pack_links() - assert result == expected_result - - # Listing returns summaries that never resolve the warehouse FQN, so the field must be - # absent from the (structured) output entirely — not emitted as a misleading `null`, which - # the query_data queryability rule would read as "not queryable". - for table, dumped in zip(result.tables, result.model_dump(by_alias=True)['tables']): - assert isinstance(table, TableSummary) - assert not isinstance(table, TableDetail) - assert 'fullyQualifiedName' not in dumped - assert 'columns' not in dumped - - sapi_includes = ['metadata', 'columnMetadata', 'sourceMetadata', 'sourceColumnMetadata'] - if branch_id: - keboola_client.storage_client.bucket_detail.assert_has_calls( - [ - call(bucket_id, branch_id='default'), - call(bucket_id.replace('c-', f'c-{branch_id}-'), branch_id='default'), - ] - ) - keboola_client.storage_client.bucket_table_list.assert_has_calls( - [ - call(bucket_id, include=sapi_includes, branch_id='default'), - call(bucket_id.replace('c-', f'c-{branch_id}-'), include=sapi_includes, branch_id='default'), - ] - ) - else: - keboola_client.storage_client.bucket_detail.assert_called_once_with(bucket_id, branch_id='default') - keboola_client.storage_client.bucket_table_list.assert_called_once_with( - bucket_id, include=sapi_includes, branch_id='default' - ) - - -@pytest.mark.parametrize( - ('raw_data', 'expected_description'), - [ - # AI-3423: curated KBC metadata wins over the stale legacy `description` field - ( - { - 'description': 'Bucket created by Transformation API', - 'metadata': [ - {'key': 'KBC.sharedDescription', 'value': 'Shared desc'}, - {'key': 'KBC.description', 'value': 'Meta desc'}, - ], - }, - 'Shared desc', - ), - # KBC.description wins over the legacy `description` field (no sharedDescription) - ( - { - 'description': 'Bucket created by Transformation API', - 'metadata': [{'key': 'KBC.description', 'value': 'Meta desc'}], - }, - 'Meta desc', - ), - # legacy `description` field used only when no KBC metadata is present - ( - {'description': 'Direct desc', 'metadata': []}, - 'Direct desc', - ), - # KBC.sharedDescription is preferred over KBC.description - ( - { - 'description': '', - 'metadata': [ - {'key': 'KBC.sharedDescription', 'value': 'Shared desc'}, - {'key': 'KBC.description', 'value': 'Meta desc'}, - ], - }, - 'Shared desc', - ), - # KBC.description used when no sharedDescription - ( - { - 'description': '', - 'metadata': [{'key': 'KBC.description', 'value': 'Meta desc'}], - }, - 'Meta desc', - ), - # None when nothing is available - ( - {'description': '', 'metadata': []}, - None, - ), - ], -) -def test_bucket_detail_description_fallback(raw_data: dict[str, Any], expected_description: str | None) -> None: - """Test BucketDetail description resolution priority.""" - base = { - 'id': 'in.c-test', - 'name': 'c-test', - 'displayName': 'test', - 'stage': 'in', - 'created': '2025-01-01T00:00:00+0000', - 'dataSizeBytes': 0, - } - bucket = BucketDetail.model_validate(base | raw_data) - assert bucket.description == expected_description - - -@pytest.mark.parametrize( - ('raw_data', 'expected_description'), - [ - # AI-3423: curated KBC.description wins over the stale legacy `description` field - ( - { - 'description': 'Table created by Transformation API', - 'metadata': [{'key': 'KBC.description', 'value': 'Meta desc'}], - 'sourceTable': {'metadata': [{'key': 'KBC.description', 'value': 'Source desc'}]}, - }, - 'Meta desc', - ), - # legacy `description` field used only when no KBC metadata is present - ( - {'description': 'Direct desc', 'metadata': []}, - 'Direct desc', - ), - # own KBC.description is preferred over sourceTable metadata - ( - { - 'description': '', - 'metadata': [{'key': 'KBC.description', 'value': 'Meta desc'}], - 'sourceTable': {'metadata': [{'key': 'KBC.description', 'value': 'Source desc'}]}, - }, - 'Meta desc', - ), - # sourceTable.metadata used as final fallback - ( - { - 'description': '', - 'metadata': [], - 'sourceTable': {'metadata': [{'key': 'KBC.description', 'value': 'Source desc'}]}, - }, - 'Source desc', - ), - # None when nothing is available - ( - {'description': '', 'metadata': []}, - None, - ), - ], -) -def test_table_detail_description_fallback(raw_data: dict[str, Any], expected_description: str | None) -> None: - """Test TableDetail description resolution priority.""" - base = { - 'id': 'in.c-test.table1', - 'name': 'table1', - 'displayName': 'table1', - } - table = TableDetail.model_validate(base | raw_data) - assert table.description == expected_description - - -_LINEAGE_METADATA = [ - {'key': 'KBC.lastUpdatedBy.component.id', 'value': 'keboola.c', 'timestamp': '2025-09-15T00:00:00+0200'}, - {'key': 'KBC.lastUpdatedBy.configuration.id', 'value': '1', 'timestamp': '2025-09-15T00:00:00+0200'}, -] - - -@pytest.mark.parametrize( - ('raw_data', 'expected_updated'), - [ - # lastChangeDate wins when later - ( - {'lastChangeDate': '2025-09-01T12:00:00+0200', 'lastImportDate': '2025-08-01T12:00:00+0200'}, - '2025-09-01T12:00:00+0200', - ), - # lastImportDate wins when later - ( - {'lastChangeDate': '2025-08-01T12:00:00+0200', 'lastImportDate': '2025-09-01T12:00:00+0200'}, - '2025-09-01T12:00:00+0200', - ), - # only lastChangeDate present - ({'lastChangeDate': '2025-08-01T12:00:00+0200'}, '2025-08-01T12:00:00+0200'), - # only lastImportDate present - ({'lastImportDate': '2025-08-01T12:00:00+0200'}, '2025-08-01T12:00:00+0200'), - # no timestamps → None - ({'lastChangeDate': None, 'lastImportDate': None}, None), - # round-trip guard: explicit 'updated' must not be overwritten - ( - {'updated': '2025-07-01T00:00:00+0200', 'lastChangeDate': '2025-09-01T12:00:00+0200'}, - '2025-07-01T00:00:00+0200', - ), - # lineage timestamp newer → promoted over lastChangeDate - ( - {'lastChangeDate': '2025-08-01T12:00:00+0200', 'metadata': _LINEAGE_METADATA}, - '2025-09-15T00:00:00+0200', - ), - # lineage timestamp older → lastChangeDate wins - ( - {'lastChangeDate': '2025-10-01T00:00:00+0200', 'metadata': _LINEAGE_METADATA}, - '2025-10-01T00:00:00+0200', - ), - # metadata only, no date fields → metadata timestamp - ({'metadata': _LINEAGE_METADATA}, '2025-09-15T00:00:00+0200'), - ], -) -def test_table_detail_updated_resolution(raw_data: dict[str, Any], expected_updated: str | None) -> None: - """Test TableDetail.updated is resolved from lastChangeDate / lastImportDate / lineage metadata.""" - base = {'id': 'in.c-test.t1', 'name': 't1', 'displayName': 't1'} - table = TableDetail.model_validate(base | raw_data).with_lineage_metadata(raw_data) - assert table.updated == expected_updated - - -@pytest.mark.parametrize( - ('raw_data', 'expected_updated'), - [ - # lastChangeDate present - ({'lastChangeDate': '2025-09-01T12:00:00+0200'}, '2025-09-01T12:00:00+0200'), - # no timestamps → None - ({'lastChangeDate': None}, None), - # round-trip guard: explicit 'updated' must not be overwritten - ( - {'updated': '2025-07-01T00:00:00+0200', 'lastChangeDate': '2025-09-01T12:00:00+0200'}, - '2025-07-01T00:00:00+0200', - ), - # lineage timestamp newer → promoted over lastChangeDate - ( - {'lastChangeDate': '2025-08-01T00:00:00+0200', 'metadata': _LINEAGE_METADATA}, - '2025-09-15T00:00:00+0200', - ), - # lineage timestamp older → lastChangeDate wins - ( - {'lastChangeDate': '2025-10-01T00:00:00+0200', 'metadata': _LINEAGE_METADATA}, - '2025-10-01T00:00:00+0200', - ), - # metadata only, no date fields → metadata timestamp - ({'metadata': _LINEAGE_METADATA}, '2025-09-15T00:00:00+0200'), - ], -) -def test_bucket_detail_updated_resolution(raw_data: dict[str, Any], expected_updated: str | None) -> None: - """Test BucketDetail.updated is resolved from lastChangeDate / lineage metadata.""" - base = { - 'id': 'in.c-test', - 'name': 'c-test', - 'displayName': 'test', - 'stage': 'in', - 'created': '2025-01-01T00:00:00+0200', - 'dataSizeBytes': 0, - } - bucket = BucketDetail.model_validate(base | raw_data).with_lineage_metadata(raw_data) - assert bucket.updated == expected_updated - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('column_metadata', 'source_column_metadata', 'expected_description', 'expected_base_type'), - [ - # own column metadata takes priority over source - ( - [ - {'key': 'KBC.description', 'value': 'Own desc'}, - {'key': 'KBC.datatype.basetype', 'value': 'STRING'}, - ], - [ - {'key': 'KBC.description', 'value': 'Source desc'}, - {'key': 'KBC.datatype.basetype', 'value': 'INTEGER'}, - ], - 'Own desc', - 'STRING', - ), - # falls back to source column metadata when own is empty - ( - [], - [ - {'key': 'KBC.description', 'value': 'Source desc'}, - {'key': 'KBC.datatype.basetype', 'value': 'INTEGER'}, - ], - 'Source desc', - 'INTEGER', - ), - # mixed: own description, source base type - ( - [{'key': 'KBC.description', 'value': 'Own desc'}], - [{'key': 'KBC.datatype.basetype', 'value': 'INTEGER'}], - 'Own desc', - 'INTEGER', - ), - # neither has metadata - ( - [], - [], - None, - None, - ), - ], -) -async def test_get_table_column_metadata_fallback( - column_metadata: list[dict[str, Any]], - source_column_metadata: list[dict[str, Any]], - expected_description: str | None, - expected_base_type: str | None, - mocker: MockerFixture, - mcp_context_client: Context, -) -> None: - """Test column description and base type fallback from sourceTable.columnMetadata.""" - raw_table = { - 'id': 'in.c-test.t1', - 'name': 't1', - 'displayName': 't1', - 'primaryKey': [], - 'created': '2025-01-01T00:00:00+0000', - 'rowsCount': 1, - 'dataSizeBytes': 100, - 'columns': ['col1'], - 'columnMetadata': {'col1': column_metadata}, - 'metadata': [], - 'bucket': {'id': 'in.c-test', 'name': 'c-test'}, - 'sourceTable': {'columnMetadata': {'col1': source_column_metadata}}, - } - - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = None - keboola_client.storage_client.table_detail = mocker.AsyncMock(return_value=raw_table) - keboola_client.storage_client.bucket_detail = mocker.AsyncMock( - side_effect=httpx.HTTPStatusError( - message='Not found', request=AsyncMock(), response=httpx.Response(status_code=404) - ) - ) - - workspace_manager = WorkspaceManager.from_state(mcp_context_client.session.state) - workspace_manager.get_table_info = mocker.AsyncMock( - return_value=DbTableInfo( - id='in.c-test.t1', - fqn=TableFqn(db_name='DB', schema_name='in.c-test', table_name='t1', quote_char='"'), - columns={ - 'col1': DbColumnInfo(name='col1', quoted_name='"col1"', native_type='VARCHAR', nullable=True), - }, - ), - ) - workspace_manager.get_quoted_name = mocker.AsyncMock(return_value='"col1"') - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='snowflake') - - result = await get_tables(mcp_context_client, table_ids=['in.c-test.t1']) - assert isinstance(result, GetTablesOutput) - assert len(result.tables) == 1 - - columns = result.tables[0].columns - assert columns is not None - assert len(columns) == 1 - assert columns[0].description == expected_description - assert columns[0].keboola_base_type == expected_base_type - - -@pytest.mark.asyncio -async def test_update_descriptions_bucket_success( - mocker: MockerFixture, mcp_context_client, mock_update_bucket_description_response -) -> None: - """Test successful update of bucket description using update_descriptions.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.bucket_metadata_update = mocker.AsyncMock( - return_value=mock_update_bucket_description_response, - ) - - result = await update_descriptions( - ctx=mcp_context_client, - updates=[DescriptionUpdate(item_id='in.c-test-bucket', description='Updated bucket description')], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 1 - assert result.successful == 1 - assert result.failed == 0 - assert len(result.results) == 1 - - bucket_result = result.results[0] - assert bucket_result.item_id == 'in.c-test-bucket' - assert bucket_result.success is True - assert bucket_result.error is None - assert bucket_result.timestamp == parse_iso_timestamp('2024-01-01T00:00:00Z') - - keboola_client.storage_client.bucket_metadata_update.assert_called_once_with( - bucket_id='in.c-test-bucket', - metadata={MetadataField.DESCRIPTION: 'Updated bucket description'}, - ) - - -@pytest.mark.asyncio -async def test_update_descriptions_table_success( - mocker: MockerFixture, mcp_context_client, mock_update_table_description_response -) -> None: - """Test successful update of table description using update_descriptions.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.table_metadata_update = mocker.AsyncMock( - return_value=mock_update_table_description_response, - ) - - result = await update_descriptions( - ctx=mcp_context_client, - updates=[DescriptionUpdate(item_id='in.c-test.test-table', description='Updated table description')], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 1 - assert result.successful == 1 - assert result.failed == 0 - assert len(result.results) == 1 - - table_result = result.results[0] - assert table_result.item_id == 'in.c-test.test-table' - assert table_result.success is True - assert table_result.error is None - assert table_result.timestamp == parse_iso_timestamp('2024-01-01T00:00:00Z') - - keboola_client.storage_client.table_metadata_update.assert_called_once_with( - table_id='in.c-test.test-table', - metadata={MetadataField.DESCRIPTION: 'Updated table description'}, - columns_metadata={}, - ) - - -@pytest.mark.asyncio -async def test_update_descriptions_column_success( - mocker: MockerFixture, mcp_context_client, mock_update_column_description_response -) -> None: - """Test successful update of column description using update_descriptions.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.table_metadata_update = mocker.AsyncMock( - return_value=mock_update_column_description_response, - ) - - result = await update_descriptions( - ctx=mcp_context_client, - updates=[ - DescriptionUpdate(item_id='in.c-test.test-table.column_name', description='Updated column description') - ], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 1 - assert result.successful == 1 - assert result.failed == 0 - assert len(result.results) == 1 - - column_result = result.results[0] - assert column_result.item_id == 'in.c-test.test-table.column_name' - assert column_result.success is True - assert column_result.error is None - assert column_result.timestamp == parse_iso_timestamp('2024-01-01T00:00:00Z') - - keboola_client.storage_client.table_metadata_update.assert_called_once_with( - table_id='in.c-test.test-table', - columns_metadata={ - 'column_name': [ - {'key': MetadataField.DESCRIPTION, 'value': 'Updated column description', 'columnName': 'column_name'} - ] - }, - ) - - -@pytest.mark.asyncio -async def test_update_descriptions_mixed_types_success( - mocker: MockerFixture, - mcp_context_client, - mock_update_bucket_description_response, - mock_update_table_description_response, -) -> None: - """Test successful update of mixed types using update_descriptions.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.bucket_metadata_update = mocker.AsyncMock( - return_value=mock_update_bucket_description_response, - ) - keboola_client.storage_client.table_metadata_update = mocker.AsyncMock( - return_value=mock_update_table_description_response, - ) - - result = await update_descriptions( - ctx=mcp_context_client, - updates=[ - DescriptionUpdate(item_id='in.c-test-bucket', description='Updated bucket description'), - DescriptionUpdate(item_id='in.c-test.test-table', description='Updated table description'), - ], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 2 - assert result.successful == 2 - assert result.failed == 0 - assert len(result.results) == 2 - - # Check bucket result - bucket_result = next(r for r in result.results if r.item_id == 'in.c-test-bucket') - assert bucket_result.success is True - assert bucket_result.error is None - - # Check table result - table_result = next(r for r in result.results if r.item_id == 'in.c-test.test-table') - assert table_result.success is True - assert table_result.error is None - - # Verify API calls - keboola_client.storage_client.bucket_metadata_update.assert_called_once_with( - bucket_id='in.c-test-bucket', - metadata={MetadataField.DESCRIPTION: 'Updated bucket description'}, - ) - keboola_client.storage_client.table_metadata_update.assert_called_once_with( - table_id='in.c-test.test-table', - metadata={MetadataField.DESCRIPTION: 'Updated table description'}, - columns_metadata={}, - ) - - -@pytest.mark.asyncio -async def test_update_descriptions_invalid_path_error(mcp_context_client) -> None: - """Test that invalid paths are handled gracefully.""" - result = await update_descriptions( - ctx=mcp_context_client, - updates=[DescriptionUpdate(item_id='invalid-path', description='This should fail')], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 1 - assert result.successful == 0 - assert result.failed == 1 - assert len(result.results) == 1 - - error_result = result.results[0] - assert error_result.item_id == 'invalid-path' - assert error_result.success is False - assert error_result.error is not None - assert 'Invalid item_id format' in error_result.error - assert error_result.timestamp is None - - -@pytest.mark.asyncio -async def test_update_descriptions_api_error_handling(mocker: MockerFixture, mcp_context_client) -> None: - """Test that API errors are handled gracefully.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.bucket_metadata_update = mocker.AsyncMock() - keboola_client.storage_client.bucket_metadata_update.side_effect = httpx.HTTPStatusError( - message='API Error', request=AsyncMock(), response=httpx.Response(status_code=500) - ) - - result = await update_descriptions( - ctx=mcp_context_client, - updates=[DescriptionUpdate(item_id='in.c-test-bucket', description='This will fail')], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 1 - assert result.successful == 0 - assert result.failed == 1 - assert len(result.results) == 1 - - error_result = result.results[0] - assert error_result.item_id == 'in.c-test-bucket' - assert error_result.success is False - assert error_result.error is not None - assert error_result.timestamp is None - - -@pytest.mark.asyncio -async def test_update_descriptions_empty_updates(mcp_context_client) -> None: - """Test that empty updates dictionary is handled.""" - result = await update_descriptions( - ctx=mcp_context_client, - updates=[], - ) - - assert isinstance(result, UpdateDescriptionsOutput) - assert result.total_processed == 0 - assert result.successful == 0 - assert result.failed == 0 - assert len(result.results) == 0 - - @pytest.mark.asyncio - async def test_get_buckets_use_serializer(mocker): - # Ideally, we'd test the output of every tool, but the required mocking would be excessive. - # Here, we test only the 'get_buckets' tool. - # The test_server.TestServer.test_tools_have_serializer() test verifies that the same serializer is used - # for all tools. - # Therefore, all tools should produce compact JSON in their unstructured output. - cfg_dict = { - 'storage_token': '123-test-storage-token', - 'storage_api_url': 'https://connection.keboola.com', - 'transport': 'stdio', - } - config = Config.from_dict(cfg_dict) - - mocker.patch( - 'keboola_mcp_server.clients.base.KeboolaServiceClient.get', - return_value={'owner': {'id': '123'}}, - ) - mocker.patch( - 'keboola_mcp_server.clients.client.AsyncStorageClient.trigger_event', - return_value={}, - ) - mocker.patch( - 'keboola_mcp_server.clients.client.AsyncStorageClient.bucket_list', - return_value=[ - { - 'uri': 'https://connection.keboola.com/v2/storage/buckets/in.c-foo', - 'id': 'in.c-foo', - 'name': 'c-foo', - 'displayName': 'foo', - 'idBranch': 202, - 'stage': 'in', - 'description': '', - 'tables': 'https://connection.keboola.com/v2/storage/buckets/in.c-foo', - 'created': '2025-06-05T08:16:36+0200', - 'lastChangeDate': '2025-06-05T08:17:12+0200', - 'updated': None, - 'isReadOnly': False, - 'dataSizeBytes': 112233, - 'rowsCount': 987, - 'isMaintenance': False, - 'backend': 'snowflake', - 'sharing': None, - 'hasExternalSchema': False, - 'databaseName': '', - 'path': 'in.c-foo', - 'isSnowflakeSharedDatabase': False, - 'color': None, - 'owner': None, - 'backendPath': ['KEBOOLA_123', 'in.c-foo'], - 'attributes': [], - } - ], - ) - expected = GetBucketsOutput( - buckets=[ - BucketDetail( - id='in.c-foo', - name='c-foo', - display_name='foo', - description='', - stage='in', - created='2025-06-05T08:16:36+0200', - updated='2025-06-05T08:17:12+0200', - data_size_bytes=112233, - ) - ], - bucket_counts=BucketCounts(total_buckets=1, input_buckets=1, output_buckets=0), - links=[ - Link( - type='ui-dashboard', - title='Buckets in the project', - url='https://connection.keboola.com/admin/projects/123/storage', - ) - ], - ) - - server = create_server(config, runtime_info=ServerRuntimeInfo(transport='stdio')) - assert isinstance(server, FastMCP) - - async with Client(server) as client: - result = await client.call_tool('get_buckets') - # check the structured output - assert GetBucketsOutput.model_validate(result.structured_content) == expected - # check the unstructured output - assert len(result.content) == 1 - assert result.content[0] == TextContent( - type='text', - # no fields with None values, no indentation, no whitespace - text=json.dumps(expected.model_dump(exclude_none=True), ensure_ascii=False, separators=(',', ':')), - ) - - -# --- Storage-branches feature tests --- - - -def _get_sb_prod_buckets() -> list[dict[str, Any]]: - """Production buckets returned from branch/default/buckets for a storage-branches project.""" - return [ - { - 'id': 'in.c-shopify', - 'name': 'c-shopify', - 'displayName': 'shopify', - 'idBranch': 8653, - 'stage': 'in', - 'description': 'Shopify data.', - 'created': '2025-08-27T11:25:42+0200', - 'lastChangeDate': '2025-09-01T10:00:00+0200', - 'updated': None, - 'isReadOnly': False, - 'dataSizeBytes': 10000, - 'rowsCount': 100, - 'backend': 'snowflake', - 'path': 'in.c-shopify', - 'backendPath': ['KBC_USE4_3047', 'in.c-shopify'], - 'metadata': [], - }, - { - 'id': 'out.c-model', - 'name': 'c-model', - 'displayName': 'model', - 'idBranch': 8653, - 'stage': 'out', - 'description': 'Model output.', - 'created': '2025-08-27T12:00:00+0200', - 'lastChangeDate': '2025-09-01T10:00:00+0200', - 'updated': None, - 'isReadOnly': False, - 'dataSizeBytes': 5000, - 'rowsCount': 50, - 'backend': 'snowflake', - 'path': 'out.c-model', - 'backendPath': ['KBC_USE4_3047', 'out.c-model'], - 'metadata': [], - }, - ] - - -def _get_sb_branch_buckets(branch_id: str = '35403') -> list[dict[str, Any]]: - """Branched buckets returned from branch/{id}/buckets for a storage-branches project.""" - return [ - # Branched version of existing prod bucket (modified in branch) - { - 'id': 'out.c-model', - 'name': 'c-model', - 'displayName': 'model', - 'idBranch': int(branch_id), - 'stage': 'out', - 'description': 'Model output (branch).', - 'created': '2025-08-27T12:00:00+0200', - 'lastChangeDate': '2026-04-15T14:30:48+0200', - 'updated': None, - 'isReadOnly': False, - 'dataSizeBytes': 7000, - 'rowsCount': 70, - 'backend': 'snowflake', - 'path': f'{branch_id}_out.c-model', - 'backendPath': ['KBC_USE4_3047', f'{branch_id}_out.c-model'], - 'metadata': [ - {'id': '100', 'key': 'KBC.createdBy.branch.id', 'value': branch_id}, - ], - }, - # New bucket created only in the branch - { - 'id': 'out.c-new-data', - 'name': 'c-new-data', - 'displayName': 'new-data', - 'idBranch': int(branch_id), - 'stage': 'out', - 'description': 'New data bucket.', - 'created': '2026-04-15T14:33:02+0200', - 'lastChangeDate': '2026-04-15T14:33:09+0200', - 'updated': None, - 'isReadOnly': False, - 'dataSizeBytes': 3000, - 'rowsCount': 10, - 'backend': 'snowflake', - 'path': f'{branch_id}_out.c-new-data', - 'backendPath': ['KBC_USE4_3047', f'{branch_id}_out.c-new-data'], - 'metadata': [ - {'id': '200', 'key': 'KBC.createdBy.branch.id', 'value': branch_id}, - ], - }, - ] - - -@pytest.mark.asyncio -async def test_list_buckets_storage_branches(mocker: MockerFixture, mcp_context_client: Context) -> None: - """Test that _list_buckets merges production and branch buckets with storage-branches feature.""" - branch_id = '35403' - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = branch_id - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - - prod_buckets = _get_sb_prod_buckets() - branch_buckets = _get_sb_branch_buckets(branch_id) - - keboola_client.storage_client.bucket_list = mocker.AsyncMock( - side_effect=lambda include=None, branch_id=None: prod_buckets if branch_id == 'default' else branch_buckets - ) - - result = await get_buckets(mcp_context_client) - - assert isinstance(result, GetBucketsOutput) - # Should have 3 buckets: in.c-shopify (prod only), out.c-model (merged), out.c-new-data (branch only) - assert len(result.buckets) == 3 - assert result.bucket_counts.total_buckets == 3 - - buckets_by_id = {b.id: b for b in result.buckets} - - # Production-only bucket unchanged - assert 'in.c-shopify' in buckets_by_id - assert buckets_by_id['in.c-shopify'].data_size_bytes == 10000 - - # Merged bucket: branch version's data_size_bytes (not summed) - assert 'out.c-model' in buckets_by_id - assert buckets_by_id['out.c-model'].data_size_bytes == 7000 - - # Branch-only bucket presented with prod-like ID - assert 'out.c-new-data' in buckets_by_id - assert buckets_by_id['out.c-new-data'].data_size_bytes == 3000 - - # bucket_list called twice: once for default, once for branch - assert keboola_client.storage_client.bucket_list.call_count == 2 - - -@pytest.mark.asyncio -async def test_find_buckets_storage_branches(mocker: MockerFixture, mcp_context_client: Context) -> None: - """Test that _find_buckets uses branch-scoped endpoints with storage-branches feature.""" - branch_id = '35403' - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = branch_id - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - - prod_bucket = _get_sb_prod_buckets()[1] # out.c-model - branch_bucket = _get_sb_branch_buckets(branch_id)[0] # out.c-model (branch) - - keboola_client.storage_client.bucket_detail = mocker.AsyncMock( - side_effect=lambda bid, branch_id=None: ( - prod_bucket - if branch_id == 'default' - else ( - branch_bucket - if branch_id == '35403' - else (_ for _ in ()).throw( - httpx.HTTPStatusError( - message='Not found', request=AsyncMock(), response=httpx.Response(status_code=404) - ) - ) - ) - ) - ) - - result = await get_buckets(mcp_context_client, ['out.c-model']) - - assert isinstance(result, GetBucketsOutput) - assert len(result.buckets) == 1 - bucket = result.buckets[0] - assert bucket.id == 'out.c-model' - # With storage-branches, data_size_bytes should be the branch version - assert bucket.data_size_bytes == 7000 - - # bucket_detail called twice (default and branch) - assert keboola_client.storage_client.bucket_detail.call_count == 2 - - -@pytest.mark.asyncio -async def test_get_table_storage_branches(mocker: MockerFixture, mcp_context_client: Context) -> None: - """Test that _get_table uses branch-scoped endpoints with storage-branches feature.""" - branch_id = '35403' - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = branch_id - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - - branch_table = { - 'id': 'out.c-model.customers', - 'name': 'customers', - 'displayName': 'customers', - 'primaryKey': ['id'], - 'created': '2026-04-15T14:33:11+0200', - 'lastImportDate': '2026-04-15T14:33:11+0200', - 'lastChangeDate': '2026-04-15T14:33:11+0200', - 'rowsCount': 10, - 'dataSizeBytes': 3072, - 'isAlias': False, - 'columns': ['id', 'name'], - 'columnMetadata': {}, - 'metadata': [ - {'id': '300', 'key': 'KBC.createdBy.branch.id', 'value': branch_id}, - ], - 'bucket': { - 'id': 'out.c-model', - 'name': 'c-model', - 'idBranch': int(branch_id), - 'backendPath': ['KBC_USE4_3047', f'{branch_id}_out.c-model'], - }, - } - - def _table_detail_sb(tid: str, branch_id: str | None = None) -> JsonDict: - if tid == 'out.c-model.customers' and branch_id == '35403': - return branch_table - raise httpx.HTTPStatusError(message='Not found', request=AsyncMock(), response=httpx.Response(status_code=404)) - - keboola_client.storage_client.table_detail = mocker.AsyncMock(side_effect=_table_detail_sb) - - workspace_manager = WorkspaceManager.from_state(mcp_context_client.session.state) - workspace_manager.get_table_info = mocker.AsyncMock( - side_effect=lambda sapi_table: DbTableInfo( - id=sapi_table['id'], - fqn=TableFqn(db_name='SAPI_TEST', schema_name=sapi_table['id'].rsplit('.', 1)[0], table_name='customers'), - columns={ - 'id': DbColumnInfo(name='id', quoted_name='#id#', native_type='INT', nullable=False), - 'name': DbColumnInfo(name='name', quoted_name='#name#', native_type='VARCHAR', nullable=True), - }, - ) - ) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='Snowflake') - workspace_manager.get_quoted_name = mocker.AsyncMock(side_effect=lambda name: f'#{name}#') - - result = await get_tables(mcp_context_client, table_ids=['out.c-model.customers']) - - assert isinstance(result, GetTablesOutput) - assert len(result.tables) == 1 - table = result.tables[0] - # Table ID should be presented as production-like - assert table.id == 'out.c-model.customers' - assert table.branch_id is None - - # table_detail called with both default and branch endpoints - keboola_client.storage_client.table_detail.assert_has_calls( - [call('out.c-model.customers', branch_id='default'), call('out.c-model.customers', branch_id=branch_id)] - ) diff --git a/tests/tools/storage/test_usage.py b/tests/tools/storage/test_usage.py deleted file mode 100644 index fe7d883e8..000000000 --- a/tests/tools/storage/test_usage.py +++ /dev/null @@ -1,244 +0,0 @@ -from typing import Any, Mapping, Sequence - -import pytest -from pytest_mock import MockerFixture - -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.tools.search import PatternMatch, SearchHit -from keboola_mcp_server.tools.storage import usage as storage_usage - - -def _sorted_usage(output: Sequence[storage_usage.UsageById]) -> list[storage_usage.UsageById]: - return sorted(output, key=lambda item: item.target_id) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('hits', 'expected'), - [ - ( - [ - SearchHit( - component_id='keboola.ex-db', - configuration_id='cfg-1', - item_type='configuration', - updated='2024-01-01T00:00:00Z', - name='Config 1', - ).set_matches([PatternMatch(scope='storage.input', patterns=['id-1', 'id-2'])]), - SearchHit( - component_id='keboola.ex-db', - configuration_id='cfg-2', - item_type='configuration', - updated='2024-01-02T00:00:00Z', - name='Config 2', - ).set_matches([PatternMatch(scope='storage.output', patterns=['id-1'])]), - ], - { - 'id-1': [ - { - 'component_id': 'keboola.ex-db', - 'configuration_id': 'cfg-1', - 'configuration_row_id': None, - 'configuration_name': 'Config 1', - 'used_in': 'storage.input', - 'timestamp': '2024-01-01T00:00:00Z', - }, - { - 'component_id': 'keboola.ex-db', - 'configuration_id': 'cfg-2', - 'configuration_row_id': None, - 'configuration_name': 'Config 2', - 'used_in': 'storage.output', - 'timestamp': '2024-01-02T00:00:00Z', - }, - ], - 'id-2': [ - { - 'component_id': 'keboola.ex-db', - 'configuration_id': 'cfg-1', - 'configuration_row_id': None, - 'configuration_name': 'Config 1', - 'used_in': 'storage.input', - 'timestamp': '2024-01-01T00:00:00Z', - } - ], - }, - ), - ([], {}), - ], - ids=['grouped_usage', 'empty_hits'], -) -async def test_find_id_usage_groups_matches( - mocker: MockerFixture, hits: list[SearchHit], expected: dict[str, list[dict[str, Any]]] -) -> None: - mocker.patch.object(storage_usage, 'fetch_configurations', autospec=True, return_value=hits) - - client = mocker.Mock() - output = await storage_usage.find_id_usage( - client, target_ids=['id-1', 'id-2'], scopes=('storage.input', 'storage.output') - ) - output_sorted = _sorted_usage(output) - - output_map = {item.target_id: [ref.model_dump() for ref in item.usage_references] for item in output_sorted} - assert output_map == expected - - -@pytest.mark.parametrize( - ('metadata', 'expected'), - [ - ([], None), - ( - [ - { - 'key': MetadataField.CREATED_BY_COMPONENT_ID, - 'value': 'keboola.ex-db', - 'timestamp': '2024-01-01T00:00:00Z', - }, - { - 'key': MetadataField.CREATED_BY_CONFIGURATION_ID, - 'value': 'cfg-1', - 'timestamp': '2024-01-02T00:00:00Z', - }, - { - 'key': MetadataField.CREATED_BY_CONFIGURATION_ROW_ID, - 'value': 'row-1', - 'timestamp': '2024-01-03T00:00:00Z', - }, - ], - { - 'component_id': 'keboola.ex-db', - 'configuration_id': 'cfg-1', - 'configuration_row_id': 'row-1', - 'used_in': None, - 'timestamp': '2024-01-03T00:00:00Z', - }, - ), - ( - [ - { - 'key': MetadataField.CREATED_BY_CONFIGURATION_ID, - 'value': 'cfg-1', - 'timestamp': '2024-01-02T00:00:00Z', - }, - ], - None, - ), - ], - ids=['empty', 'complete', 'missing_component'], -) -def test_get_created_by(metadata: list[Mapping[str, Any]], expected: dict[str, Any] | None) -> None: - result = storage_usage.get_created_by(metadata) - assert result.model_dump() if result else None is expected - - -@pytest.mark.parametrize( - ('metadata', 'expected'), - [ - ([], None), - ( - [ - { - 'key': MetadataField.UPDATED_BY_COMPONENT_ID, - 'value': 'keboola.ex-db', - 'timestamp': '2024-01-01T00:00:00Z', - }, - { - 'key': MetadataField.UPDATED_BY_CONFIGURATION_ID, - 'value': 'cfg-1', - 'timestamp': '2024-01-02T00:00:00Z', - }, - { - 'key': MetadataField.UPDATED_BY_CONFIGURATION_ROW_ID, - 'value': 'row-1', - 'timestamp': '2024-01-03T00:00:00Z', - }, - ], - { - 'component_id': 'keboola.ex-db', - 'configuration_id': 'cfg-1', - 'configuration_row_id': 'row-1', - 'used_in': None, - 'timestamp': '2024-01-03T00:00:00Z', - }, - ), - ( - [ - { - 'key': MetadataField.UPDATED_BY_COMPONENT_ID, - 'value': 'keboola.ex-db', - 'timestamp': '2024-01-01T00:00:00Z', - }, - { - 'key': MetadataField.UPDATED_BY_CONFIGURATION_ID, - 'value': 'cfg-1', - 'timestamp': '2024-01-02T00:00:00Z', - }, - ], - { - 'component_id': 'keboola.ex-db', - 'configuration_id': 'cfg-1', - 'configuration_row_id': None, - 'used_in': None, - 'timestamp': '2024-01-02T00:00:00Z', - }, - ), - ( - [ - { - 'key': MetadataField.UPDATED_BY_CONFIGURATION_ID, - 'value': 'cfg-1', - 'timestamp': '2024-01-02T00:00:00Z', - }, - ], - None, - ), - ], - ids=['empty', 'complete-config-row', 'complete-config', 'missing_component'], -) -def test_get_last_updated_by(metadata: list[Mapping[str, Any]], expected: dict[str, Any] | None) -> None: - result = storage_usage.get_last_updated_by(metadata) - assert result.model_dump() if result else None is expected - - -@pytest.mark.parametrize( - ('metadata', 'expected'), - [ - ({'metadata': [{'key': 'a', 'value': '1'}]}, [{'key': 'a', 'value': '1'}]), - ({'key': 'a', 'value': '1'}, [{'key': 'a', 'value': '1'}]), - ({'metadata': ['bad']}, []), - ({'metadata': [{'key': 'a', 'value': '1'}, 'bad']}, [{'key': 'a', 'value': '1'}]), - ([{'key': 'a', 'value': '1'}, 'bad'], [{'key': 'a', 'value': '1'}]), - ], - ids=['metadata_list', 'single_item', 'invalid_metadata_list', 'mixed_metadata_list', 'mixed_list'], -) -def test_coerce_metadata_list(metadata: Any, expected: list[Mapping[str, Any]]) -> None: - assert storage_usage._coerce_metadata_list(metadata) == expected - - -@pytest.mark.parametrize( - ('metadata', 'keys', 'expected'), - [ - ([], [MetadataField.CREATED_BY_COMPONENT_ID], None), - ( - [ - {'key': MetadataField.CREATED_BY_COMPONENT_ID, 'timestamp': '2024-01-01T00:00:00Z'}, - {'key': MetadataField.CREATED_BY_COMPONENT_ID, 'timestamp': '2024-01-02T00:00:00Z'}, - ], - [MetadataField.CREATED_BY_COMPONENT_ID], - '2024-01-02T00:00:00Z', - ), - ( - [ - {'key': MetadataField.CREATED_BY_COMPONENT_ID, 'timestamp': '2024-01-01T00:00:00Z'}, - {'key': MetadataField.CREATED_BY_CONFIGURATION_ID, 'timestamp': '2024-01-02T00:00:00Z'}, - ], - [MetadataField.CREATED_BY_CONFIGURATION_ID], - '2024-01-02T00:00:00Z', - ), - ], - ids=['empty', 'latest', 'filter_keys'], -) -def test_get_latest_metadata_timestamp( - metadata: list[Mapping[str, Any]], keys: Sequence[str], expected: str | None -) -> None: - assert storage_usage._get_latest_metadata_timestamp(metadata, keys) == expected diff --git a/tests/tools/test_data_apps.py b/tests/tools/test_data_apps.py deleted file mode 100644 index 0289f436e..000000000 --- a/tests/tools/test_data_apps.py +++ /dev/null @@ -1,2907 +0,0 @@ -import sys -from types import ModuleType -from typing import Literal, cast - -import pytest -from fastmcp import Context - -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import DATA_APP_COMPONENT_ID, KeboolaClient -from keboola_mcp_server.clients.data_science import AppRunResponse, DataAppConfig, DataAppResponse -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.data_apps import ( - _APP_RUN_LOG_LINES, - _APP_RUN_MESSAGE_LIMIT, - _QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE, - _STORAGE_QUERY_DATA_FUNCTION_CODE, - MAX_DNS_LABEL_LENGTH, - AppRunInfo, - DataApp, - DataAppSlugTooLongError, - DataAppSummary, - ModifiedDataAppOutput, - _build_data_app_config, - _fetch_data_app, - _fetch_latest_run, - _get_authorization, - _get_data_app_slug, - _get_query_function_code, - _get_secrets, - _inject_query_to_source_code, - _prune_empty_storage_objects, - _update_existing_data_app_config, - _uses_basic_authentication, - deploy_data_app, - get_data_apps, - modify_streamlit_data_app, -) - - -@pytest.fixture -def data_app() -> DataApp: - return DataApp( - name='test', - component_id='test', - configuration_id='test', - data_app_id='test', - project_id='test', - branch_id='test', - config_version='test', - type='test', - auto_suspend_after_seconds=3600, - configuration={}, - state='test', - ) - - -def _make_data_app_response( - component_id: str = DATA_APP_COMPONENT_ID, - data_app_id: str = 'app-123', - config_id: str = 'cfg-123', -) -> DataAppResponse: - """Helper to create a DataAppResponse with sensible defaults.""" - return DataAppResponse( - id=data_app_id, - project_id='proj-1', - component_id=component_id, - branch_id='branch-1', - config_id=config_id, - config_version='1', - type='streamlit', - state='running', - desired_state='running', - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('current_state', 'action', 'error_match'), - [ - ('starting', 'stop', 'Data app is currently "starting", could not be stopped at the moment.'), - ('restarting', 'stop', 'Data app is currently "starting", could not be stopped at the moment.'), - ('stopping', 'deploy', 'Data app is currently "stopping", could not be started at the moment.'), - ], -) -async def test_deploy_data_app_when_current_state_contradicts_with_action( - mocker, - data_app: DataApp, - current_state: str, - action: Literal['deploy', 'stop'], - error_match: str, - mcp_context_client: Context, -) -> None: - """call deploy_data_app with mocked data_app and given state expecting ValueError with proper error message.""" - data_app.state = current_state - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', return_value=data_app) - with pytest.raises(ValueError, match=error_match): - await deploy_data_app( - ctx=mcp_context_client, action=cast(Literal['deploy', 'stop'], action), configuration_id='cfg-123' - ) - - -def test_get_data_app_slug(): - assert _get_data_app_slug('My Cool App') == 'my-cool-app' - assert _get_data_app_slug('App 123') == 'app-123' - assert _get_data_app_slug('Weird!@# Name$$$') == 'weird-name' - - -@pytest.mark.parametrize( - ('name', 'expected_slug', 'expected_error'), - [ - pytest.param('a' * MAX_DNS_LABEL_LENGTH, 'a' * MAX_DNS_LABEL_LENGTH, None, id='at_max_length'), - pytest.param('a' * (MAX_DNS_LABEL_LENGTH + 1), None, DataAppSlugTooLongError, id='exceeds_max_length'), - pytest.param('a' * 70 + '!!!', None, DataAppSlugTooLongError, id='long_name_with_special_chars'), - pytest.param('a' * 30 + '!' * 50 + 'b' * 30, 'a' * 30 + 'b' * 30, None, id='shortened_by_special_chars'), - ], -) -def test_get_data_app_slug_length_validation(name, expected_slug, expected_error): - """Test DNS label length validation in slug generation.""" - if expected_error: - with pytest.raises(expected_error): - _get_data_app_slug(name) - else: - slug = _get_data_app_slug(name) - assert slug == expected_slug - - -def test_get_authorization_mapping(): - auth_true = _get_authorization(True) - assert auth_true['app_proxy']['auth_providers'] == [{'id': 'simpleAuth', 'type': 'password'}] - assert auth_true['app_proxy']['auth_rules'] == [ - {'type': 'pathPrefix', 'value': '/', 'auth_required': True, 'auth': ['simpleAuth']} - ] - - auth_false = _get_authorization(False) - assert auth_false['app_proxy']['auth_providers'] == [] - assert auth_false['app_proxy']['auth_rules'] == [{'type': 'pathPrefix', 'value': '/', 'auth_required': False}] - - -def test_is_authorized_behavior(): - assert _uses_basic_authentication(_get_authorization(True)) is True - assert _uses_basic_authentication(_get_authorization(False)) is False - - -def test_inject_query_to_source_code_when_already_included(): - query_code = _STORAGE_QUERY_DATA_FUNCTION_CODE - backend = 'bigquery' - source_code = f"""prelude{query_code}postlude""" - result = _inject_query_to_source_code(source_code, backend) - assert result == source_code - - -def test_inject_query_to_source_code_with_markers(): - src = ( - 'import pandas as pd\n\n' - '# ### INJECTED_CODE ####\n' - '# will be replaced\n' - '# ### END_OF_INJECTED_CODE ####\n\n' - "print('hello')\n" - ) - backend = 'bigquery' - query_code = _STORAGE_QUERY_DATA_FUNCTION_CODE - result = _inject_query_to_source_code(src, backend) - - assert result.startswith('import pandas as pd') - assert query_code in result - assert result.endswith("print('hello')\n") - - -def test_inject_query_to_source_code_with_placeholder(): - src = 'header\n{QUERY_DATA_FUNCTION}\nfooter\n' - query_code = _QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE - backend = 'snowflake' - result = _inject_query_to_source_code(src, backend) - - # Injected once via format(), original source (with placeholder) appended afterwards - assert query_code in result - assert '{QUERY_DATA_FUNCTION}' not in result - assert result.startswith('header') - assert result.strip().endswith('footer') - - -def test_inject_query_to_source_code_default_path(): - src = "print('x')\n" - query_code = _QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE - backend = 'snowflake' - result = _inject_query_to_source_code(src, backend) - assert result.startswith(query_code) - assert result.endswith(src) - - -def _load_query_data_function(code: str, result_pages: list[JsonDict], mocker): - """Load injected query_data code with mocked httpx/pandas modules for isolated testing.""" - calls: JsonDict = {'get': [], 'post': []} - result_pages = [page.copy() for page in result_pages] - - class FakeResponse: - def __init__(self, payload: JsonDict) -> None: - self._payload = payload - - def raise_for_status(self) -> None: - return None - - def json(self) -> JsonDict: - return self._payload - - class FakeClient: - def __init__(self, *, timeout, limits) -> None: - self.timeout = timeout - self.limits = limits - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc, tb) -> None: - return None - - def post(self, url: str, json: JsonDict, headers: JsonDict) -> FakeResponse: - calls['post'].append({'url': url, 'json': json, 'headers': headers}) - return FakeResponse({'queryJobId': 'job-1'}) - - def get(self, url: str, headers: JsonDict, params: JsonDict | None = None) -> FakeResponse: - calls['get'].append({'url': url, 'headers': headers, 'params': params}) - if url.endswith('/queries/job-1'): - return FakeResponse({'status': 'completed', 'statements': [{'id': 'stmt-1'}]}) - if url.endswith('/results'): - return FakeResponse(result_pages.pop(0)) - raise AssertionError(f'Unexpected GET URL: {url}') - - httpx_module = ModuleType('httpx') - httpx_module.Timeout = lambda **kwargs: kwargs - httpx_module.Limits = lambda **kwargs: kwargs - httpx_module.Client = FakeClient - - pandas_module = ModuleType('pandas') - pandas_module.DataFrame = lambda rows: rows - - mocker.patch.dict(sys.modules, {'httpx': httpx_module, 'pandas': pandas_module}) - - namespace: dict[str, object] = {} - exec(code, namespace) - return namespace['query_data'], calls - - -def test_query_service_query_data_paginates_results(mocker, monkeypatch) -> None: - query_data, calls = _load_query_data_function( - _QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE, - [ - { - 'status': 'completed', - 'columns': [{'name': 'id'}], - 'numberOfRows': 3, - 'data': [['1'], ['2']], - }, - { - 'status': 'completed', - 'columns': [{'name': 'id'}], - 'numberOfRows': 3, - 'data': [['3']], - }, - ], - mocker, - ) - monkeypatch.setenv('BRANCH_ID', '123') - monkeypatch.setenv('WORKSPACE_ID', '456') - monkeypatch.setenv('KBC_TOKEN', 'test-token') - monkeypatch.setenv('KBC_URL', 'https://connection.keboola.com') - - result = query_data('SELECT * FROM test') - - assert result == [{'id': '1'}, {'id': '2'}, {'id': '3'}] - result_calls = [call for call in calls['get'] if call['url'].endswith('/results')] - assert len(result_calls) == 2 - assert result_calls[0]['params']['offset'] == 0 - assert result_calls[1]['params']['offset'] == 2 - assert 'pageSize' in result_calls[0]['params'] - - -def test_query_service_query_data_stops_on_short_page_without_total_count(mocker, monkeypatch) -> None: - query_data, calls = _load_query_data_function( - _QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE, - [ - { - 'status': 'completed', - 'columns': [{'name': 'id'}], - 'data': [['1'], ['2']], - } - ], - mocker, - ) - monkeypatch.setenv('BRANCH_ID', '123') - monkeypatch.setenv('WORKSPACE_ID', '456') - monkeypatch.setenv('KBC_TOKEN', 'test-token') - monkeypatch.setenv('KBC_URL', 'https://connection.keboola.com') - - result = query_data('SELECT * FROM test') - - assert result == [{'id': '1'}, {'id': '2'}] - result_calls = [call for call in calls['get'] if call['url'].endswith('/results')] - assert len(result_calls) == 1 - - -def test_build_data_app_config_merges_defaults_and_secrets(): - name = 'My App' - src = "print('hello')" - pkgs = ['pandas'] - secrets = {'FOO': 'bar'} - backend = 'snowflake' - - config = _build_data_app_config(name, src, pkgs, 'basic-auth', secrets, backend) - - params = config['parameters'] - assert params['dataApp']['slug'] == 'my-app' - assert params['script'] == [_inject_query_to_source_code(src, backend)] - # Default packages are included and deduplicated - assert 'pandas' in params['packages'] - assert 'httpx' in params['packages'] - # Secrets carried over - assert params['dataApp']['secrets'] == secrets - # Authentication reflects flag - assert config['authorization'] == _get_authorization(True) - - -@pytest.mark.parametrize( - ('block', 'expected'), - [ - # An empty storage block collapses to empty so the caller omits the `storage` key. - ({}, {}), - # Empty `input`/`output` objects are dropped — PHP would serialize them as `[]` and the - # mapping editor (Writable Tables) then silently fails to add entries (AI-3135). - ({'input': {}, 'output': {}}, {}), - # The canonical empty state uses empty *arrays*, which must be preserved verbatim. - ({'output': {'tables': []}}, {'output': {'tables': []}}), - # Mixed: drop the empty `input`, keep the populated `output`. - ( - { - 'input': {}, - 'output': {'tables': [{'destination': 'in.c-main.t', 'unload_strategy': 'direct-grant'}]}, - }, - {'output': {'tables': [{'destination': 'in.c-main.t', 'unload_strategy': 'direct-grant'}]}}, - ), - # Nested empty objects inside a table entry are pruned; scalars and arrays survive. - ( - {'output': {'tables': [{'destination': 'in.c-main.t', 'table_metadata': {}}]}}, - {'output': {'tables': [{'destination': 'in.c-main.t'}]}}, - ), - ], -) -def test_prune_empty_storage_objects(block, expected) -> None: - """Empty objects are stripped (they collapse to `[]` server-side); empty arrays are kept.""" - assert _prune_empty_storage_objects(block) == expected - - -def test_build_data_app_config_create_omits_empty_storage() -> None: - """Streamlit create must not persist an empty `storage` object (it becomes `[]` server-side, AI-3135).""" - config = _build_data_app_config('My App', "print('hi')", [], 'no-auth', {}, 'snowflake') - serialized = DataAppConfig.model_validate(config).model_dump(by_alias=True, exclude_none=True) - assert 'storage' not in serialized - - -def test_update_existing_data_app_config(): - existing = { - 'parameters': { - 'dataApp': { - 'slug': 'old-slug', - 'secrets': {'FOO': 'old', 'KEEP': 'x'}, - }, - 'script': ['old'], - 'packages': ['numpy'], - }, - 'authorization': {}, - } - - new = _update_existing_data_app_config( - existing_config=existing, - name='New Name', - source_code='new-code', - packages=['pandas'], - authentication_type='basic-auth', - secrets={'FOO': 'new', 'NEW': 'y'}, - sql_dialect='snowflake', - ) - - assert new['parameters']['dataApp']['slug'] == 'new-name' - assert new['parameters']['script'] == [_inject_query_to_source_code('new-code', 'snowflake')] - # Removed previous packages - assert 'numpy' not in new['parameters']['packages'] - # Packages combined with defaults - assert sorted(new['parameters']['packages']) == sorted(['pandas', 'httpx']) - # Secrets merged - assert new['parameters']['dataApp']['secrets'] == {'FOO': 'old', 'KEEP': 'x', 'NEW': 'y'} - # Authentication updated - assert new['authorization'] == _get_authorization(True) - - -def test_update_existing_data_app_config_preserves_existing_secrets(): - existing = { - 'parameters': { - 'dataApp': { - 'slug': 'old-slug', - 'secrets': { - 'WORKSPACE_ID': 'wid-old', - 'BRANCH_ID': 'branch-old', - 'KEEP': 'x', - }, - }, - 'script': ['old'], - 'packages': ['numpy'], - }, - 'authorization': {}, - } - - new = _update_existing_data_app_config( - existing_config=existing, - name='New Name', - source_code='new-code', - packages=['pandas'], - authentication_type='basic-auth', - secrets={'WORKSPACE_ID': 'wid-new', 'BRANCH_ID': 'branch-new', 'NEW': 'y'}, - sql_dialect='snowflake', - ) - - assert new['parameters']['dataApp']['secrets'] == { - 'WORKSPACE_ID': 'wid-old', - 'BRANCH_ID': 'branch-old', - 'KEEP': 'x', - 'NEW': 'y', - } - - -def test_get_secrets(): - secrets = _get_secrets( - workspace_id='wid-1234', - branch_id='123', - ) - assert secrets == { - 'WORKSPACE_ID': 'wid-1234', - 'BRANCH_ID': '123', - } - - -def test_update_existing_data_app_config_keeps_previous_properties_when_undefined(): - existing_authorization = { - 'app_proxy': { - 'auth_providers': [{'id': 'oidc', 'type': 'oidc', 'issuer_url': 'https://issuer'}], - 'auth_rules': [{'type': 'pathPrefix', 'value': '/', 'auth_required': True, 'auth': ['oidc']}], - } - } - existing = { - 'parameters': { - 'dataApp': { - 'slug': 'old-slug', - 'secrets': {'KEEP': 'secret'}, - }, - 'script': ['old'], - 'packages': ['numpy'], - }, - 'authorization': existing_authorization, - } - - new = _update_existing_data_app_config( - existing_config=existing, - name='', - source_code='', - packages=[], - authentication_type='default', - secrets={}, - sql_dialect='snowflake', - ) - - # Deepcopy makes it equal-but-not-identical. - assert new['authorization'] == existing_authorization - assert new['parameters']['script'] == ['old'] - # verify the rest of the config is still updated - assert new['parameters']['dataApp']['slug'] == 'old-slug' - assert 'numpy' in new['parameters']['packages'] - assert 'httpx' in new['parameters']['packages'] - assert new['parameters']['dataApp']['secrets']['KEEP'] == 'secret' - - -@pytest.mark.parametrize( - ('existing_storage', 'storage_key_present', 'expected_storage'), - [ - # Leftover empty object (serialized as `[]` server-side) is removed on re-save (AI-3135). - ({}, False, None), - # Leftover array-shaped storage is removed. - ([], False, None), - # Empty `input`/`output` containers are pruned away to nothing -> key removed. - ({'input': {}, 'output': {}}, False, None), - # A valid storage block is preserved untouched. - ( - {'output': {'tables': [{'destination': 'in.c-main.t', 'unload_strategy': 'direct-grant'}]}}, - True, - {'output': {'tables': [{'destination': 'in.c-main.t', 'unload_strategy': 'direct-grant'}]}}, - ), - ], -) -def test_update_existing_data_app_config_normalizes_storage( - existing_storage, storage_key_present, expected_storage -) -> None: - """Streamlit re-save repairs a broken/empty leftover `storage` shape instead of preserving it.""" - existing = { - 'parameters': {'dataApp': {'slug': 'x', 'secrets': {}}, 'script': ['old'], 'packages': []}, - 'storage': existing_storage, - } - new = _update_existing_data_app_config( - existing_config=existing, - name='', - source_code='', - packages=[], - authentication_type='default', - secrets={}, - sql_dialect='snowflake', - ) - assert ('storage' in new) is storage_key_present - if storage_key_present: - assert new['storage'] == expected_storage - - -def test_update_existing_data_app_config_no_authorization_key(): - """Existing configs that lack an `authorization` key must not crash when authentication_type='default'.""" - existing = { - 'parameters': { - 'dataApp': {'slug': 'x', 'secrets': {}}, - 'script': ['old'], - 'packages': [], - }, - } - new = _update_existing_data_app_config( - existing_config=existing, - name='', - source_code='', - packages=[], - authentication_type='default', - secrets={}, - sql_dialect='snowflake', - ) - assert 'authorization' not in new - - -def test_update_existing_data_app_config_basic_auth_overwrites_oidc(): - """Explicit 'basic-auth' must replace an existing OIDC block.""" - existing = { - 'parameters': { - 'dataApp': {'slug': 'x', 'secrets': {}}, - 'script': ['old'], - 'packages': [], - }, - 'authorization': { - 'app_proxy': { - 'auth_providers': [{'id': 'oidc', 'type': 'oidc'}], - 'auth_rules': [{'type': 'pathPrefix', 'value': '/', 'auth_required': True, 'auth': ['oidc']}], - } - }, - } - new = _update_existing_data_app_config( - existing_config=existing, - name='', - source_code='', - packages=[], - authentication_type='basic-auth', - secrets={}, - sql_dialect='snowflake', - ) - assert new['authorization']['app_proxy']['auth_rules'] == [ - {'type': 'pathPrefix', 'value': '/', 'auth_required': True, 'auth': ['simpleAuth']} - ] - - -def test_get_query_function_code_selects_snippets(): - assert _get_query_function_code('snowflake') == _QUERY_SERVICE_QUERY_DATA_FUNCTION_CODE - assert _get_query_function_code('bigquery') == _STORAGE_QUERY_DATA_FUNCTION_CODE - with pytest.raises(ValueError, match='Unsupported SQL dialect'): - _get_query_function_code('UNKNOWN') - - -@pytest.mark.parametrize( - 'values', - [ - { - 'type': 'streamlit', - 'state': 'created', - }, - { - 'type': 'streamlit', - 'state': 'running', - }, - { - 'type': 'streamlit', - 'state': 'stopped', - }, - { - 'type': 'something else', - 'state': 'something else', - }, - ], -) -def test_data_app_summary_from_dict_minimal(values: JsonDict) -> None: - """Test creating DataAppSummary from dict with required fields.""" - data_app = { - 'component_id': 'comp-1', - 'configuration_id': 'cfg-1', - 'data_app_id': 'app-1', - 'project_id': 'proj-1', - 'branch_id': 'branch-1', - 'config_version': 'v1', - 'deployment_url': 'https://example.com/app', - 'auto_suspend_after_seconds': 3600, - } - data_app.update(values) - model = DataAppSummary.model_validate(data_app) - assert model.component_id == 'comp-1' - assert model.configuration_id == 'cfg-1' - assert model.state == values['state'] - assert model.type == values['type'] - assert model.deployment_url == 'https://example.com/app' - assert model.auto_suspend_after_seconds == 3600 - - -class TestGetDataAppsFiltering: - """Tests for get_data_apps filtering behavior by component_id.""" - - @pytest.mark.asyncio - async def test_get_data_apps_filters_by_component_id(self, mocker, mcp_context_client: Context) -> None: - """When listing data apps, only apps with DATA_APP_COMPONENT_ID are returned.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - # Mock list_data_apps to return apps with different component_ids - keboola_client.data_science_client.list_data_apps = mocker.AsyncMock( - return_value=[ - _make_data_app_response(component_id=DATA_APP_COMPONENT_ID, data_app_id='app-1'), - _make_data_app_response(component_id='keboola.sandboxes', data_app_id='app-2'), - _make_data_app_response(component_id=DATA_APP_COMPONENT_ID, data_app_id='app-3'), - _make_data_app_response(component_id='other.component', data_app_id='app-4'), - ] - ) - - # Mock ProjectLinksManager - mock_link = Link(type='ui-dashboard', title='Data Apps', url='https://example.com/data-apps') - mocker.patch( - 'keboola_mcp_server.tools.data_apps.ProjectLinksManager.from_client', - return_value=mocker.AsyncMock(get_data_app_dashboard_link=mocker.MagicMock(return_value=mock_link)), - ) - - result = await get_data_apps(ctx=mcp_context_client) - - # Only apps with DATA_APP_COMPONENT_ID should be returned - assert len(result.data_apps) == 2 - data_app_ids = [app.data_app_id for app in result.data_apps] - assert 'app-1' in data_app_ids - assert 'app-3' in data_app_ids - assert 'app-2' not in data_app_ids - assert 'app-4' not in data_app_ids - - @pytest.mark.asyncio - async def test_get_data_apps_returns_empty_when_no_matching_apps(self, mocker, mcp_context_client: Context) -> None: - """When no apps match DATA_APP_COMPONENT_ID, an empty list is returned.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - # Mock list_data_apps to return apps with different component_ids - keboola_client.data_science_client.list_data_apps = mocker.AsyncMock( - return_value=[ - _make_data_app_response(component_id='keboola.sandboxes', data_app_id='app-1'), - _make_data_app_response(component_id='other.component', data_app_id='app-2'), - ] - ) - - mock_link = Link(type='ui-dashboard', title='Data Apps', url='https://example.com/data-apps') - mocker.patch( - 'keboola_mcp_server.tools.data_apps.ProjectLinksManager.from_client', - return_value=mocker.AsyncMock(get_data_app_dashboard_link=mocker.MagicMock(return_value=mock_link)), - ) - - result = await get_data_apps(ctx=mcp_context_client) - - assert len(result.data_apps) == 0 - - -class TestFetchDataAppValidation: - """Tests for _fetch_data_app component_id validation.""" - - @pytest.mark.asyncio - async def test_fetch_data_app_by_data_app_id_validates_component_id( - self, mocker, keboola_client: KeboolaClient - ) -> None: - """When fetching by data_app_id, raises ValueError if component_id doesn't match.""" - wrong_component_id = 'keboola.sandboxes' - data_app_id = 'app-123' - - keboola_client.data_science_client.get_data_app = mocker.AsyncMock( - return_value=_make_data_app_response(component_id=wrong_component_id, data_app_id=data_app_id) - ) - - with pytest.raises(ValueError, match=f'Data app tools only support {DATA_APP_COMPONENT_ID} component'): - await _fetch_data_app(keboola_client, data_app_id=data_app_id, configuration_id=None) - - @pytest.mark.asyncio - async def test_fetch_data_app_by_configuration_id_validates_component_id( - self, mocker, keboola_client: KeboolaClient - ) -> None: - """When fetching by configuration_id, raises ValueError if component_id doesn't match.""" - wrong_component_id = 'keboola.sandboxes' - configuration_id = 'cfg-123' - data_app_id = 'app-123' - - # Mock configuration_detail to return valid config - keboola_client.storage_client.configuration_detail = mocker.AsyncMock( - return_value={ - 'id': configuration_id, - 'name': 'test-app', - 'description': 'test', - 'configuration': {'parameters': {'id': data_app_id}}, - 'version': 1, - } - ) - - # Mock get_data_app to return app with wrong component_id - keboola_client.data_science_client.get_data_app = mocker.AsyncMock( - return_value=_make_data_app_response( - component_id=wrong_component_id, data_app_id=data_app_id, config_id=configuration_id - ) - ) - - with pytest.raises(ValueError, match=f'Data app tools only support {DATA_APP_COMPONENT_ID} component'): - await _fetch_data_app(keboola_client, data_app_id=None, configuration_id=configuration_id) - - @pytest.mark.asyncio - async def test_fetch_data_app_by_data_app_id_succeeds_with_correct_component( - self, mocker, keboola_client: KeboolaClient - ) -> None: - """When component_id matches DATA_APP_COMPONENT_ID, fetch succeeds.""" - data_app_id = 'app-123' - config_id = 'cfg-123' - - data_app_response = _make_data_app_response( - component_id=DATA_APP_COMPONENT_ID, data_app_id=data_app_id, config_id=config_id - ) - - keboola_client.data_science_client.get_data_app = mocker.AsyncMock(return_value=data_app_response) - keboola_client.storage_client.configuration_detail = mocker.AsyncMock( - return_value={ - 'id': config_id, - 'name': 'test-app', - 'description': 'test', - 'configuration': {'parameters': {'id': data_app_id}, 'authorization': {}, 'storage': {}}, - 'version': 1, - } - ) - - result = await _fetch_data_app(keboola_client, data_app_id=data_app_id, configuration_id=None) - - assert result.data_app_id == data_app_id - assert result.component_id == DATA_APP_COMPONENT_ID - - @pytest.mark.asyncio - async def test_fetch_data_app_requires_either_id(self, keboola_client: KeboolaClient) -> None: - """When neither data_app_id nor configuration_id is provided, raises ValueError.""" - with pytest.raises(ValueError, match='Either data_app_id or configuration_id must be provided'): - await _fetch_data_app(keboola_client, data_app_id=None, configuration_id=None) - - -# ============================================================================= -# FOLDER METADATA TESTS -# ============================================================================= - - -@pytest.mark.parametrize( - ( - 'configuration_id', - 'folder', - 'app_count', - 'app_folders', - 'expect_folder_metadata', - 'expect_folder_delete', - 'expect_hint', - ), - [ - # Create path (no configuration_id) - ('', 'Analytics', 0, [], True, False, False), - ('', ' Analytics ', 0, [], True, False, False), # whitespace stripped - ('', None, 5, [], False, False, False), - ('', None, 25, ['Analytics'], False, False, True), - # Update path (with configuration_id) - ('cfg-1', 'Analytics', 0, [], True, False, False), - ('cfg-1', None, 5, [], False, False, False), - ('cfg-1', None, 25, ['Analytics'], False, False, True), - ('cfg-1', '', 5, [], False, True, False), # empty string → delete - ], - ids=[ - 'create_folder_provided', - 'create_folder_whitespace_stripped', - 'create_no_folder_few', - 'create_no_folder_many_with_hint', - 'update_folder_provided', - 'update_no_folder_few', - 'update_no_folder_many_with_hint', - 'update_folder_empty_deletes', - ], -) -@pytest.mark.asyncio -async def test_modify_streamlit_data_app_folder( - mocker, - mcp_context_client: Context, - workspace_manager, - configuration_id: str, - folder, - app_count: int, - app_folders: list[str], - expect_folder_metadata: bool, - expect_folder_delete: bool, - expect_hint: bool, -) -> None: - """Test folder metadata and change_summary hint for modify_streamlit_data_app (create and update paths).""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - workspace_manager.get_workspace_id = mocker.AsyncMock(return_value=1) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='snowflake') - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='default') - - keboola_client.storage_client.project_id = mocker.AsyncMock(return_value='proj-1') - - # Dummy encrypted config - encrypted_config = { - 'parameters': {'script': ['SELECT 1']}, - 'storage': {}, - 'authorization': {'app_proxy': {'auth_providers': [], 'auth_rules': []}}, - } - keboola_client.encryption_client = mocker.AsyncMock() - keboola_client.encryption_client.encrypt = mocker.AsyncMock(return_value=encrypted_config) - - data_app_response = _make_data_app_response(config_id=configuration_id or 'new-cfg-1') - - if configuration_id: - # Update path - existing_data_app = DataApp( - name='My App', - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - data_app_id='app-1', - project_id='proj-1', - branch_id='default', - config_version='2', - type='streamlit', - auto_suspend_after_seconds=900, - configuration=encrypted_config, - state='stopped', - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', return_value=existing_data_app) - mocker.patch( - 'keboola_mcp_server.tools.data_apps.modify_streamlit_data_app_internal', - mocker.AsyncMock(return_value=(existing_data_app, encrypted_config, None)), - ) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={}) - else: - # Create path - mocker.patch( - 'keboola_mcp_server.tools.data_apps.DataAppConfig.model_validate', - return_value=mocker.MagicMock(authorization={'app_proxy': {'auth_providers': [], 'auth_rules': []}}), - ) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.data_science_client.create_data_app = mocker.AsyncMock(return_value=data_app_response) - - mocker.patch( - 'keboola_mcp_server.tools.components.utils.get_config_folders', - mocker.AsyncMock(return_value=(app_count, app_folders, False)), - ) - keboola_client.storage_client.configuration_metadata_get = mocker.AsyncMock( - return_value=[{'id': 'meta-1', 'key': MetadataField.CONFIGURATION_FOLDER_NAME, 'value': 'OldFolder'}] - ) - keboola_client.storage_client.configuration_metadata_delete = mocker.AsyncMock() - - result = await modify_streamlit_data_app( - ctx=mcp_context_client, - name='My App', - description='desc', - source_code='import streamlit as st\n{QUERY_DATA_FUNCTION}\nst.write("hello")', - packages=[], - authentication_type='no-auth', - configuration_id=configuration_id, - change_description='test', - folder=folder, - ) - - assert isinstance(result, ModifiedDataAppOutput) - metadata_calls = [ - call - for call in keboola_client.storage_client.configuration_metadata_update.call_args_list - if call.kwargs.get('metadata', {}).get(MetadataField.CONFIGURATION_FOLDER_NAME) - ] - if expect_folder_metadata: - assert len(metadata_calls) == 1 - assert metadata_calls[0].kwargs['metadata'] == {MetadataField.CONFIGURATION_FOLDER_NAME: folder.strip()} - else: - assert len(metadata_calls) == 0 - if expect_folder_delete: - keboola_client.storage_client.configuration_metadata_delete.assert_called_once_with( - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - metadata_id='meta-1', - ) - else: - keboola_client.storage_client.configuration_metadata_delete.assert_not_called() - if expect_hint: - assert result.change_summary is not None - assert str(app_count) in result.change_summary - else: - assert result.change_summary is None - - -@pytest.mark.parametrize( - ('configuration_id', 'fail_at', 'state', 'expected_response'), - [ - # Update: failure at the re-fetch step, running app keeps the "redeploy required" hint. - ('cfg-1', '_fetch_data_app', 'running', 'updated (redeploy required to apply changes in the running app)'), - # Update: failure at the FIRST post-write step (metadata) on a stopped app -> still partial, no redeploy hint. - ('cfg-1', 'set_cfg_update_metadata', 'stopped', 'updated'), - # Create: failure at a post-write step. - ('', 'set_cfg_creation_metadata', None, 'created'), - ], - ids=['update_refetch_running', 'update_metadata_stopped', 'create_metadata'], -) -@pytest.mark.asyncio -async def test_modify_streamlit_data_app_partial_success_when_response_building_fails( - mocker, - mcp_context_client: Context, - workspace_manager, - configuration_id: str, - fail_at: str, - state: str | None, - expected_response: str, -) -> None: - """Regression for AJDA-2852: when the API write commits but a post-write response-building step raises, - the tool must return a truthful partial success (not a ToolError) so the agent does not retry and - double-apply the change. Parametrized over which post-write step fails to prove the partial path is reached - regardless of failure point, and that the response wording still reflects the app state.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - workspace_manager.get_workspace_id = mocker.AsyncMock(return_value=1) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='snowflake') - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='default') - keboola_client.storage_client.project_id = mocker.AsyncMock(return_value='proj-1') - - encrypted_config = { - 'parameters': {'script': ['SELECT 1']}, - 'storage': {}, - 'authorization': {'app_proxy': {'auth_providers': [], 'auth_rules': []}}, - } - keboola_client.encryption_client = mocker.AsyncMock() - keboola_client.encryption_client.encrypt = mocker.AsyncMock(return_value=encrypted_config) - - boom = RuntimeError('boom while building response') - - if configuration_id: - existing_data_app = DataApp( - name='My App', - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - data_app_id='app-1', - project_id='proj-1', - branch_id='default', - config_version='27', - type='streamlit', - auto_suspend_after_seconds=900, - configuration=encrypted_config, - state=state, - ) - mocker.patch( - 'keboola_mcp_server.tools.data_apps.modify_streamlit_data_app_internal', - mocker.AsyncMock(return_value=(existing_data_app, encrypted_config, None)), - ) - # The write commits and returns the new version... - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={'version': 28}) - # ...but a post-write enrichment step blows up. - mocker.patch(f'keboola_mcp_server.tools.data_apps.{fail_at}', side_effect=boom) - committing_mock = keboola_client.storage_client.configuration_update - else: - mocker.patch( - 'keboola_mcp_server.tools.data_apps.DataAppConfig.model_validate', - return_value=mocker.MagicMock(authorization={'app_proxy': {'auth_providers': [], 'auth_rules': []}}), - ) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.data_science_client.create_data_app = mocker.AsyncMock( - return_value=_make_data_app_response(config_id='new-cfg-1') - ) - # The app is created, but a post-write enrichment step blows up. - mocker.patch(f'keboola_mcp_server.tools.data_apps.{fail_at}', side_effect=boom) - committing_mock = keboola_client.data_science_client.create_data_app - - # Must NOT raise (no ToolError) even though a post-write step failed. - result = await modify_streamlit_data_app( - ctx=mcp_context_client, - name='My App', - description='desc', - source_code='import streamlit as st\n{QUERY_DATA_FUNCTION}\nst.write("hello")', - packages=['pandas'], - authentication_type='no-auth', - configuration_id=configuration_id, - change_description='test', - ) - - assert isinstance(result, ModifiedDataAppOutput) - # The committing write happened exactly once and is never retried within the call. - committing_mock.assert_awaited_once() - assert result.change_summary is not None - # The response truthfully reports the change landed and warns against retrying. - assert 'do not retry' in result.change_summary.lower() - # Response wording mirrors the success path (redeploy hint only for running/starting apps). - assert result.response == expected_response - if configuration_id: - assert 'WAS updated' in result.change_summary - # New version surfaced from the write response despite the failed re-fetch. - assert '28' in result.change_summary - assert result.data_app.config_version == '28' - else: - assert 'WAS created' in result.change_summary - - -@pytest.mark.asyncio -async def test_partial_output_helpers_never_raise_when_summary_construction_fails(mocker) -> None: - """The partial-output helpers promise 'MUST NOT raise' even if the primary DataAppSummary construction - fails -- they must fall back to a summary built from primitives instead of letting a committed write - surface as a ToolError (AJDA-2852).""" - from keboola_mcp_server.tools.data_apps import _partial_create_output, _partial_update_output - - links_manager = mocker.MagicMock() - links_manager.get_data_app_links.return_value = [] - - # --- update helper: force the primary model_validate to raise --- - data_app_pre = DataApp( - name='My App', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-1', - data_app_id='app-1', - project_id='proj-1', - branch_id='default', - config_version='27', - type='streamlit', - configuration={'authorization': {}}, - state='running', - ) - mocker.patch.object(DataAppSummary, 'model_validate', side_effect=RuntimeError('validate boom')) - update_out = _partial_update_output( - data_app_pre=data_app_pre, - links_manager=links_manager, - configuration_id='cfg-1', - name='My App', - new_version='28', - ) - assert isinstance(update_out, ModifiedDataAppOutput) - assert update_out.data_app.configuration_id == 'cfg-1' - assert update_out.data_app.data_app_id == 'app-1' - assert update_out.data_app.config_version == '28' # falls back to the new version from the write response - assert update_out.response == 'updated (redeploy required to apply changes in the running app)' - - # --- create helper: force the primary from_api_response to raise --- - data_app_resp = _make_data_app_response(config_id='new-cfg-1', data_app_id='app-2') - mocker.patch.object(DataAppSummary, 'from_api_response', side_effect=RuntimeError('from_api boom')) - create_out = _partial_create_output( - data_app_resp=data_app_resp, - links_manager=links_manager, - validated_config=mocker.MagicMock(authorization={}), - name='My App', - ) - assert isinstance(create_out, ModifiedDataAppOutput) - assert create_out.data_app.configuration_id == 'new-cfg-1' - assert create_out.data_app.data_app_id == 'app-2' - assert 'WAS created' in (create_out.change_summary or '') - - -@pytest.mark.asyncio -async def test_modify_streamlit_data_app_update_skips_metadata_when_version_missing( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """When the update response carries no numeric version, set_cfg_update_metadata must be SKIPPED rather than - stamped with the (now-stale) pre-update version, which would record a misleading UPDATED_BY_MCP version - (review hardening on AJDA-2852). The tool still returns a normal success.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - workspace_manager.get_workspace_id = mocker.AsyncMock(return_value=1) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value='snowflake') - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='default') - keboola_client.storage_client.project_id = mocker.AsyncMock(return_value='proj-1') - - encrypted_config = { - 'parameters': {'script': ['SELECT 1']}, - 'storage': {}, - 'authorization': {'app_proxy': {'auth_providers': [], 'auth_rules': []}}, - } - keboola_client.encryption_client = mocker.AsyncMock() - keboola_client.encryption_client.encrypt = mocker.AsyncMock(return_value=encrypted_config) - - existing_data_app = DataApp( - name='My App', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-1', - data_app_id='app-1', - project_id='proj-1', - branch_id='default', - config_version='27', - type='streamlit', - auto_suspend_after_seconds=900, - configuration=encrypted_config, - state='stopped', - ) - mocker.patch( - 'keboola_mcp_server.tools.data_apps.modify_streamlit_data_app_internal', - mocker.AsyncMock(return_value=(existing_data_app, encrypted_config, None)), - ) - # Committing write succeeds but the response carries no version. - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={}) - # Let the rest of the (best-effort) response building succeed. - set_meta = mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_update_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=existing_data_app)) - - result = await modify_streamlit_data_app( - ctx=mcp_context_client, - name='My App', - description='desc', - source_code='import streamlit as st\n{QUERY_DATA_FUNCTION}\nst.write("hello")', - packages=['pandas'], - authentication_type='no-auth', - configuration_id='cfg-1', - change_description='test', - ) - - assert isinstance(result, ModifiedDataAppOutput) - assert result.response == 'updated' - # The misleading pre-update-version stamp must NOT happen when the new version is unknown. - set_meta.assert_not_awaited() - - -# ===== Tests for modify_python_js_data_app ===== - - -from keboola_mcp_server.clients.data_science import ( # noqa: E402 - AppGitRepoResponse, - CreatedGitCredentialResponse, -) -from keboola_mcp_server.tools.data_apps import ( # noqa: E402 - CreatedGitCredentialOutput, - ModifiedPythonJsDataAppOutput, - _is_draft_config, - _update_existing_code_data_app_config, - create_python_js_data_app_git_credential, - modify_python_js_data_app, -) - - -def _make_python_js_data_app_response( - data_app_id: str = 'app-pyjs-1', - config_id: str = 'cfg-pyjs-1', -) -> DataAppResponse: - return DataAppResponse( - id=data_app_id, - project_id='proj-1', - component_id=DATA_APP_COMPONENT_ID, - branch_id='branch-1', - config_id=config_id, - config_version='1', - type='python-js', - state='created', - desired_state='created', - url='https://demo.canary-orion.keboola.dev', - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('missing_arg', 'kwargs', 'error_match'), - [ - ( - 'slug', - {'name': 'A', 'description': ''}, - 'slug is required', - ), - ], -) -async def test_modify_python_js_data_app_create_validates_required_args( - mcp_context_client: Context, - missing_arg: str, - kwargs: dict, - error_match: str, -) -> None: - """Create path raises clear ValueError when slug is missing.""" - with pytest.raises(ValueError, match=error_match): - await modify_python_js_data_app(ctx=mcp_context_client, **kwargs) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('disallowed_arg', 'kwargs', 'error_match'), - [ - ( - 'slug', - {'name': 'A', 'description': '', 'configuration_id': 'cfg-1', 'slug': 'new'}, - 'slug cannot be changed', - ), - ], -) -async def test_modify_python_js_data_app_update_rejects_create_only_args( - mcp_context_client: Context, - disallowed_arg: str, - kwargs: dict, - error_match: str, -) -> None: - """Update path rejects slug (immutable subdomain).""" - with pytest.raises(ValueError, match=error_match): - await modify_python_js_data_app(ctx=mcp_context_client, **kwargs) - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_calls_full_provisioning_chain( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """Create path: POST /apps with type=python-js + useManagedGitRepo, fetch repo URL. Git - credential creation is now a separate tool — not exercised here.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - app_response = _make_python_js_data_app_response() - keboola_client.data_science_client.create_data_app = mocker.AsyncMock(return_value=app_response) - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url='git@managed.repo:org/app.git', - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - - # avoid hitting Storage API for metadata helpers - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_creation_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - result = await modify_python_js_data_app( - ctx=mcp_context_client, - name='My App', - description='desc', - slug='my-app', - auto_suspend_after_seconds=300, - ) - - assert isinstance(result, ModifiedPythonJsDataAppOutput) - assert result.response == 'created' - assert result.repo_url == 'https://managed.repo/org/app.git' - assert result.data_app.repo_url == 'https://managed.repo/org/app.git' - assert result.data_app.type == 'python-js' - - # Verify the create payload was python-js + managed repo - create_kwargs = keboola_client.data_science_client.create_data_app.await_args.kwargs - assert create_kwargs['app_type'] == 'python-js' - assert create_kwargs['use_managed_git_repo'] is True - # Verify auto_suspend_after_seconds flows through and we don't pin runtime.image (the - # platform now picks a default for python-js apps). - serialized = create_kwargs['configuration'].model_dump(by_alias=True, exclude_none=True) - assert serialized['parameters']['autoSuspendAfterSeconds'] == 300 - assert serialized['parameters']['dataApp']['slug'] == 'my-app' - assert 'image' not in serialized.get('runtime', {}) - # Created with the auto-workspace flag so the platform provisions a per-app workspace - # and sets WORKSPACE_ID itself. - assert serialized['runtime']['workspace'] == {'enabled': True} - # KBC_TOKEN / KBC_URL / BRANCH_ID are injected by the platform at runtime — the MCP must - # not bake them into the stored config. - assert 'secrets' not in serialized['parameters']['dataApp'] - # Prod apps must NOT be marked as draft (UT-4000). - assert 'isDraft' not in serialized['parameters']['dataApp'] - # Default `authentication_type='default'` produces basic-auth on create (safe-by-default). - assert serialized['authorization']['app_proxy']['auth_providers'] == [{'id': 'simpleAuth', 'type': 'password'}] - assert serialized['authorization']['app_proxy']['auth_rules'] == [ - {'type': 'pathPrefix', 'value': '/', 'auth_required': True, 'auth': ['simpleAuth']} - ] - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('authentication_type', 'expect_basic_auth'), - [ - ('default', True), - ('basic-auth', True), - ('no-auth', False), - ], -) -async def test_modify_python_js_data_app_create_authentication_type( - mocker, - mcp_context_client: Context, - workspace_manager, - authentication_type: str, - expect_basic_auth: bool, -) -> None: - """Create path translates authentication_type to the right authorization block: - 'default' and 'basic-auth' → password-protected; 'no-auth' → public.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - app_response = _make_python_js_data_app_response() - keboola_client.data_science_client.create_data_app = mocker.AsyncMock(return_value=app_response) - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url='git@managed.repo:org/app.git', - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_creation_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - _ = await modify_python_js_data_app( - ctx=mcp_context_client, - name='My App', - description='desc', - slug='my-app', - authentication_type=cast(Literal['no-auth', 'basic-auth', 'default'], authentication_type), - ) - - serialized = keboola_client.data_science_client.create_data_app.await_args.kwargs['configuration'].model_dump( - by_alias=True, exclude_none=True - ) - auth_rule = serialized['authorization']['app_proxy']['auth_rules'][0] - if expect_basic_auth: - assert auth_rule['auth_required'] is True - assert auth_rule['auth'] == ['simpleAuth'] - else: - assert auth_rule['auth_required'] is False - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_update_patches_storage_config( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """Update path: fetch storage config → merge updates → PATCH via configuration_update.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - existing_data_app = DataApp( - name='Old', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-1', - data_app_id='app-1', - project_id='proj-1', - branch_id='branch-1', - config_version='2', - type='python-js', - configuration={ - 'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'old-slug'}}, - 'runtime': {'image': {'version': 'old-version'}}, - }, - state='stopped', - ) - updated_data_app = existing_data_app.model_copy(update={'config_version': '3', 'name': 'New'}) - - mocker.patch( - 'keboola_mcp_server.tools.data_apps._fetch_data_app', - mocker.AsyncMock(side_effect=[existing_data_app, updated_data_app]), - ) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={}) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url='git@managed.repo:org/app.git', - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_update_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - result = await modify_python_js_data_app( - ctx=mcp_context_client, - name='New', - description='new desc', - configuration_id='cfg-1', - auto_suspend_after_seconds=600, - ) - - assert isinstance(result, ModifiedPythonJsDataAppOutput) - assert result.response == 'updated' - # The PATCH should carry merged config - patch_kwargs = keboola_client.storage_client.configuration_update.await_args.kwargs - new_cfg = patch_kwargs['configuration'] - assert new_cfg['parameters']['autoSuspendAfterSeconds'] == 600 - # The platform now picks a default image for python-js apps; the MCP must NOT overwrite a - # legacy image pin already in the config, but must NOT force it to any new value either. - assert new_cfg['runtime']['image']['version'] == 'old-version' - # slug must remain untouched (immutable) - assert new_cfg['parameters']['dataApp']['slug'] == 'old-slug' - # Update does NOT backfill `runtime.workspace` — only the create path sets it. - assert 'workspace' not in new_cfg['runtime'] - # KBC_TOKEN / KBC_URL / BRANCH_ID are injected by the platform at runtime — the MCP must - # not write them back into the stored config on update either. - assert 'secrets' not in new_cfg['parameters']['dataApp'] - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_without_workspace_feature( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """When the project lacks `data-apps-storage-workspace`, the create path omits - `runtime.workspace` and falls back to injecting WORKSPACE_ID via `parameters.dataApp.secrets`.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.has_feature = mocker.AsyncMock(return_value=False) - - workspace_manager.get_workspace_id = mocker.AsyncMock(return_value='wid-legacy') - - app_response = _make_python_js_data_app_response() - keboola_client.data_science_client.create_data_app = mocker.AsyncMock(return_value=app_response) - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url='git@managed.repo:org/app.git', - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_creation_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - _ = await modify_python_js_data_app( - ctx=mcp_context_client, - name='My App', - description='desc', - slug='my-app', - ) - - serialized = keboola_client.data_science_client.create_data_app.await_args.kwargs['configuration'].model_dump( - by_alias=True, exclude_none=True - ) - # Without the workspace feature, the `runtime` block carries no fields the MCP wants to - # set (image is platform-default, workspace is off), so it's omitted entirely. - assert 'runtime' not in serialized - assert serialized['parameters']['dataApp']['secrets'] == {'WORKSPACE_ID': 'wid-legacy'} - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_update_injects_workspace_id_without_feature( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """When the project lacks `data-apps-storage-workspace`, the update path merges WORKSPACE_ID - into the existing secrets map without overwriting any keys already present.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.has_feature = mocker.AsyncMock(return_value=False) - - workspace_manager.get_workspace_id = mocker.AsyncMock(return_value='wid-legacy') - - existing_data_app = DataApp( - name='Old', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-1', - data_app_id='app-1', - project_id='proj-1', - branch_id='branch-1', - config_version='2', - type='python-js', - configuration={ - 'parameters': { - 'autoSuspendAfterSeconds': 900, - 'dataApp': {'slug': 'old-slug', 'secrets': {'KEEP': 'x'}}, - }, - 'runtime': {'image': {'version': 'old-version'}}, - }, - state='stopped', - ) - updated_data_app = existing_data_app.model_copy(update={'config_version': '3'}) - - mocker.patch( - 'keboola_mcp_server.tools.data_apps._fetch_data_app', - mocker.AsyncMock(side_effect=[existing_data_app, updated_data_app]), - ) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={}) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url='git@managed.repo:org/app.git', - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_update_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - await modify_python_js_data_app( - ctx=mcp_context_client, - name='Old', - description='desc', - configuration_id='cfg-1', - auto_suspend_after_seconds=600, - ) - - patch_kwargs = keboola_client.storage_client.configuration_update.await_args.kwargs - new_cfg = patch_kwargs['configuration'] - assert new_cfg['parameters']['dataApp']['secrets'] == {'KEEP': 'x', 'WORKSPACE_ID': 'wid-legacy'} - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_passes_storage_through( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """Create path forwards a caller-supplied `storage` block (with direct-grant) into the DSAPI payload.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - app_response = _make_python_js_data_app_response() - keboola_client.data_science_client.create_data_app = mocker.AsyncMock(return_value=app_response) - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url='git@managed.repo:org/app.git', - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_creation_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - storage = { - 'output': { - 'tables': [ - {'destination': 'in.c-ex-generic-v2.earthquake_events', 'unload_strategy': 'direct-grant'}, - ], - }, - } - - _ = await modify_python_js_data_app( - ctx=mcp_context_client, - name='My App', - description='desc', - slug='my-app', - storage=storage, - ) - - serialized = keboola_client.data_science_client.create_data_app.await_args.kwargs['configuration'].model_dump( - by_alias=True, exclude_none=True - ) - assert serialized['storage'] == storage - - -@pytest.mark.parametrize('passed_storage', [{}, {'input': {}, 'output': {}}], ids=['empty', 'all_empty_objects']) -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_omits_empty_storage( - mocker, - mcp_context_client: Context, - workspace_manager, - passed_storage, -) -> None: - """Create path never persists an empty `storage` block (it collapses to `[]` server-side, AI-3135).""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - app_response = _make_python_js_data_app_response() - keboola_client.data_science_client.create_data_app = mocker.AsyncMock(return_value=app_response) - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url='git@managed.repo:org/app.git', - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_creation_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - _ = await modify_python_js_data_app( - ctx=mcp_context_client, - name='My App', - description='desc', - slug='my-app', - storage=passed_storage, - ) - - serialized = keboola_client.data_science_client.create_data_app.await_args.kwargs['configuration'].model_dump( - by_alias=True, exclude_none=True - ) - assert 'storage' not in serialized - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_update_replaces_storage( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """Update path: a non-empty `storage` argument replaces the entire stored storage block.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - existing_data_app = DataApp( - name='Old', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-1', - data_app_id='app-1', - project_id='proj-1', - branch_id='branch-1', - config_version='2', - type='python-js', - configuration={ - 'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'old-slug'}}, - 'runtime': {'image': {'version': 'old-version'}}, - 'storage': {'input': {'tables': [{'source': 'in.c-main.stale', 'destination': 'stale.csv'}]}}, - }, - state='stopped', - ) - updated_data_app = existing_data_app.model_copy(update={'config_version': '3', 'name': 'New'}) - - mocker.patch( - 'keboola_mcp_server.tools.data_apps._fetch_data_app', - mocker.AsyncMock(side_effect=[existing_data_app, updated_data_app]), - ) - keboola_client.storage_client.configuration_update = mocker.AsyncMock(return_value={}) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url='git@managed.repo:org/app.git', - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_update_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - new_storage = { - 'output': { - 'tables': [ - {'destination': 'in.c-ex-generic-v2.earthquake_events', 'unload_strategy': 'direct-grant'}, - ], - }, - } - - await modify_python_js_data_app( - ctx=mcp_context_client, - name='New', - description='new desc', - configuration_id='cfg-1', - auto_suspend_after_seconds=600, - storage=new_storage, - ) - - patch_kwargs = keboola_client.storage_client.configuration_update.await_args.kwargs - assert patch_kwargs['configuration']['storage'] == new_storage - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_storage_validation_rejects_missing_source( - mcp_context_client: Context, -) -> None: - """An output table with neither `source` nor `unload_strategy='direct-grant'` must be rejected. - - The `@tool_errors()` decorator wraps the underlying RecoverableValidationError into a - fastmcp ToolError before it surfaces to the caller. - """ - from fastmcp.exceptions import ToolError - - with pytest.raises(ToolError, match="'source' is a required property"): - await modify_python_js_data_app( - ctx=mcp_context_client, - name='My App', - description='desc', - slug='my-app', - storage={'output': {'tables': [{'destination': 'in.c-ex.foo'}]}}, - ) - - -def test_update_existing_code_data_app_config_leaves_legacy_image_pin_alone() -> None: - """The MCP no longer manages `runtime.image.version` — the platform picks a default for - python-js apps. A legacy pin already in the stored config must survive the deepcopy - verbatim (we don't overwrite it, even though we also don't set it ourselves).""" - existing = { - 'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'x'}}, - 'runtime': {'image': {'version': 'old'}}, - } - new = _update_existing_code_data_app_config(existing, auto_suspend_after_seconds=600) - assert new['runtime']['image']['version'] == 'old' - assert new['parameters']['autoSuspendAfterSeconds'] == 600 - # original must not be mutated - assert existing['parameters']['autoSuspendAfterSeconds'] == 900 - - -def test_update_existing_code_data_app_config_default_auth_preserves_existing() -> None: - """`authentication_type='default'` must not touch an existing authorization block (e.g. OIDC).""" - existing_authorization = { - 'app_proxy': { - 'auth_providers': [{'id': 'oidc', 'type': 'oidc', 'issuer_url': 'https://issuer'}], - 'auth_rules': [{'type': 'pathPrefix', 'value': '/', 'auth_required': True, 'auth': ['oidc']}], - } - } - existing = { - 'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'x'}}, - 'authorization': existing_authorization, - } - new = _update_existing_code_data_app_config(existing, auto_suspend_after_seconds=900, authentication_type='default') - # Deepcopy makes it equal-but-not-identical. - assert new['authorization'] == existing_authorization - - -def test_update_existing_code_data_app_config_basic_auth_overwrites() -> None: - existing = { - 'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'x'}}, - 'authorization': {'app_proxy': {'auth_providers': [], 'auth_rules': []}}, - } - new = _update_existing_code_data_app_config( - existing, auto_suspend_after_seconds=900, authentication_type='basic-auth' - ) - assert new['authorization']['app_proxy']['auth_rules'] == [ - {'type': 'pathPrefix', 'value': '/', 'auth_required': True, 'auth': ['simpleAuth']} - ] - - -def test_update_existing_code_data_app_config_preserves_legacy_secrets() -> None: - """Legacy configs written by older MCP versions may carry a `secrets` block. We no longer - write secrets (platform injects KBC_TOKEN/KBC_URL/BRANCH_ID at runtime), but the deepcopy - of the existing config must leave any pre-existing keys untouched.""" - existing = { - 'parameters': { - 'autoSuspendAfterSeconds': 900, - 'dataApp': { - 'slug': 'x', - 'secrets': {'WORKSPACE_ID': 'wid-legacy', 'KEEP': 'x'}, - }, - }, - } - new = _update_existing_code_data_app_config( - existing, - auto_suspend_after_seconds=900, - ) - assert new['parameters']['dataApp']['secrets'] == {'WORKSPACE_ID': 'wid-legacy', 'KEEP': 'x'} - - -def test_update_existing_code_data_app_config_no_auth_overwrites() -> None: - existing = { - 'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'x'}}, - 'authorization': { - 'app_proxy': { - 'auth_providers': [{'id': 'simpleAuth', 'type': 'password'}], - 'auth_rules': [{'type': 'pathPrefix', 'value': '/', 'auth_required': True, 'auth': ['simpleAuth']}], - } - }, - } - new = _update_existing_code_data_app_config(existing, auto_suspend_after_seconds=900, authentication_type='no-auth') - assert new['authorization']['app_proxy']['auth_rules'] == [ - {'type': 'pathPrefix', 'value': '/', 'auth_required': False} - ] - - -@pytest.mark.parametrize( - ('passed_storage', 'expected_storage_key_present', 'expected_storage'), - [ - # None preserves the existing storage block untouched - (None, True, {'input': {'tables': [{'source': 'in.c-main.kept', 'destination': 'kept.csv'}]}}), - # Empty dict is an explicit wipe — the `storage` key is removed entirely (never persisted as - # `{}`, which the backend collapses to `[]` and breaks the mapping editor, AI-3135). - ({}, False, None), - # A block that prunes down to nothing (only empty objects) is treated like a wipe. - ({'input': {}, 'output': {}}, False, None), - # Non-empty dict replaces the existing block wholesale. - ( - {'output': {'tables': [{'destination': 'in.c-main.new', 'unload_strategy': 'direct-grant'}]}}, - True, - {'output': {'tables': [{'destination': 'in.c-main.new', 'unload_strategy': 'direct-grant'}]}}, - ), - # Empty mapping containers are pruned from an otherwise-populated block. - ( - { - 'input': {}, - 'output': {'tables': [{'destination': 'in.c-main.new', 'unload_strategy': 'direct-grant'}]}, - }, - True, - {'output': {'tables': [{'destination': 'in.c-main.new', 'unload_strategy': 'direct-grant'}]}}, - ), - ], -) -def test_update_existing_code_data_app_config_storage_semantics( - passed_storage, expected_storage_key_present, expected_storage -) -> None: - """`storage=None` preserves; an empty/all-empty block wipes (removes the key); a non-empty dict - replaces wholesale (with empty mapping containers pruned).""" - existing = { - 'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'x'}}, - 'storage': {'input': {'tables': [{'source': 'in.c-main.kept', 'destination': 'kept.csv'}]}}, - } - new = _update_existing_code_data_app_config( - existing, - auto_suspend_after_seconds=900, - storage=passed_storage, - ) - assert ('storage' in new) is expected_storage_key_present - if expected_storage_key_present: - assert new['storage'] == expected_storage - - -# ===== Tests for deploy_data_app with mode and python-js ===== - - -@pytest.mark.asyncio -async def test_deploy_data_app_python_js_skips_storage_config_version_and_passes_mode( - mocker, - mcp_context_client: Context, -) -> None: - """python-js deploy: no configVersion fetch from Storage, mode forwarded to DSAPI.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - - pyjs_app = DataApp( - name='py-app', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-1', - data_app_id='app-1', - project_id='proj-1', - branch_id='branch-1', - config_version='1', - type='python-js', - configuration={'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'x'}}}, - state='stopped', - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=pyjs_app)) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_logs', mocker.AsyncMock(return_value=[])) - - # If the code accidentally calls storage_client.configuration_version_latest, this AsyncMock raises. - keboola_client.storage_client.configuration_version_latest = mocker.AsyncMock( - side_effect=AssertionError('Should not call configuration_version_latest for python-js apps') - ) - - _ = await deploy_data_app( - ctx=mcp_context_client, - action='deploy', - configuration_id='cfg-1', - mode='dev', - ) - - keboola_client.data_science_client.deploy_data_app.assert_awaited_once_with('app-1', None, mode='dev') - keboola_client.storage_client.configuration_version_latest.assert_not_called() - - -@pytest.mark.asyncio -async def test_deploy_data_app_streamlit_still_passes_config_version( - mocker, - mcp_context_client: Context, - data_app: DataApp, # streamlit fixture -) -> None: - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - data_app.state = 'stopped' - data_app.type = 'streamlit' - data_app.configuration = {'authorization': {'app_proxy': {'auth_providers': [], 'auth_rules': []}}} - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=data_app)) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_logs', mocker.AsyncMock(return_value=[])) - keboola_client.storage_client.configuration_version_latest = mocker.AsyncMock(return_value=7) - - _ = await deploy_data_app(ctx=mcp_context_client, action='deploy', configuration_id='cfg-streamlit') - - keboola_client.data_science_client.deploy_data_app.assert_awaited_once_with(data_app.data_app_id, '7', mode=None) - - -# ===== Tests for modify_python_js_data_app draft create path ===== - - -def _make_python_js_parent_data_app( - *, - data_app_id: str = 'app-prod-1', - configuration_id: str = 'cfg-prod-1', - repo_url: str | None = 'https://managed.repo/org/prod.git', - type: str = 'python-js', - is_draft: bool = False, -) -> DataApp: - """Build a DataApp the way `_fetch_data_app` would when looking up the parent. - - `is_draft=True` models a caller mistakenly passing a draft as the parent — drafts cannot - parent another draft. - """ - data_app_block: dict = {'slug': 'demo'} - if is_draft: - data_app_block['isDraft'] = True - data_app_block['parentConfigurationId'] = 'cfg-grandparent' - return DataApp( - name='Prod App', - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - data_app_id=data_app_id, - project_id='proj-1', - branch_id='branch-1', - config_version='1', - type=type, - configuration={'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': data_app_block}}, - state='running', - repo_url=repo_url, - ) - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_draft_uses_external_git( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """When `parent_configuration_id` is set, the new app is a draft: no managed repo of its own, - `parameters.dataApp.git` populated with the parent's repo URL + a freshly minted prod-app token, - and the config is encrypted before being sent to data-science.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - parent_repo = 'https://managed.repo/org/prod.git' - parent_data_app_id = 'app-prod-1' - parent = _make_python_js_parent_data_app( - data_app_id=parent_data_app_id, configuration_id='cfg-prod-1', repo_url=parent_repo - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=parent)) - - keboola_client.data_science_client.create_app_git_credential = mocker.AsyncMock( - return_value=CreatedGitCredentialResponse( - id='cred-1', type='http_token', permissions='readWrite', secret='token-xyz' - ) - ) - twin_response = _make_python_js_data_app_response(data_app_id='app-dev-1', config_id='cfg-dev-1') - keboola_client.data_science_client.create_data_app = mocker.AsyncMock(return_value=twin_response) - # The draft has no managed repo, so get_app_git_repo must NOT be called for it. - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - side_effect=AssertionError('Should not fetch a git repo URL for a draft') - ) - - keboola_client.storage_client.project_id = mocker.AsyncMock(return_value='proj-1') - - # Mock encryption: walk the dict and prefix `KBC::cipher::` onto any value whose key starts with '#'. - async def fake_encrypt(value, *, project_id=None, component_id=None, config_id=None): - def walk(node): - if isinstance(node, dict): - return { - k: (f'KBC::cipher::{v}' if k.startswith('#') and isinstance(v, str) else walk(v)) - for k, v in node.items() - } - return node - - return walk(value) - - keboola_client.encryption_client = mocker.AsyncMock() - keboola_client.encryption_client.encrypt = mocker.AsyncMock(side_effect=fake_encrypt) - - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_creation_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - result = await modify_python_js_data_app( - ctx=mcp_context_client, - name='Dev Twin', - description='dev iteration twin', - slug='demo-dev-abc123', - parent_configuration_id='cfg-prod-1', - branch='iter-feat', - ) - - assert isinstance(result, ModifiedPythonJsDataAppOutput) - assert result.response == 'created' - assert result.repo_url == parent_repo - assert result.branch == 'iter-feat' - assert result.git_clone_url is not None - assert result.git_clone_url.startswith('https://kai:token-xyz@managed.repo/') - - # Credential was minted on the parent, not the new dev twin. - keboola_client.data_science_client.create_app_git_credential.assert_awaited_once_with(parent_data_app_id) - - # create_data_app received use_managed_git_repo=False and the external-git block. - create_kwargs = keboola_client.data_science_client.create_data_app.await_args.kwargs - assert create_kwargs['app_type'] == 'python-js' - assert create_kwargs['use_managed_git_repo'] is False - serialized = create_kwargs['configuration'].model_dump(by_alias=True, exclude_none=True) - git_block = serialized['parameters']['dataApp']['git'] - assert git_block == { - 'repository': parent_repo, - 'username': 'kai', - '#password': 'KBC::cipher::token-xyz', - 'branch': 'iter-feat', - } - - # Dev twin is marked as draft so the UI hides it from the main data-apps list (UT-4000). - assert serialized['parameters']['dataApp']['isDraft'] is True - - # Encryption was actually called (so `#password` is ciphertext on the wire). - keboola_client.encryption_client.encrypt.assert_awaited_once() - encrypt_kwargs = keboola_client.encryption_client.encrypt.await_args.kwargs - assert encrypt_kwargs['component_id'] == DATA_APP_COMPONENT_ID - assert encrypt_kwargs['project_id'] == 'proj-1' - - # No managed-repo lookup happened on the dev twin. - keboola_client.data_science_client.get_app_git_repo.assert_not_called() - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_draft_defaults_branch_to_init( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """Omitting `branch` pins the draft to the literal `init` branch (sensible default for the very - first draft of a brand-new prod app — descriptive branches are agent-supplied on edits).""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - parent = _make_python_js_parent_data_app() - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=parent)) - keboola_client.data_science_client.create_app_git_credential = mocker.AsyncMock( - return_value=CreatedGitCredentialResponse( - id='cred-1', type='http_token', permissions='readWrite', secret='token-xyz' - ) - ) - keboola_client.data_science_client.create_data_app = mocker.AsyncMock( - return_value=_make_python_js_data_app_response() - ) - keboola_client.storage_client.project_id = mocker.AsyncMock(return_value='proj-1') - keboola_client.encryption_client = mocker.AsyncMock() - keboola_client.encryption_client.encrypt = mocker.AsyncMock(side_effect=lambda v, **_: v) - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_creation_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - result = await modify_python_js_data_app( - ctx=mcp_context_client, - name='Draft', - description='draft iteration', - slug='demo-draft', - parent_configuration_id='cfg-prod-1', - ) - - assert result.branch == 'init' - # And the stored config carries the same branch as the pin and the parent linkage. - create_kwargs = keboola_client.data_science_client.create_data_app.await_args.kwargs - serialized = create_kwargs['configuration'].model_dump(by_alias=True, exclude_none=True) - assert serialized['parameters']['dataApp']['git']['branch'] == 'init' - assert serialized['parameters']['dataApp']['parentConfigurationId'] == 'cfg-prod-1' - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('kwargs', 'error_match'), - [ - ( - { - 'name': 'A', - 'description': '', - 'parent_configuration_id': 'cfg-prod-1', - }, - 'slug is required', - ), - ], -) -async def test_modify_python_js_data_app_create_draft_still_requires_slug( - mcp_context_client: Context, - kwargs: dict, - error_match: str, -) -> None: - """`parent_configuration_id` does not waive any required create-time argument.""" - with pytest.raises(ValueError, match=error_match): - await modify_python_js_data_app(ctx=mcp_context_client, **kwargs) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('disallowed_arg', 'kwargs', 'error_match'), - [ - ( - 'parent_configuration_id', - { - 'name': 'A', - 'description': '', - 'configuration_id': 'cfg-1', - 'parent_configuration_id': 'cfg-prod-1', - }, - 'parent_configuration_id is only valid when creating a draft', - ), - ( - 'branch', - { - 'name': 'A', - 'description': '', - 'configuration_id': 'cfg-1', - 'branch': 'add-revenue-filter', - }, - 'branch is only valid when creating a draft', - ), - ], -) -async def test_modify_python_js_data_app_update_rejects_draft_args( - mcp_context_client: Context, - disallowed_arg: str, - kwargs: dict, - error_match: str, -) -> None: - """The update path rejects draft-only args (`parent_configuration_id`, `branch`).""" - with pytest.raises(ValueError, match=error_match): - await modify_python_js_data_app(ctx=mcp_context_client, **kwargs) - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_draft_rejects_when_parent_is_streamlit( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """A Streamlit `parent_configuration_id` is rejected — only python-js prods can parent a draft.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - streamlit_parent = _make_python_js_parent_data_app(type='streamlit', repo_url=None) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=streamlit_parent)) - - with pytest.raises(ValueError, match='only python-js prod apps can parent a draft'): - await modify_python_js_data_app( - ctx=mcp_context_client, - name='Draft', - description='', - slug='demo-draft', - parent_configuration_id='cfg-prod-1', - ) - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_draft_rejects_when_parent_is_draft( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """A python-js *draft* parent is rejected with a clear message (not the misleading 'no repo URL' - error): drafts can't parent another draft, so no credential is minted.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - # A draft has no repo_url of its own; the guard must fire before the repo_url check below. - draft_parent = _make_python_js_parent_data_app(is_draft=True, repo_url=None) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=draft_parent)) - - with pytest.raises(ValueError, match=r'is itself a python-js \*\*draft\*\*'): - await modify_python_js_data_app( - ctx=mcp_context_client, - name='Draft', - description='', - slug='demo-draft', - parent_configuration_id='cfg-prod-1', - ) - - keboola_client.data_science_client.create_app_git_credential.assert_not_called() - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_draft_rejects_when_parent_missing_repo_url( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """Defensive: parent's repo lookup returned no URL — surface a clear error.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - parent = _make_python_js_parent_data_app(repo_url=None) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=parent)) - - with pytest.raises(ValueError, match='has no managed git repo URL'): - await modify_python_js_data_app( - ctx=mcp_context_client, - name='Dev Twin', - description='', - slug='demo-dev', - parent_configuration_id='cfg-prod-1', - ) - - -@pytest.mark.asyncio -async def test_modify_python_js_data_app_create_prod_calls_get_app_git_repo_for_url( - mocker, - mcp_context_client: Context, - workspace_manager, -) -> None: - """Prod creates always go through get_app_git_repo (no short-circuit branch).""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - keboola_client.has_feature = mocker.AsyncMock(return_value=True) - workspace_manager.get_branch_id = mocker.AsyncMock(return_value='branch-1') - - keboola_client.data_science_client.create_data_app = mocker.AsyncMock( - return_value=_make_python_js_data_app_response() - ) - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url=None, - https_url='https://managed.repo/org/prod.git', - is_managed_git_repo=True, - ) - ) - mocker.patch('keboola_mcp_server.tools.data_apps.set_cfg_creation_metadata', mocker.AsyncMock()) - mocker.patch('keboola_mcp_server.tools.data_apps.apply_folder_metadata', mocker.AsyncMock(return_value=None)) - - result = await modify_python_js_data_app(ctx=mcp_context_client, name='Prod', description='', slug='demo') - - assert result.repo_url == 'https://managed.repo/org/prod.git' - keboola_client.data_science_client.get_app_git_repo.assert_awaited_once() - create_kwargs = keboola_client.data_science_client.create_data_app.await_args.kwargs - assert create_kwargs['use_managed_git_repo'] is True - # No git block on prod create. - serialized = create_kwargs['configuration'].model_dump(by_alias=True, exclude_none=True) - assert 'git' not in serialized['parameters']['dataApp'] - - -# ===== Tests for create_python_js_data_app_git_credential ===== - - -@pytest.mark.asyncio -async def test_create_python_js_data_app_git_credential_happy_path( - mocker, - mcp_context_client: Context, -) -> None: - """Resolves configuration_id → data_app_id, mints an http_token credential, and embeds the - one-time secret into a ready-to-use HTTPS clone URL.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - - pyjs_app = DataApp( - name='my-app', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-pyjs-1', - data_app_id='app-pyjs-1', - project_id='proj-1', - branch_id='branch-1', - config_version='1', - type='python-js', - configuration={'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'my-app'}}}, - state='running', - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=pyjs_app)) - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url='git@managed.repo:org/app.git', - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - keboola_client.data_science_client.create_app_git_credential = mocker.AsyncMock( - return_value=CreatedGitCredentialResponse( - id='cred-99', - type='http_token', - name='', - permissions='readWrite', - secret='token-xyz', - ) - ) - - result = await create_python_js_data_app_git_credential( - ctx=mcp_context_client, - configuration_id='cfg-pyjs-1', - ) - - assert isinstance(result, CreatedGitCredentialOutput) - assert result.response == 'created' - assert result.configuration_id == 'cfg-pyjs-1' - assert result.data_app_id == 'app-pyjs-1' - assert result.credential_id == 'cred-99' - assert result.secret == 'token-xyz' - assert result.git_clone_url == 'https://kai:token-xyz@managed.repo/org/app.git' - assert result.permissions == 'readWrite' - - keboola_client.data_science_client.create_app_git_credential.assert_awaited_once_with( - data_app_id='app-pyjs-1', - ) - - -@pytest.mark.asyncio -async def test_create_python_js_data_app_git_credential_url_encodes_secret( - mocker, - mcp_context_client: Context, -) -> None: - """Secrets containing URL-reserved characters must be percent-encoded in `git_clone_url`.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - - pyjs_app = DataApp( - name='my-app', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-pyjs-1', - data_app_id='app-pyjs-1', - project_id='proj-1', - branch_id='branch-1', - config_version='1', - type='python-js', - configuration={'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'my-app'}}}, - state='running', - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=pyjs_app)) - keboola_client.data_science_client.get_app_git_repo = mocker.AsyncMock( - return_value=AppGitRepoResponse( - ssh_url=None, - https_url='https://managed.repo/org/app.git', - is_managed_git_repo=True, - ) - ) - keboola_client.data_science_client.create_app_git_credential = mocker.AsyncMock( - return_value=CreatedGitCredentialResponse( - id='cred-99', - type='http_token', - name='', - permissions='readWrite', - secret='ab/cd:ef@gh', - ) - ) - - result = await create_python_js_data_app_git_credential( - ctx=mcp_context_client, - configuration_id='cfg-pyjs-1', - ) - - # Reserved characters in the secret (/, :, @) must be percent-encoded so the URL parses - # back to the original token when git authenticates. - assert result.git_clone_url == 'https://kai:ab%2Fcd%3Aef%40gh@managed.repo/org/app.git' - - -@pytest.mark.asyncio -async def test_create_python_js_data_app_git_credential_rejects_streamlit_app( - mocker, - mcp_context_client: Context, -) -> None: - """Streamlit apps have no managed git repo — must raise a clear ValueError.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - - streamlit_app = DataApp( - name='streamlit-app', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-streamlit-1', - data_app_id='app-streamlit-1', - project_id='proj-1', - branch_id='branch-1', - config_version='1', - type='streamlit', - configuration={'parameters': {'dataApp': {'slug': 'streamlit-app'}}}, - state='running', - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=streamlit_app)) - - with pytest.raises(ValueError, match='only supports python-js data apps'): - await create_python_js_data_app_git_credential( - ctx=mcp_context_client, - configuration_id='cfg-streamlit-1', - ) - - keboola_client.data_science_client.create_app_git_credential.assert_not_called() - - -@pytest.mark.asyncio -async def test_create_python_js_data_app_git_credential_rejects_draft( - mocker, - mcp_context_client: Context, -) -> None: - """Drafts have no managed repo of their own — the tool must reject them early (before touching - get_app_git_repo) and point at the parent prod app, rather than falling through to the - misleading https_url=None 'platform bug' error.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - - draft = _make_python_js_draft_data_app( - configuration_id='cfg-draft-1', data_app_id='app-draft-1', parent_configuration_id='cfg-prod-1' - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=draft)) - - with pytest.raises(ValueError, match=r'is a python-js \*\*draft\*\*') as excinfo: - await create_python_js_data_app_git_credential( - ctx=mcp_context_client, - configuration_id='cfg-draft-1', - ) - - # The error must steer the caller to the parent prod app, and no repo/credential calls happen. - assert 'parentConfigurationId="cfg-prod-1"' in str(excinfo.value) - keboola_client.data_science_client.get_app_git_repo.assert_not_called() - keboola_client.data_science_client.create_app_git_credential.assert_not_called() - - -@pytest.mark.asyncio -async def test_create_python_js_data_app_git_credential_invalid_configuration_id( - mocker, - mcp_context_client: Context, -) -> None: - """Regression smoke test: _fetch_data_app's component_id validation still surfaces through the new tool.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client = mocker.AsyncMock() - - # Simulate a configuration_id that resolves to a non-data-app component_id, mirroring how - # _fetch_data_app raises today. - mocker.patch( - 'keboola_mcp_server.tools.data_apps._fetch_data_app', - mocker.AsyncMock( - side_effect=ValueError( - f'Data app tools only support {DATA_APP_COMPONENT_ID} component, but the data app ' - f'"app-x" has component_id "keboola.sandboxes".' - ) - ), - ) - - with pytest.raises(ValueError, match=f'Data app tools only support {DATA_APP_COMPONENT_ID} component'): - await create_python_js_data_app_git_credential( - ctx=mcp_context_client, - configuration_id='cfg-bogus', - ) - - keboola_client.data_science_client.create_app_git_credential.assert_not_called() - - -# ===== Tests for get_data_apps drafts list (detail path, python-js prod) ===== - - -from keboola_mcp_server.tools.data_apps import ( # noqa: E402 - DeletedDraftOutput, - delete_python_js_data_app_draft, -) - - -def _make_python_js_prod_data_app( - *, - configuration_id: str = 'cfg-prod-1', - data_app_id: str = 'app-prod-1', - repo_url: str | None = 'https://managed.repo/org/prod.git', - state: str = 'running', -) -> DataApp: - """A python-js **prod** app — no `isDraft` flag, no `parentConfigurationId`.""" - return DataApp( - name='Prod App', - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - data_app_id=data_app_id, - project_id='proj-1', - branch_id='branch-1', - config_version='1', - type='python-js', - configuration={'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'prod'}}}, - state=state, - repo_url=repo_url, - ) - - -def _make_python_js_draft_data_app( - *, - configuration_id: str, - data_app_id: str, - parent_configuration_id: str, - branch: str = 'init', - state: str = 'created', -) -> DataApp: - """A python-js **draft** app — `isDraft=true` and `parentConfigurationId` set.""" - return DataApp( - name=f'Draft {configuration_id}', - component_id=DATA_APP_COMPONENT_ID, - configuration_id=configuration_id, - data_app_id=data_app_id, - project_id='proj-1', - branch_id='branch-1', - config_version='1', - type='python-js', - configuration={ - 'parameters': { - 'autoSuspendAfterSeconds': 900, - 'dataApp': { - 'slug': f'draft-{configuration_id}', - 'isDraft': True, - 'parentConfigurationId': parent_configuration_id, - 'git': {'repository': 'https://managed.repo/org/prod.git', 'branch': branch}, - }, - }, - }, - state=state, - repo_url=None, - ) - - -def _build_storage_config_entry(*, cfg_id: str, parent_configuration_id: str | None, is_draft: bool = True) -> dict: - """Mirror the shape returned by `storage_client.configuration_list` for python-js apps. - - `is_draft=False` with a `parent_configuration_id` set models a misconfigured non-draft that - points at a prod but lacks the `isDraft` flag — it must NOT be surfaced as a draft. - """ - data_app_block: dict = {'slug': f'app-{cfg_id}'} - if parent_configuration_id is not None: - data_app_block['parentConfigurationId'] = parent_configuration_id - if is_draft: - data_app_block['isDraft'] = True - return { - 'id': cfg_id, - 'name': f'app-{cfg_id}', - 'configuration': { - 'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': data_app_block}, - }, - 'version': 1, - } - - -@pytest.mark.parametrize( - ('configuration', 'expected'), - [ - ({'parameters': {'dataApp': {'isDraft': True}}}, True), - ({'parameters': {'dataApp': {'isDraft': False}}}, False), - ({'parameters': {'dataApp': {'slug': 'prod'}}}, False), - ({'parameters': {}}, False), - ({}, False), - # Malformed/corrupted shapes must be treated as "not a draft", never raise AttributeError. - ({'parameters': {'dataApp': 'corrupted'}}, False), - ({'parameters': 'corrupted'}, False), - ({'parameters': None}, False), - ], - ids=[ - 'is_draft', - 'not_draft', - 'no_flag', - 'no_data_app', - 'empty', - 'data_app_not_mapping', - 'parameters_not_mapping', - 'parameters_none', - ], -) -def test_is_draft_config(configuration: dict, expected: bool) -> None: - """`_is_draft_config` is true only for `isDraft=true` and is shape-safe against malformed configs.""" - assert _is_draft_config(configuration) is expected - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - 'n_drafts', - [0, 1, 3], - ids=['no_drafts', 'one_draft', 'three_drafts'], -) -async def test_get_data_apps_detail_for_prod_lists_drafts( - mocker, - mcp_context_client: Context, - n_drafts: int, -) -> None: - """When fetching detail for a python-js prod, the response includes a `drafts: [...]` array - of every draft configured against it. Includes the 0-draft case to guard the empty path.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - prod_cfg_id = 'cfg-prod-1' - prod = _make_python_js_prod_data_app(configuration_id=prod_cfg_id) - draft_cfg_ids = [f'cfg-draft-{i}' for i in range(n_drafts)] - drafts = { - cfg_id: _make_python_js_draft_data_app( - configuration_id=cfg_id, - data_app_id=f'app-{cfg_id}', - parent_configuration_id=prod_cfg_id, - ) - for cfg_id in draft_cfg_ids - } - # Throw in a config that's parented to a DIFFERENT prod to verify we filter properly. - foreign_cfg = _build_storage_config_entry(cfg_id='cfg-other-draft', parent_configuration_id='cfg-prod-other') - configs = [ - _build_storage_config_entry(cfg_id=cfg_id, parent_configuration_id=prod_cfg_id) for cfg_id in draft_cfg_ids - ] - configs.append(foreign_cfg) - # A config that points at THIS prod but lacks `isDraft` is a misconfiguration, not a draft — - # it must be excluded (and never even fetched, or fake_fetch below would KeyError). - configs.append( - _build_storage_config_entry(cfg_id='cfg-non-draft-child', parent_configuration_id=prod_cfg_id, is_draft=False) - ) - # Also include the prod's own config (no parentConfigurationId) — must not be matched. - configs.append(_build_storage_config_entry(cfg_id=prod_cfg_id, parent_configuration_id=None)) - - keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=configs) - - async def fake_fetch(client, *, configuration_id, data_app_id): - if configuration_id == prod_cfg_id: - return prod - return drafts[configuration_id] - - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', side_effect=fake_fetch) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_logs', mocker.AsyncMock(return_value=[])) - - result = await get_data_apps(ctx=mcp_context_client, configuration_ids=[prod_cfg_id]) - assert len(result.data_apps) == 1 - detail = result.data_apps[0] - assert isinstance(detail, DataApp) - returned_draft_ids = sorted(d.configuration_id for d in detail.drafts) - assert returned_draft_ids == sorted(draft_cfg_ids) - # All drafts fetched successfully, so nothing was omitted. - assert detail.drafts_unavailable == 0 - - -@pytest.mark.asyncio -async def test_get_data_apps_detail_for_prod_counts_unavailable_drafts( - mocker, - mcp_context_client: Context, -) -> None: - """A transient DSAPI failure on one draft's detail fetch must NOT silently shrink the list: - the surviving draft is still returned and `drafts_unavailable` counts the omission so the - caller can tell "temporarily unreachable" from "deleted".""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - prod_cfg_id = 'cfg-prod-1' - prod = _make_python_js_prod_data_app(configuration_id=prod_cfg_id) - ok_cfg_id, failing_cfg_id = 'cfg-draft-ok', 'cfg-draft-fail' - ok_draft = _make_python_js_draft_data_app( - configuration_id=ok_cfg_id, data_app_id=f'app-{ok_cfg_id}', parent_configuration_id=prod_cfg_id - ) - configs = [ - _build_storage_config_entry(cfg_id=ok_cfg_id, parent_configuration_id=prod_cfg_id), - _build_storage_config_entry(cfg_id=failing_cfg_id, parent_configuration_id=prod_cfg_id), - _build_storage_config_entry(cfg_id=prod_cfg_id, parent_configuration_id=None), - ] - keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=configs) - - async def fake_fetch(client, *, configuration_id, data_app_id): - if configuration_id == prod_cfg_id: - return prod - if configuration_id == failing_cfg_id: - raise RuntimeError('transient DSAPI failure (timeout)') - return ok_draft - - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', side_effect=fake_fetch) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_logs', mocker.AsyncMock(return_value=[])) - - result = await get_data_apps(ctx=mcp_context_client, configuration_ids=[prod_cfg_id]) - detail = result.data_apps[0] - assert isinstance(detail, DataApp) - assert [d.configuration_id for d in detail.drafts] == [ok_cfg_id] - assert detail.drafts_unavailable == 1 - - -@pytest.mark.asyncio -async def test_get_data_apps_detail_for_draft_returns_empty_drafts( - mocker, - mcp_context_client: Context, -) -> None: - """Fetching detail for a draft must not recurse — its `drafts` array stays empty and the - cheap `configuration_list` lookup is skipped entirely.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - draft = _make_python_js_draft_data_app( - configuration_id='cfg-draft-1', data_app_id='app-draft-1', parent_configuration_id='cfg-prod-1' - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=draft)) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_logs', mocker.AsyncMock(return_value=[])) - keboola_client.storage_client.configuration_list = mocker.AsyncMock( - side_effect=AssertionError('Drafts must not trigger a drafts lookup') - ) - - result = await get_data_apps(ctx=mcp_context_client, configuration_ids=['cfg-draft-1']) - detail = result.data_apps[0] - assert isinstance(detail, DataApp) - assert detail.drafts == [] - keboola_client.storage_client.configuration_list.assert_not_called() - - -@pytest.mark.asyncio -async def test_get_data_apps_detail_for_streamlit_returns_empty_drafts( - mocker, - mcp_context_client: Context, - data_app: DataApp, -) -> None: - """Streamlit apps have no draft concept — the detail path must not call `configuration_list`.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - data_app.type = 'streamlit' - data_app.configuration = {'parameters': {'dataApp': {'slug': 'sl'}}} - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=data_app)) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_logs', mocker.AsyncMock(return_value=[])) - keboola_client.storage_client.configuration_list = mocker.AsyncMock( - side_effect=AssertionError('Streamlit apps must not trigger a drafts lookup') - ) - - result = await get_data_apps(ctx=mcp_context_client, configuration_ids=['cfg-streamlit-1']) - detail = result.data_apps[0] - assert isinstance(detail, DataApp) - assert detail.drafts == [] - keboola_client.storage_client.configuration_list.assert_not_called() - - -# ===== Tests for delete_python_js_data_app_draft ===== - - -@pytest.mark.asyncio -async def test_delete_python_js_data_app_draft_success( - mocker, - mcp_context_client: Context, -) -> None: - """Happy path: deletes the data app via DSAPI only and returns the parent configuration_id so - the agent can pivot back. The Storage config must NOT be deleted by the tool — DSAPI already - moves it to the trash, and a second delete would purge it from the trash permanently.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - draft = _make_python_js_draft_data_app( - configuration_id='cfg-draft-1', data_app_id='app-draft-1', parent_configuration_id='cfg-prod-1' - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=draft)) - keboola_client.data_science_client.delete_data_app = mocker.AsyncMock(return_value=None) - keboola_client.storage_client.configuration_delete = mocker.AsyncMock() - - result = await delete_python_js_data_app_draft(ctx=mcp_context_client, configuration_id='cfg-draft-1') - - assert isinstance(result, DeletedDraftOutput) - assert result.response == 'deleted' - assert result.configuration_id == 'cfg-draft-1' - assert result.data_app_id == 'app-draft-1' - assert result.parent_configuration_id == 'cfg-prod-1' - keboola_client.data_science_client.delete_data_app.assert_awaited_once_with('app-draft-1') - keboola_client.storage_client.configuration_delete.assert_not_called() - # The config link pivots to the parent prod and is labelled as such — not with the draft's name, - # which would mislabel a link pointing at a different configuration. - config_link = next(link for link in result.links if 'Data App Configuration' in link.title) - assert 'data-apps/cfg-prod-1' in config_link.url - assert 'parent prod app' in config_link.title - assert draft.name not in config_link.title - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('configuration', 'error_match'), - [ - ( - {'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'prod'}}}, - 'is a python-js .*prod.* app, not a draft', - ), - ( - {'parameters': {'autoSuspendAfterSeconds': 900, 'dataApp': {'slug': 'prod', 'isDraft': False}}}, - 'is a python-js .*prod.* app, not a draft', - ), - ], - ids=['no_isDraft_key', 'isDraft_false'], -) -async def test_delete_python_js_data_app_draft_refuses_prod( - mocker, - mcp_context_client: Context, - configuration: dict, - error_match: str, -) -> None: - """Refusing to delete prod apps is the single safety check — both shapes (missing flag or - explicit `false`) must be rejected, and neither delete endpoint must be called.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - prod = DataApp( - name='Prod', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-prod-1', - data_app_id='app-prod-1', - project_id='proj-1', - branch_id='branch-1', - config_version='1', - type='python-js', - configuration=configuration, - state='running', - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=prod)) - keboola_client.data_science_client.delete_data_app = mocker.AsyncMock() - keboola_client.storage_client.configuration_delete = mocker.AsyncMock() - - with pytest.raises(ValueError, match=error_match): - await delete_python_js_data_app_draft(ctx=mcp_context_client, configuration_id='cfg-prod-1') - - keboola_client.data_science_client.delete_data_app.assert_not_called() - keboola_client.storage_client.configuration_delete.assert_not_called() - - -@pytest.mark.asyncio -async def test_delete_python_js_data_app_draft_refuses_streamlit( - mocker, - mcp_context_client: Context, -) -> None: - """Streamlit apps have no draft concept — the tool must refuse and never call delete.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - streamlit_app = DataApp( - name='SL', - component_id=DATA_APP_COMPONENT_ID, - configuration_id='cfg-sl-1', - data_app_id='app-sl-1', - project_id='proj-1', - branch_id='branch-1', - config_version='1', - type='streamlit', - configuration={'parameters': {'dataApp': {'slug': 'sl'}}}, - state='running', - ) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=streamlit_app)) - keboola_client.data_science_client.delete_data_app = mocker.AsyncMock() - keboola_client.storage_client.configuration_delete = mocker.AsyncMock() - - with pytest.raises(ValueError, match='only supports python-js data apps'): - await delete_python_js_data_app_draft(ctx=mcp_context_client, configuration_id='cfg-sl-1') - - keboola_client.data_science_client.delete_data_app.assert_not_called() - keboola_client.storage_client.configuration_delete.assert_not_called() - - -def _make_failed_app_run(**overrides) -> AppRunResponse: - payload = { - 'id': 'run-1', - 'appId': 'app-prod-1', - 'state': 'failed', - 'createdAt': '2026-06-12T10:36:20+00:00', - 'startedAt': None, - 'stoppedAt': '2026-06-12T10:36:21+00:00', - 'startupLogs': None, - 'failureReason': { - 'reason': 'ConfigDecryptionFailed', - 'message': 'failed to decrypt key "#API_KEY"', - }, - 'mode': 'prod', - } - payload.update(overrides) - return AppRunResponse.model_validate(payload) - - -def test_app_run_info_flattens_failure_reason() -> None: - info = AppRunInfo.from_api_response(_make_failed_app_run()) - assert info.state == 'failed' - assert info.created_at == '2026-06-12T10:36:20+00:00' - assert info.stopped_at == '2026-06-12T10:36:21+00:00' - assert info.failure_reason == 'ConfigDecryptionFailed' - assert info.failure_message == 'failed to decrypt key "#API_KEY"' - assert info.startup_logs == [] - - -def test_app_run_info_handles_successful_run_without_failure_reason() -> None: - info = AppRunInfo.from_api_response( - _make_failed_app_run(state='finished', failureReason=None, startupLogs='booting\nready') - ) - assert info.failure_reason is None - assert info.failure_message is None - assert info.startup_logs == ['booting', 'ready'] - - -def test_app_run_info_truncates_long_logs_and_message() -> None: - long_logs = '\n'.join(f'line-{i}' for i in range(100)) - long_message = 'x' * (_APP_RUN_MESSAGE_LIMIT + 1000) - info = AppRunInfo.from_api_response( - _make_failed_app_run( - startupLogs=long_logs, - failureReason={'reason': 'StartupProbeFailed', 'message': long_message}, - ) - ) - # The error tail is what matters: keep the LAST lines/chars, marking message truncation with an ellipsis. - assert info.startup_logs == [f'line-{i}' for i in range(100 - _APP_RUN_LOG_LINES, 100)] - assert len(info.failure_message) == _APP_RUN_MESSAGE_LIMIT - assert info.failure_message.startswith('…') - - -@pytest.mark.asyncio -async def test_fetch_latest_run_returns_newest_run_info(mocker, mcp_context_client: Context) -> None: - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.data_science_client.list_app_runs = mocker.AsyncMock(return_value=[_make_failed_app_run()]) - - info = await _fetch_latest_run(keboola_client, 'app-prod-1') - - keboola_client.data_science_client.list_app_runs.assert_awaited_once_with('app-prod-1', limit=1) - assert info is not None - assert info.failure_reason == 'ConfigDecryptionFailed' - - -@pytest.mark.asyncio -@pytest.mark.parametrize('list_app_runs_behavior', ['empty', 'raises']) -async def test_fetch_latest_run_degrades_to_none( - mocker, mcp_context_client: Context, list_app_runs_behavior: str -) -> None: - """Diagnostics must not break the detail fetch: no runs (brand-new app) and a failing runs - endpoint (older DSAPI) both surface as `None` rather than an error.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - if list_app_runs_behavior == 'empty': - mock = mocker.AsyncMock(return_value=[]) - else: - mock = mocker.AsyncMock(side_effect=RuntimeError('404 Not Found')) - keboola_client.data_science_client.list_app_runs = mock - - assert await _fetch_latest_run(keboola_client, 'app-prod-1') is None - - -@pytest.mark.asyncio -async def test_get_data_apps_detail_includes_last_run_failure(mocker, mcp_context_client: Context) -> None: - """The detail path must surface the latest AppRun's failure so agents can diagnose apps whose - setup-phase failures (e.g. invalid secrets) produce no container logs at all.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - prod_cfg_id = 'cfg-prod-1' - prod = _make_python_js_prod_data_app(configuration_id=prod_cfg_id, state='stopped') - keboola_client.storage_client.configuration_list = mocker.AsyncMock(return_value=[]) - keboola_client.data_science_client.list_app_runs = mocker.AsyncMock(return_value=[_make_failed_app_run()]) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_data_app', mocker.AsyncMock(return_value=prod)) - mocker.patch('keboola_mcp_server.tools.data_apps._fetch_logs', mocker.AsyncMock(return_value=[])) - - result = await get_data_apps(ctx=mcp_context_client, configuration_ids=[prod_cfg_id]) - - assert len(result.data_apps) == 1 - detail = result.data_apps[0] - assert isinstance(detail, DataApp) - assert detail.deployment_info is not None - last_run = detail.deployment_info.last_run - assert last_run is not None - assert last_run.state == 'failed' - assert last_run.failure_reason == 'ConfigDecryptionFailed' - assert last_run.failure_message == 'failed to decrypt key "#API_KEY"' diff --git a/tests/tools/test_doc.py b/tests/tools/test_doc.py deleted file mode 100644 index 5b9a69db6..000000000 --- a/tests/tools/test_doc.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest -from mcp.server.fastmcp import Context -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.ai_service import DocsQuestionResponse -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.tools.doc import DocsAnswer, docs_query - - -@pytest.fixture -def mock_docs_response() -> DocsQuestionResponse: - """Mock response from the AI service client docs_question method.""" - return DocsQuestionResponse( - text='This is a test answer to the documentation query.', - source_urls=['https://docs.keboola.com/page1', 'https://docs.keboola.com/page2'], - ) - - -@pytest.mark.asyncio -async def test_docs_query( - mocker: MockerFixture, - mcp_context_client: Context, - mock_docs_response: DocsQuestionResponse, -): - """Tests docs_query tool with a mocked AI service client response.""" - context = mcp_context_client - keboola_client = KeboolaClient.from_state(context.session.state) - keboola_client.ai_service_client.docs_question = mocker.AsyncMock(return_value=mock_docs_response) - - query = 'How do I create a transformation?' - result = await docs_query(context, query) - - assert isinstance(result, DocsAnswer) - assert result.text == mock_docs_response.text - assert result.source_urls == mock_docs_response.source_urls - - keboola_client.ai_service_client.docs_question.assert_called_once_with(query) diff --git a/tests/tools/test_jobs.py b/tests/tools/test_jobs.py deleted file mode 100644 index 90e430f8c..000000000 --- a/tests/tools/test_jobs.py +++ /dev/null @@ -1,440 +0,0 @@ -from datetime import datetime -from typing import Any, Type, Union - -import pytest -from httpx import HTTPError -from mcp.server.fastmcp import Context -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.jobs import ( - GetJobsDetailOutput, - GetJobsListOutput, - JobDetail, - JobListItem, - JobLogEvent, - get_jobs, - run_job, -) - - -@pytest.fixture -def mock_jobs() -> list[dict[str, Any]]: - """list of mock jobs - simulating the api response.""" - return [ - { - 'id': '123', - 'status': 'success', - 'component': 'keboola.ex-aws-s3', - 'config': 'config-123', - 'isFinished': True, - 'createdTime': '2024-01-01T00:00:00Z', - 'startTime': '2024-01-01T00:00:01Z', - 'endTime': '2024-01-01T00:00:02Z', - 'not_a_desired_field': 'Should not be in the result', - }, - { - 'id': '124', - 'status': 'processing', - 'component': 'keboola.ex-aws-s3', - 'config': 'config-124', - 'isFinished': False, - 'createdTime': '2024-01-01T00:00:00Z', - 'startTime': '2024-01-01T00:00:01Z', - 'endTime': '2024-01-01T00:00:02Z', - 'not_a_desired_field': 'Should not be in the result', - }, - ] - - -@pytest.fixture -def mock_job() -> dict[str, Any]: - """mock job - simulating the api response.""" - return { - 'id': '123', - 'status': 'success', - 'component': 'keboola.ex-aws-s3', - 'config': 'config-123', - 'isFinished': True, - 'createdTime': '2024-01-01T00:00:00Z', - 'startTime': '2024-01-01T00:00:01Z', - 'endTime': '2024-01-01T00:00:02Z', - 'url': 'https://connection.keboola.com/jobs/123', - 'configData': {'source': 'file.csv'}, - 'configRow': '1', - 'runId': '456', - 'durationSeconds': 100, - 'result': {'import': 'successful'}, - 'metrics': {'rows': 1000}, - } - - -@pytest.fixture -def iso_format() -> str: - return '%Y-%m-%dT%H:%M:%SZ' - - -@pytest.mark.asyncio -async def test_get_jobs_listing( - mocker: MockerFixture, - mcp_context_client: Context, - mock_jobs: list[dict[str, Any]], - iso_format: str, -): - """Tests get_jobs tool when listing jobs.""" - context = mcp_context_client - keboola_client = KeboolaClient.from_state(context.session.state) - keboola_client.jobs_queue_client.search_jobs_by = mocker.AsyncMock(return_value=mock_jobs) - - result = await get_jobs(ctx=context) - - assert isinstance(result, GetJobsListOutput) - assert len(result.jobs) == 2 - assert all(isinstance(job, JobListItem) for job in result.jobs) - assert all(returned.id == expected['id'] for returned, expected in zip(result.jobs, mock_jobs)) - assert all(returned.status == expected['status'] for returned, expected in zip(result.jobs, mock_jobs)) - assert all(returned.component_id == expected['component'] for returned, expected in zip(result.jobs, mock_jobs)) - assert all(returned.config_id == expected['config'] for returned, expected in zip(result.jobs, mock_jobs)) - assert all(returned.is_finished == expected['isFinished'] for returned, expected in zip(result.jobs, mock_jobs)) - assert all( - returned.created_time is not None - and returned.created_time.replace(tzinfo=None) == datetime.strptime(expected['createdTime'], iso_format) - for returned, expected in zip(result.jobs, mock_jobs) - ) - assert all( - returned.start_time is not None - and returned.start_time.replace(tzinfo=None) == datetime.strptime(expected['startTime'], iso_format) - for returned, expected in zip(result.jobs, mock_jobs) - ) - assert all( - returned.end_time is not None - and returned.end_time.replace(tzinfo=None) == datetime.strptime(expected['endTime'], iso_format) - for returned, expected in zip(result.jobs, mock_jobs) - ) - assert all(hasattr(returned, 'not_a_desired_field') is False for returned in result.jobs) - assert len(result.links) == 1 - - keboola_client.jobs_queue_client.search_jobs_by.assert_called_once_with( - status=None, - component_id=None, - config_id=None, - limit=100, - offset=0, - sort_by='startTime', - sort_order='desc', - ) - - -@pytest.mark.asyncio -async def test_get_jobs_detail( - mocker: MockerFixture, mcp_context_client: Context, mock_job: dict[str, Any], iso_format: str -): - """Tests get_jobs tool when retrieving a specific job.""" - context = mcp_context_client - keboola_client = KeboolaClient.from_state(context.session.state) - keboola_client.jobs_queue_client.get_job_detail = mocker.AsyncMock(return_value=mock_job) - - result = await get_jobs(ctx=context, job_ids=('123',)) - - assert isinstance(result, GetJobsDetailOutput) - assert len(result.jobs) == 1 - job = result.jobs[0] - assert isinstance(job, JobDetail) - assert job.id == mock_job['id'] - assert job.status == mock_job['status'] - assert job.component_id == mock_job['component'] - assert job.config_id == mock_job['config'] - assert job.is_finished == mock_job['isFinished'] - assert job.created_time is not None - assert job.created_time.replace(tzinfo=None) == datetime.strptime(mock_job['createdTime'], iso_format) - assert job.start_time is not None - assert job.start_time.replace(tzinfo=None) == datetime.strptime(mock_job['startTime'], iso_format) - assert job.end_time is not None - assert job.end_time.replace(tzinfo=None) == datetime.strptime(mock_job['endTime'], iso_format) - assert job.url == mock_job['url'] - assert job.config_data == mock_job['configData'] - assert job.config_row == mock_job['configRow'] - assert job.run_id == mock_job['runId'] - assert job.duration_seconds == mock_job['durationSeconds'] - assert job.result == mock_job['result'] - - keboola_client.jobs_queue_client.get_job_detail.assert_called_once_with('123') - - -@pytest.mark.asyncio -async def test_get_jobs_listing_with_component_and_config_id( - mocker: MockerFixture, mcp_context_client: Context, mock_jobs: list[dict[str, Any]] -): - """ - Tests get_jobs tool with config_id and component_id. With config_id, the tool will return - only jobs for the given config_id and component_id. - """ - context = mcp_context_client - keboola_client = KeboolaClient.from_state(context.session.state) - keboola_client.jobs_queue_client.search_jobs_by = mocker.AsyncMock(return_value=mock_jobs) - - result = await get_jobs(ctx=context, job_ids=[], component_id='keboola.ex-aws-s3', config_id='config-123') - - assert len(result.jobs) == 2 - assert all(isinstance(job, JobListItem) for job in result.jobs) - assert all(returned.id == expected['id'] for returned, expected in zip(result.jobs, mock_jobs)) - assert all(returned.status == expected['status'] for returned, expected in zip(result.jobs, mock_jobs)) - - keboola_client.jobs_queue_client.search_jobs_by.assert_called_once_with( - status=None, - component_id='keboola.ex-aws-s3', - config_id='config-123', - sort_by='startTime', - sort_order='desc', - limit=100, - offset=0, - ) - - -@pytest.mark.asyncio -async def test_get_jobs_listing_with_component_id_without_config_id( - mocker: MockerFixture, mcp_context_client: Context, mock_jobs: list[dict[str, Any]] -): - """Tests get_jobs tool with component_id and without config_id. - It will return all jobs for the given component_id.""" - context = mcp_context_client - keboola_client = KeboolaClient.from_state(context.session.state) - keboola_client.jobs_queue_client.search_jobs_by = mocker.AsyncMock(return_value=mock_jobs) - - result = await get_jobs(ctx=context, component_id='keboola.ex-aws-s3') - - assert len(result.jobs) == 2 - assert all(isinstance(job, JobListItem) for job in result.jobs) - assert all(returned.id == expected['id'] for returned, expected in zip(result.jobs, mock_jobs)) - assert all(returned.status == expected['status'] for returned, expected in zip(result.jobs, mock_jobs)) - - keboola_client.jobs_queue_client.search_jobs_by.assert_called_once_with( - status=None, - component_id='keboola.ex-aws-s3', - config_id=None, - limit=100, - offset=0, - sort_by='startTime', - sort_order='desc', - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - 'configuration_row_ids', - [ - None, - ['row-1', 'row-2'], - ], -) -async def test_run_job( - mocker: MockerFixture, - mcp_context_client: Context, - mock_job: dict[str, Any], - configuration_row_ids: list[str] | None, -): - """Tests run_job tool with and without configuration_row_ids.""" - context = mcp_context_client - keboola_client = KeboolaClient.from_state(context.session.state) - mock_job['result'] = [] # simulate empty list as returned by create job endpoint - mock_job['status'] = 'created' # simulate created status as returned by create job endpoint - keboola_client.jobs_queue_client.create_job = mocker.AsyncMock(return_value=mock_job) - - component_id = mock_job['component'] - configuration_id = mock_job['config'] - job_detail = await run_job( - ctx=context, - component_id=component_id, - configuration_id=configuration_id, - configuration_row_ids=configuration_row_ids, - ) - - assert isinstance(job_detail, JobDetail) - assert job_detail.result == {} - assert job_detail.id == mock_job['id'] - assert job_detail.status == mock_job['status'] - assert job_detail.component_id == component_id - assert job_detail.config_id == configuration_id - assert set(job_detail.links) == { - Link( - type='ui-detail', title='Job: 123', url='https://connection.test.keboola.com/admin/projects/69420/queue/123' - ), - Link( - type='ui-dashboard', - title='Jobs in the project', - url='https://connection.test.keboola.com/admin/projects/69420/queue', - ), - } - - keboola_client.jobs_queue_client.create_job.assert_called_once_with( - component_id=component_id, - configuration_id=configuration_id, - configuration_row_ids=configuration_row_ids, - ) - - -@pytest.mark.asyncio -async def test_run_job_fail(mocker: MockerFixture, mcp_context_client: Context, mock_job: dict[str, Any]): - """Tests run_job tool when job creation fails.""" - context = mcp_context_client - keboola_client = KeboolaClient.from_state(context.session.state) - keboola_client.jobs_queue_client.create_job = mocker.AsyncMock(side_effect=HTTPError('Job creation failed')) - - component_id = mock_job['component'] - configuration_id = mock_job['config'] - - with pytest.raises(HTTPError): - await run_job(ctx=context, component_id=component_id, configuration_id=configuration_id) - - keboola_client.jobs_queue_client.create_job.assert_called_once_with( - component_id=component_id, - configuration_id=configuration_id, - configuration_row_ids=None, - ) - - -@pytest.mark.parametrize( - ('field_name', 'input_value', 'expected_result'), - [ - ('result', [], {}), # empty list is not a valid result type but we convert it to {}, no error - ('result', {}, {}), # expected empty dict, no error - ('result', {'result': []}, {'result': []}), # expected result type, no error - ('result', None, {}), # None is valid and converted to {} - ( - 'result', - ['result1', 'result2'], - ValueError, - ), # list is not a valid result type, we raise an error - ('configData', [], {}), # empty list is not a valid config_data type but we convert it to {}, no error - ('configData', {}, {}), # expected empty dict, no error - ('configData', ['configData1', 'configData2'], ValueError), # list is not a valid config_data type, - ], -) -def test_job_detail_model_validate_dict_fields( - field_name: str, - input_value: Union[list, dict, None], - expected_result: Union[dict, Type[Exception]], - mock_job: dict[str, Any], -): - """Tests JobDetail model validate for result field. - :param input_value: The input value to validate - simulating the api response. - :param expected_result: The expected result. - :param mock_job: The mock job details - expecting api response. - """ - mock_job[field_name] = input_value - mock_job['links'] = [] - if isinstance(expected_result, type) and issubclass(expected_result, Exception): - with pytest.raises(expected_result): - JobDetail.model_validate(mock_job) - else: - job_detail = JobDetail.model_validate(mock_job) - if field_name == 'result': - assert job_detail.result == expected_result - elif field_name == 'configData': - assert job_detail.config_data == expected_result - - -def test_job_log_event_model(): - """Tests JobLogEvent model validates correctly.""" - event = JobLogEvent.model_validate( - { - 'message': 'Processing started', - 'type': 'info', - 'created': '2024-01-01T00:00:01Z', - } - ) - assert event.message == 'Processing started' - assert event.type == 'info' - assert event.created is not None - - -def test_job_detail_with_logs(mock_job: dict[str, Any]): - """Tests JobDetail accepts optional logs field.""" - mock_job['links'] = [] - mock_job['logs'] = [ - {'message': 'Started', 'type': 'info', 'created': '2024-01-01T00:00:01Z'}, - {'message': 'Error happened', 'type': 'error', 'created': '2024-01-01T00:00:02Z'}, - ] - detail = JobDetail.model_validate(mock_job) - assert detail.logs is not None - assert len(detail.logs) == 2 - assert detail.logs[0].message == 'Started' - assert detail.logs[1].type == 'error' - - -def test_job_detail_without_logs(mock_job: dict[str, Any]): - """Tests JobDetail works without logs (backwards compatible).""" - mock_job['links'] = [] - detail = JobDetail.model_validate(mock_job) - assert detail.logs is None - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('get_jobs_kwargs', 'mock_events', 'expected_list_events_kwargs', 'expected_log_messages'), - [ - pytest.param( - {}, - None, - None, - None, - id='without_logs', - ), - pytest.param( - {'include_logs': True, 'log_tail_lines': 50}, - [ - {'uuid': 'evt-2', 'message': 'Finished', 'type': 'success', 'created': '2024-01-01T00:00:02Z'}, - {'uuid': 'evt-1', 'message': 'Started', 'type': 'info', 'created': '2024-01-01T00:00:01Z'}, - ], - {'job_id': '123', 'limit': 50}, - ['Started', 'Finished'], - id='with_logs', - ), - pytest.param( - {'include_logs': True, 'log_event_types': ['error']}, - [ - {'uuid': 'evt-3', 'message': 'Error happened', 'type': 'error', 'created': '2024-01-01T00:00:03Z'}, - {'uuid': 'evt-2', 'message': 'Finished row', 'type': 'info', 'created': '2024-01-01T00:00:02Z'}, - {'uuid': 'evt-1', 'message': 'Started', 'type': 'info', 'created': '2024-01-01T00:00:01Z'}, - ], - {'job_id': '123', 'limit': 50}, - ['Error happened'], - id='logs_type_filter', - ), - pytest.param( - {'include_logs': True}, - [], - {'job_id': '123', 'limit': 50}, - [], - id='logs_empty_events', - ), - ], -) -async def test_get_jobs_detail_logs( - mocker: MockerFixture, - mcp_context_client: Context, - mock_job: dict[str, Any], - get_jobs_kwargs: dict[str, Any], - mock_events: list[dict[str, Any]] | None, - expected_list_events_kwargs: dict[str, Any] | None, - expected_log_messages: list[str] | None, -): - """Tests get_jobs log-fetching behavior when retrieving job details.""" - context = mcp_context_client - keboola_client = KeboolaClient.from_state(context.session.state) - keboola_client.jobs_queue_client.get_job_detail = mocker.AsyncMock(return_value=mock_job) - keboola_client.storage_client.list_events = mocker.AsyncMock(return_value=mock_events or []) - - result = await get_jobs(ctx=context, job_ids=('123',), **get_jobs_kwargs) - - assert isinstance(result, GetJobsDetailOutput) - job = result.jobs[0] - - if expected_log_messages is None: - assert job.logs is None - keboola_client.storage_client.list_events.assert_not_called() - else: - assert job.logs is not None - assert [log.message for log in job.logs] == expected_log_messages - keboola_client.storage_client.list_events.assert_called_once_with(**expected_list_events_kwargs) diff --git a/tests/tools/test_oauth.py b/tests/tools/test_oauth.py deleted file mode 100644 index 77a5d6d16..000000000 --- a/tests/tools/test_oauth.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Tests for OAuth URL generation tools.""" - -from typing import Any, Mapping - -import pytest -from mcp.server.fastmcp import Context - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.tools.oauth import create_oauth_url - - -@pytest.fixture -def mock_token_response() -> Mapping[str, Any]: - """Mock valid response from the token creation endpoint.""" - return { - 'token': 'KBC_TOKEN_12345', - 'description': 'Short-lived token for OAuth URL - keboola.ex-google-analytics-v4/config-123', - 'expiresIn': 3600, - } - - -@pytest.mark.asyncio -async def test_create_oauth_url_success(mcp_context_client: Context, mock_token_response: Mapping[str, Any]) -> None: - """Test successful OAuth URL creation.""" - # Mock the storage client's token_create method to return the token response - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.token_create.return_value = mock_token_response - keboola_client.storage_api_url = 'https://connection.test.keboola.com' - - component_id = 'keboola.ex-google-analytics-v4' - config_id = 'config-123' - - result = await create_oauth_url(component_id=component_id, config_id=config_id, ctx=mcp_context_client) - - # Verify the storage client was called with correct parameters - keboola_client.storage_client.token_create.assert_called_once_with( - description=f'Short-lived token for OAuth URL - {component_id}/{config_id}', - component_access=[component_id], - expires_in=3600, - ) - - # Verify the response is the URL string - assert isinstance(result, str) - - expected_url = ( - f'https://external.keboola.com/oauth/index.html' - f'?token=KBC_TOKEN_12345' - f'&sapiUrl=https%3A%2F%2Fconnection.test.keboola.com' - f'#/{component_id}/{config_id}' - ) - assert result == expected_url - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('component_id', 'config_id'), - [ - ('keboola.ex-google-analytics-v4', 'my-config-123'), - ('keboola.ex-gmail', 'gmail-config-456'), - ('other.component', 'test-config'), - ], -) -async def test_create_oauth_url_different_components( - mcp_context_client: Context, - mock_token_response: Mapping[str, Any], - component_id: str, - config_id: str, -) -> None: - """Test OAuth URL creation for different components.""" - # Mock the storage client - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.token_create.return_value = mock_token_response - - result = await create_oauth_url(component_id=component_id, config_id=config_id, ctx=mcp_context_client) - - # Verify component-specific parameters were used - assert isinstance(result, str) - assert f'#/{component_id}/{config_id}' in result - - # Verify the API call included the correct component access - call_args = keboola_client.storage_client.token_create.call_args - assert call_args[1]['component_access'] == [component_id] - assert component_id in call_args[1]['description'] - assert config_id in call_args[1]['description'] - - -@pytest.mark.asyncio -async def test_create_oauth_url_token_creation_failure( - mcp_context_client: Context, -) -> None: - """Test OAuth URL creation when token creation fails.""" - # Mock the storage client to raise an exception - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.token_create.side_effect = Exception('Token creation failed') - - with pytest.raises(Exception, match='Token creation failed'): - await create_oauth_url( - component_id='keboola.ex-google-analytics-v4', config_id='config-123', ctx=mcp_context_client - ) - - -@pytest.mark.asyncio -async def test_create_oauth_url_missing_token_in_response(mcp_context_client: Context) -> None: - """Test OAuth URL creation when token is missing from response.""" - # Mock response without token field - invalid_response = { - 'description': 'Short-lived token for OAuth URL', - 'expiresIn': 3600, - } - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.token_create.return_value = invalid_response - - with pytest.raises(KeyError): - await create_oauth_url( - component_id='keboola.ex-google-analytics-v4', config_id='config-123', ctx=mcp_context_client - ) diff --git a/tests/tools/test_project.py b/tests/tools/test_project.py deleted file mode 100644 index 3e0ea2497..000000000 --- a/tests/tools/test_project.py +++ /dev/null @@ -1,248 +0,0 @@ -import pytest -from mcp.server.fastmcp import Context -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.project import ( - ProjectInfo, - _get_toolset_restrictions, - _resolve_branch_context, - get_project_info, - update_project_description, -) -from keboola_mcp_server.workspace import WorkspaceManager - - -@pytest.mark.parametrize( - ('role', 'expected_substring', 'expect_none'), - [ - # readonly: all writes blocked - ('readonly', 'read-only tools are available', False), - ('READONLY', 'read-only tools are available', False), - # regular roles: no schedules - ('guest', 'can manage flows', False), - ('guest', 'cannot set their schedules', False), - # empty role: no schedules - ('', 'cannot set their schedules', False), - # admin/share: no restrictions - ('admin', None, True), - ('share', None, True), - ], -) -def test_get_toolset_restrictions(role: str, expected_substring: str | None, expect_none: bool) -> None: - result = _get_toolset_restrictions(role) - if expect_none: - assert result is None - else: - assert result is not None - assert expected_substring in result - if role: - assert role.lower() in result - else: - assert 'unknown' in result - - -_DEFAULT_BRANCH = {'id': 123, 'name': 'Main', 'isDefault': True} -_DEV_BRANCH = {'id': 456, 'name': 'feature-x', 'isDefault': False} - - -@pytest.mark.parametrize( - ( - 'token_role', - 'expected_user_role', - 'expected_restriction_substrings', - 'restriction_is_none', - 'sql_dialect', - 'expected_fqn_example', - 'client_branch_id', - 'expected_branch_id', - 'expected_branch_name', - 'expected_is_dev', - ), - [ - # developer role on default branch - ( - 'developer', - 'developer', - ['cannot set their schedules'], - False, - 'Snowflake', - '"DATABASE"."SCHEMA"."TABLE"', - None, - 123, - 'Main', - False, - ), - # guest role on default branch - ( - 'guest', - 'guest', - ['cannot set their schedules'], - False, - 'BigQuery', - '`project`.`dataset`.`table`', - None, - 123, - 'Main', - False, - ), - # no role on default branch - ( - None, - 'unknown', - ['cannot set their schedules'], - False, - 'Snowflake', - '"DATABASE"."SCHEMA"."TABLE"', - None, - 123, - 'Main', - False, - ), - # readonly role on default branch - ( - 'readonly', - 'readonly', - ['read-only'], - False, - 'BigQuery', - '`project`.`dataset`.`table`', - None, - 123, - 'Main', - False, - ), - # admin role on default branch - ('admin', 'admin', [], True, 'Snowflake', '"DATABASE"."SCHEMA"."TABLE"', None, 123, 'Main', False), - # admin role on a dev branch — exercises the dev-branch resolution path - ('admin', 'admin', [], True, 'Snowflake', '"DATABASE"."SCHEMA"."TABLE"', '456', 456, 'feature-x', True), - ], -) -@pytest.mark.asyncio -async def test_get_project_info( - mocker: MockerFixture, - mcp_context_client: Context, - token_role: str | None, - expected_user_role: str, - expected_restriction_substrings: list[str], - restriction_is_none: bool, - sql_dialect: str, - expected_fqn_example: str, - client_branch_id: str | None, - expected_branch_id: int, - expected_branch_name: str, - expected_is_dev: bool, -) -> None: - admin_data = {'role': token_role} if token_role is not None else {} - token_data = { - 'owner': {'id': 'proj-123', 'name': 'Test Project'}, - 'organization': {'id': 'org-456'}, - 'admin': admin_data, - } - metadata = [ - {'key': MetadataField.PROJECT_DESCRIPTION, 'value': 'A test project.'}, - {'key': 'other', 'value': 'ignore'}, - ] - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.branch_id = client_branch_id - keboola_client.storage_client.verify_token = mocker.AsyncMock(return_value=token_data) - keboola_client.storage_client.branch_metadata_get = mocker.AsyncMock(return_value=metadata) - keboola_client.storage_client.branches_list = mocker.AsyncMock(return_value=[_DEFAULT_BRANCH, _DEV_BRANCH]) - workspace_manager = WorkspaceManager.from_state(mcp_context_client.session.state) - workspace_manager.get_sql_dialect = mocker.AsyncMock(return_value=sql_dialect) - workspace_manager.get_workspace_id = mocker.AsyncMock(return_value=789) - - project_id = 'proj-123' - base_url = 'https://connection.test.keboola.com' - links = [Link(type='ui-detail', title='Project Dashboard', url=f'{base_url}/admin/projects/{project_id}')] - mock_links_manager = mocker.Mock() - mock_links_manager.get_project_links.return_value = links - mocker.patch( - 'keboola_mcp_server.tools.project.ProjectLinksManager.from_client', - new=mocker.AsyncMock(return_value=mock_links_manager), - ) - - result = await get_project_info(mcp_context_client) - - assert isinstance(result, ProjectInfo) - assert result.project_id == 'proj-123' - assert result.project_name == 'Test Project' - assert result.organization_id == 'org-456' - assert result.project_description == 'A test project.' - assert result.sql_dialect == sql_dialect - assert result.workspace_id == 789 - assert result.links == links - assert result.user_role == expected_user_role - assert expected_fqn_example in result.llm_instruction - assert result.branch_id == expected_branch_id - assert result.branch_name == expected_branch_name - assert result.is_development_branch is expected_is_dev - - if restriction_is_none: - assert result.toolset_restrictions is None - else: - assert result.toolset_restrictions is not None - for substring in expected_restriction_substrings: - assert substring in result.toolset_restrictions - - -@pytest.mark.parametrize( - ('client_branch_id', 'branches', 'expected_id', 'expected_name', 'expected_is_dev'), - [ - # default branch resolution - (None, [_DEFAULT_BRANCH, _DEV_BRANCH], 123, 'Main', False), - # dev branch resolution by id (string vs int safe) - ('456', [_DEFAULT_BRANCH, _DEV_BRANCH], 456, 'feature-x', True), - (456, [_DEFAULT_BRANCH, _DEV_BRANCH], 456, 'feature-x', True), - # defensive: branch id present but not in list (should not happen, but covered) - ('999', [_DEFAULT_BRANCH], '999', 'unknown', True), - # defensive: empty list when on default - (None, [], 'default', 'unknown', False), - ], -) -@pytest.mark.asyncio -async def test_resolve_branch_context( - mocker: MockerFixture, - client_branch_id: str | int | None, - branches: list[dict], - expected_id: str | int, - expected_name: str, - expected_is_dev: bool, -) -> None: - client = mocker.Mock() - client.branch_id = client_branch_id - client.storage_client.branches_list = mocker.AsyncMock(return_value=branches) - - branch_id, branch_name, is_dev = await _resolve_branch_context(client) - - assert branch_id == expected_id - assert branch_name == expected_name - assert is_dev is expected_is_dev - - -@pytest.mark.parametrize( - 'description', - [ - 'New description', - '', - ], -) -@pytest.mark.asyncio -async def test_update_project_description( - mocker: MockerFixture, - mcp_context_client: Context, - description: str, -) -> None: - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.branch_metadata_update = mocker.AsyncMock( - return_value=[{'key': 'KBC.projectDescription', 'value': description}] - ) - - result = await update_project_description(mcp_context_client, description=description) - - assert result is None - keboola_client.storage_client.branch_metadata_update.assert_called_once_with( - {MetadataField.PROJECT_DESCRIPTION: description} - ) diff --git a/tests/tools/test_search.py b/tests/tools/test_search.py deleted file mode 100644 index 4728b4a73..000000000 --- a/tests/tools/test_search.py +++ /dev/null @@ -1,1559 +0,0 @@ -from typing import Any, cast -from unittest.mock import call - -import pytest -from fastmcp import Context -from fastmcp.exceptions import ToolError -from pytest_mock import MockerFixture - -from keboola_mcp_server.clients.ai_service import ComponentSuggestionResponse, SuggestedComponent -from keboola_mcp_server.clients.base import JsonDict -from keboola_mcp_server.clients.client import DATA_APP_COMPONENT_ID, KeboolaClient -from keboola_mcp_server.clients.storage import GlobalSearchResponse -from keboola_mcp_server.config import MetadataField -from keboola_mcp_server.links import Link -from keboola_mcp_server.tools.search import ( - SearchHit, - SearchItemType, - SearchOutput, - SearchSpec, - SuggestedComponentOutput, - find_component_id, - search, -) - - -class TestSearch: - """Test cases for the search tool function.""" - - @pytest.fixture(autouse=True) - def _mock_features(self, mocker: MockerFixture, mcp_context_client: Context): - """Disable storage-branches (no dual-fetch) and global-search (legacy textual path) features.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.has_feature = mocker.AsyncMock(return_value=False) - keboola_client.storage_client.is_enabled = mocker.AsyncMock(return_value=False) - - @pytest.mark.asyncio - async def test_search_no_patterns(self, mcp_context_client: Context): - with pytest.raises(ToolError, match='At least one search pattern must be provided.'): - await search(ctx=mcp_context_client, patterns=[]) - - with pytest.raises(ToolError, match='At least one search pattern must be provided.'): - await search(ctx=mcp_context_client, patterns=['']) - - @pytest.mark.asyncio - async def test_search_success(self, mocker: MockerFixture, mcp_context_client: Context): - """Test successful search with regex patterns.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - project_id = await keboola_client.storage_client.project_id() - - # Mock bucket_list - keboola_client.storage_client.bucket_list = mocker.AsyncMock( - return_value=[ - {'id': 'in.c-test-bucket', 'name': 'test-bucket', 'created': '2024-01-01T00:00:00Z'}, - ] - ) - - # Mock bucket_table_list - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock( - return_value=[ - { - 'id': 'in.c-test-bucket.test-table', - 'name': 'test-table', - 'created': '2024-01-01T00:00:00Z', - } - ] - ) - - # Mock component_list - return different results based on component type - def component_list_side_effect(component_type, include=None): - if component_type == 'extractor': - return [ - { - 'id': 'keboola.ex-db-mysql', - 'name': 'MySQL Extractor', - 'configurations': [ - { - 'id': 'test-config', - 'name': 'Test MySQL Config', - 'created': '2024-01-02T00:00:00Z', - 'rows': [], - } - ], - } - ] - return [] - - keboola_client.storage_client.component_list = mocker.AsyncMock(side_effect=component_list_side_effect) - - # Mock workspace_list - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - result = await search( - ctx=mcp_context_client, - patterns=['test'], - item_types=(cast(SearchItemType, 'table'), cast(SearchItemType, 'configuration')), - limit=20, - offset=0, - ) - - assert isinstance(result, SearchOutput) - assert result.total == 2 - assert result.branch_scope == 'current-branch' - assert result.hits == [ - SearchHit( - component_id='keboola.ex-db-mysql', - configuration_id='test-config', - item_type='configuration', - updated='2024-01-02T00:00:00Z', - name='Test MySQL Config', - links=[ - Link( - type='ui-detail', - title='Configuration: Test MySQL Config', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/components/keboola.ex-db-mysql/test-config' - ), - ) - ], - ), - SearchHit( - table_id='in.c-test-bucket.test-table', - item_type='table', - updated='2024-01-01T00:00:00Z', - name='test-table', - links=[ - Link( - type='ui-detail', - title='Table: test-table', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/storage/in.c-test-bucket/table/test-table' - ), - ) - ], - ), - ] - - @pytest.mark.asyncio - async def test_enumeration_filters_to_requested_item_types( - self, mocker: MockerFixture, mcp_context_client: Context - ): - """The legacy enumeration path must not leak configuration hits when only configuration-row is requested.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - keboola_client.storage_client.bucket_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - def component_list_side_effect(component_type, include=None): - if component_type == 'extractor': - return [ - { - 'id': 'keboola.ex-db-mysql', - 'name': 'MySQL Extractor', - 'configurations': [ - { - 'id': 'test-config', - 'name': 'test config', - 'created': '2024-01-02T00:00:00Z', - 'rows': [{'id': 'test-row', 'name': 'test row', 'created': '2024-01-03T00:00:00Z'}], - } - ], - } - ] - return [] - - keboola_client.storage_client.component_list = mocker.AsyncMock(side_effect=component_list_side_effect) - - result = await search( - ctx=mcp_context_client, - patterns=['test'], - item_types=(cast(SearchItemType, 'configuration-row'),), - ) - - assert {hit.item_type for hit in result.hits} == {'configuration-row'} - assert result.total == 1 - assert result.by_type == {'configuration-row': 1} - - @pytest.mark.asyncio - async def test_search_with_regex_pattern(self, mocker: MockerFixture, mcp_context_client: Context): - """Test search with regex patterns.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - project_id = await keboola_client.storage_client.project_id() - - # Mock bucket_list - keboola_client.storage_client.bucket_list = mocker.AsyncMock( - return_value=[ - {'id': 'in.c-customer-data', 'name': 'customer-data', 'created': '2024-01-01T00:00:00Z'}, - {'id': 'in.c-product-data', 'name': 'product-data', 'created': '2024-01-02T00:00:00Z'}, - ] - ) - - # Mock other endpoints - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.component_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - result = await search( - ctx=mcp_context_client, - patterns=['customer.*'], - item_types=(cast(SearchItemType, 'bucket'),), - mode='regex', - ) - - assert result.hits == [ - SearchHit( - bucket_id='in.c-customer-data', - item_type='bucket', - updated='2024-01-01T00:00:00Z', - name='customer-data', - links=[ - Link( - type='ui-detail', - title='Bucket: customer-data', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/storage/in.c-customer-data' - ), - ) - ], - ), - ] - - @pytest.mark.asyncio - async def test_search_default_parameters(self, mocker: MockerFixture, mcp_context_client: Context): - """Test search with default parameters (limit=50, offset=0, all item types).""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - # Create 60 buckets to verify default limit of 50 is applied - # Use lastChangeDate to ensure predictable sorting (most recent = bucket-059) - buckets = [ - { - 'id': f'in.c-test-bucket-{i:03d}', - 'name': f'test-bucket-{i:03d}', - 'created': '2024-01-01T00:00:00Z', - 'lastChangeDate': f'2024-01-01T{i:02d}:00:00Z', - } - for i in range(60) - ] - keboola_client.storage_client.bucket_list = mocker.AsyncMock(return_value=buckets) - - # Mock other endpoints - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.component_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - # Call without specifying limit, offset, or item_types - result = await search(ctx=mcp_context_client, patterns=['test']) - - # Should return exactly 50 items (default limit), not all 60 - assert len(result.hits) == 50, f'Expected default limit of 50, got {len(result.hits)}' - assert result.total == 60 - # The first item should be the most recently updated - assert result.hits[0].bucket_id == 'in.c-test-bucket-059' - - @pytest.mark.asyncio - async def test_search_limit_out_of_range(self, mocker: MockerFixture, mcp_context_client: Context): - """Test search with limit out of range gets clamped to default (50).""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - # Create 60 buckets to verify limit clamping - # Use lastChangeDate to ensure predictable sorting - buckets = [ - { - 'id': f'in.c-test-bucket-{i:03d}', - 'name': f'test-bucket-{i:03d}', - 'created': '2024-01-01T00:00:00Z', - 'lastChangeDate': f'2024-01-01T{i:02d}:00:00Z', - } - for i in range(60) - ] - keboola_client.storage_client.bucket_list = mocker.AsyncMock(return_value=buckets) - - # Mock other endpoints - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.component_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - # Test with limit too high (> MAX_GLOBAL_SEARCH_LIMIT = 100) - result = await search(ctx=mcp_context_client, patterns=['test'], limit=200) - # Should be overridden to DEFAULT_GLOBAL_SEARCH_LIMIT = 50 - assert len(result.hits) == 50, f'Expected limit to be overridden to 50, got {len(result.hits)}' - - # Test with limit too low (<= 0) - result = await search(ctx=mcp_context_client, patterns=['test'], limit=0) - assert len(result.hits) == 50, f'Expected limit to be overridden to 50, got {len(result.hits)}' - - # Test with negative limit - result = await search(ctx=mcp_context_client, patterns=['test'], limit=-5) - assert len(result.hits) == 50, f'Expected limit to be overridden to 50, got {len(result.hits)}' - - @pytest.mark.asyncio - async def test_search_negative_offset(self, mocker: MockerFixture, mcp_context_client: Context): - """Test search with negative offset gets clamped to 0.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - # Create buckets with predictable order - # Use lastChangeDate to ensure bucket-009 is the most recent - buckets = [ - { - 'id': f'in.c-test-bucket-{i:03d}', - 'name': f'test-bucket-{i:03d}', - 'created': '2024-01-01T00:00:00Z', - 'lastChangeDate': f'2024-01-01T{i:02d}:00:00Z', - } - for i in range(10) - ] - keboola_client.storage_client.bucket_list = mocker.AsyncMock(return_value=buckets) - - # Mock other endpoints - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.component_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - # Test with negative offset - result = await search(ctx=mcp_context_client, patterns=['test'], offset=-10, limit=5) - # Should be overridden to offset=0, returning first 5 items - assert len(result.hits) == 5 - # First item should be the most recently updated (bucket-009) - assert result.hits[0].bucket_id == 'in.c-test-bucket-009' - - # Verify it matches the result with offset=0 - result_with_zero_offset = await search(ctx=mcp_context_client, patterns=['test'], offset=0, limit=5) - assert result == result_with_zero_offset, 'Negative offset should behave the same as offset=0' - - @pytest.mark.asyncio - async def test_search_pagination(self, mocker: MockerFixture, mcp_context_client: Context): - """Test search with pagination.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - project_id = await keboola_client.storage_client.project_id() - - # Mock bucket_list with multiple items - buckets = [ - {'id': f'in.c-bucket-{i}', 'name': f'test-bucket-{i}', 'created': f'2024-01-{i:02d}T00:00:00Z'} - for i in range(1, 11) - ] - keboola_client.storage_client.bucket_list = mocker.AsyncMock(return_value=buckets) - - # Mock other endpoints - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.component_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - # Test pagination - result = await search(ctx=mcp_context_client, patterns=['test'], limit=2, offset=0) - assert result.hits == [ - SearchHit( - bucket_id='in.c-bucket-10', - item_type='bucket', - updated='2024-01-10T00:00:00Z', - name='test-bucket-10', - links=[ - Link( - type='ui-detail', - title='Bucket: test-bucket-10', - url=(f'https://connection.test.keboola.com/admin/projects/{project_id}/storage/in.c-bucket-10'), - ) - ], - ), - SearchHit( - bucket_id='in.c-bucket-9', - item_type='bucket', - updated='2024-01-09T00:00:00Z', - name='test-bucket-9', - links=[ - Link( - type='ui-detail', - title='Bucket: test-bucket-9', - url=(f'https://connection.test.keboola.com/admin/projects/{project_id}/storage/in.c-bucket-9'), - ) - ], - ), - ] - - result = await search(ctx=mcp_context_client, patterns=['test'], limit=1, offset=2) - assert result.hits == [ - SearchHit( - bucket_id='in.c-bucket-8', - item_type='bucket', - updated='2024-01-08T00:00:00Z', - name='test-bucket-8', - links=[ - Link( - type='ui-detail', - title='Bucket: test-bucket-8', - url=(f'https://connection.test.keboola.com/admin/projects/{project_id}/storage/in.c-bucket-8'), - ) - ], - ) - ] - - @pytest.mark.asyncio - async def test_search_matches_description(self, mocker: MockerFixture, mcp_context_client: Context): - """Test search matches description field.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - project_id = await keboola_client.storage_client.project_id() - - # Mock bucket_list with description - keboola_client.storage_client.bucket_list = mocker.AsyncMock( - return_value=[ - { - 'id': 'in.c-my-bucket', - 'name': 'my-bucket', - 'created': '2024-01-01T00:00:00Z', - 'metadata': [{'key': MetadataField.DESCRIPTION, 'value': 'This contains test data'}], - } - ] - ) - - # Mock other endpoints - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.component_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - result = await search(ctx=mcp_context_client, patterns=['test'], item_types=(cast(SearchItemType, 'bucket'),)) - - assert result.hits == [ - SearchHit( - bucket_id='in.c-my-bucket', - item_type='bucket', - updated='2024-01-01T00:00:00Z', - name='my-bucket', - description='This contains test data', - links=[ - Link( - type='ui-detail', - title='Bucket: my-bucket', - url=(f'https://connection.test.keboola.com/admin/projects/{project_id}/storage/in.c-my-bucket'), - ) - ], - ) - ] - - @pytest.mark.asyncio - async def test_search_hits_sorting(self, mocker: MockerFixture, mcp_context_client: Context): - """Test search hits sorting.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - project_id = await keboola_client.storage_client.project_id() - - keboola_client.storage_client.bucket_list = mocker.AsyncMock( - return_value=[ - {'id': 'in.c-test-bucket-a', 'name': 'test-bucket-a', 'created': '2024-01-01T00:00:00Z'}, - { - 'id': 'in.c-test-bucket-b', - 'name': 'test-bucket-b', - 'created': '2024-01-01T00:00:00Z', - 'lastChangeDate': '2024-01-02T00:00:00Z', - }, - {'id': 'in.c-test-bucket-c', 'name': 'test-bucket-c'}, - ] - ) - - def _bucket_table_list_side_effect(bucket_id: str, include: Any = None, **kwargs: Any) -> list[JsonDict]: - if bucket_id == 'in.c-test-bucket-a': - return [ - {'id': 'in.c-test-bucket-a.test-table', 'name': 'test-table', 'created': '2024-01-01T00:00:00Z'} - ] - elif bucket_id == 'in.c-test-bucket-b': - return [ - { - 'id': 'in.c-test-bucket-b.test-table', - 'name': 'test-table', - 'created': '2024-01-01T00:00:00Z', - 'lastChangeDate': '2024-01-02T00:00:00Z', - } - ] - else: - return [] - - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(side_effect=_bucket_table_list_side_effect) - - def _component_list_side_effect( - component_type: str | None = None, include: Any | None = None - ) -> list[JsonDict]: - if not component_type: - return [ - { - 'id': 'keboola.ex-db-mysql', - 'name': 'MySQL Extractor', - 'configurations': [ - { - 'id': 'test-config-a', - 'name': 'Test MySQL Config A', - 'created': '2024-01-03T00:00:00Z', - 'rows': [], - }, - { - 'id': 'test-config-b', - 'name': 'Test MySQL Config B', - 'created': '2024-01-03T00:00:00Z', - 'currentVersion': { - 'created': '2024-01-04T00:00:00Z', - }, - 'rows': [], - }, - ], - } - ] - else: - return [] - - keboola_client.storage_client.component_list = mocker.AsyncMock(side_effect=_component_list_side_effect) - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - result = await search(ctx=mcp_context_client, patterns=['test'], limit=20, offset=0) - - assert result.hits == [ - SearchHit( - component_id='keboola.ex-db-mysql', - configuration_id='test-config-b', - item_type='configuration', - updated='2024-01-04T00:00:00Z', - name='Test MySQL Config B', - links=[ - Link( - type='ui-detail', - title='Configuration: Test MySQL Config B', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/components/keboola.ex-db-mysql/test-config-b' - ), - ) - ], - ), - SearchHit( - component_id='keboola.ex-db-mysql', - configuration_id='test-config-a', - item_type='configuration', - updated='2024-01-03T00:00:00Z', - name='Test MySQL Config A', - links=[ - Link( - type='ui-detail', - title='Configuration: Test MySQL Config A', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/components/keboola.ex-db-mysql/test-config-a' - ), - ) - ], - ), - SearchHit( - table_id='in.c-test-bucket-b.test-table', - item_type='table', - updated='2024-01-02T00:00:00Z', - name='test-table', - links=[ - Link( - type='ui-detail', - title='Table: test-table', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/storage/in.c-test-bucket-b/table/test-table' - ), - ) - ], - ), - SearchHit( - bucket_id='in.c-test-bucket-b', - item_type='bucket', - updated='2024-01-02T00:00:00Z', - name='test-bucket-b', - links=[ - Link( - type='ui-detail', - title='Bucket: test-bucket-b', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/storage/in.c-test-bucket-b' - ), - ) - ], - ), - SearchHit( - table_id='in.c-test-bucket-a.test-table', - item_type='table', - updated='2024-01-01T00:00:00Z', - name='test-table', - links=[ - Link( - type='ui-detail', - title='Table: test-table', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/storage/in.c-test-bucket-a/table/test-table' - ), - ) - ], - ), - SearchHit( - bucket_id='in.c-test-bucket-a', - item_type='bucket', - updated='2024-01-01T00:00:00Z', - name='test-bucket-a', - links=[ - Link( - type='ui-detail', - title='Bucket: test-bucket-a', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/storage/in.c-test-bucket-a' - ), - ) - ], - ), - SearchHit( - bucket_id='in.c-test-bucket-c', - item_type='bucket', - updated='', - name='test-bucket-c', - links=[ - Link( - type='ui-detail', - title='Bucket: test-bucket-c', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/storage/in.c-test-bucket-c' - ), - ) - ], - ), - ] - - keboola_client.storage_client.bucket_list.assert_has_calls( - [call(branch_id='default'), call(branch_id='default')] - ) - keboola_client.storage_client.bucket_table_list.assert_has_calls( - [ - call('in.c-test-bucket-a', include=['columns', 'columnMetadata'], branch_id='default'), - call('in.c-test-bucket-b', include=['columns', 'columnMetadata'], branch_id='default'), - call('in.c-test-bucket-c', include=['columns', 'columnMetadata'], branch_id='default'), - ] - ) - keboola_client.storage_client.component_list.assert_called_once_with(None, include=['configuration', 'rows']) - keboola_client.storage_client.workspace_list.assert_not_called() - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('tables_data', 'search_pattern', 'expected_count', 'expected_first_table_id'), - [ - # Test: search finds table by matching column name - ( - [ - { - 'id': 'in.c-test-bucket.users', - 'name': 'users', - 'created': '2024-01-01T00:00:00Z', - 'columns': ['id', 'email', 'name'], - 'columnMetadata': {}, - } - ], - 'email', - 1, - 'in.c-test-bucket.users', - ), - # Test: search finds table by matching column description - ( - [ - { - 'id': 'in.c-test-bucket.customers', - 'name': 'customers', - 'created': '2024-01-01T00:00:00Z', - 'columns': ['id', 'contact_info'], - 'columnMetadata': { - 'contact_info': [{'key': MetadataField.DESCRIPTION, 'value': 'Customer email address'}] - }, - } - ], - 'email', - 1, - 'in.c-test-bucket.customers', - ), - # Test: table appears only once when both table name and column match - ( - [ - { - 'id': 'in.c-test-bucket.customer_data', - 'name': 'customer_data', - 'created': '2024-01-01T00:00:00Z', - 'columns': ['customer_id', 'name', 'email'], - 'columnMetadata': {}, - } - ], - 'customer', - 1, - 'in.c-test-bucket.customer_data', - ), - # Test: handles tables without columns or columnMetadata gracefully - ( - [ - { - 'id': 'in.c-test-bucket.table1', - 'name': 'table1', - 'created': '2024-01-01T00:00:00Z', - # No 'columns' field - # No 'columnMetadata' field - }, - { - 'id': 'in.c-test-bucket.table2', - 'name': 'table2', - 'created': '2024-01-01T00:00:00Z', - 'columns': [], # Empty columns - 'columnMetadata': {}, # Empty metadata - }, - ], - 'test', - 2, - 'in.c-test-bucket.table2', # table2 comes first due to reverse sorting by ID - ), - ], - ) - async def test_search_table_by_columns( - self, - mocker: MockerFixture, - mcp_context_client: Context, - tables_data: list[JsonDict], - search_pattern: str, - expected_count: int, - expected_first_table_id: str, - ): - """Test search functionality with table columns and metadata.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - # Mock bucket_list - keboola_client.storage_client.bucket_list = mocker.AsyncMock( - return_value=[ - {'id': 'in.c-test-bucket', 'name': 'test-bucket', 'created': '2024-01-01T00:00:00Z'}, - ] - ) - - # Mock bucket_table_list with provided test data - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(return_value=tables_data) - - result = await search( - ctx=mcp_context_client, patterns=[search_pattern], item_types=(cast(SearchItemType, 'table'),) - ) - - assert len(result.hits) == expected_count - if expected_count > 0: - assert result.hits[0].table_id == expected_first_table_id - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ( - 'patterns', - 'scopes', - 'component_configurations', - 'expected_hits', - ), - [ - ( - ['alpha', 'beta'], - ('parameters', 'storage.input'), - [ - { - 'id': 'test-config', - 'name': 'Test Config', - 'created': '2024-01-02T00:00:00Z', - 'configuration': { - 'parameters': {'query': 'alpha'}, - 'storage': {'input': [{'source': 'beta'}]}, - }, - 'rows': [], - } - ], - [ - ( - 'test-config', - [ - {'scope': 'parameters.query', 'patterns': ['alpha']}, - {'scope': 'storage.input[0].source', 'patterns': ['beta']}, - ], - ) - ], - ), - ( - ['gamma'], - tuple(), - [ - { - 'id': 'test-config', - 'name': 'Test Config', - 'created': '2024-01-02T00:00:00Z', - 'configuration': { - 'parameters': {'query': 'alpha'}, - 'storage': { - 'input': [{'source': 'beta'}, {'source': 'gamma'}], - 'output': [{'destination': 'gamma'}], - }, - }, - 'rows': [], - } - ], - [ - ( - 'test-config', - [ - {'scope': 'storage.input[1].source', 'patterns': ['gamma']}, - {'scope': 'storage.output[0].destination', 'patterns': ['gamma']}, - ], - ) - ], - ), - ( - ['alpha'], - ('parameters',), - [ - { - 'id': 'test-config', - 'name': 'Test Config', - 'created': '2024-01-02T00:00:00Z', - 'configuration': { - 'parameters': {'query': 'alpha'}, - 'storage': {'input': [{'source': 'alpha'}]}, - }, - 'rows': [], - } - ], - [('test-config', [{'scope': 'parameters.query', 'patterns': ['alpha']}])], - ), - ( - ['alpha'], - ('authorization.#apiKey',), - [ - { - 'id': 'test-config', - 'name': 'Test Config', - 'created': '2024-01-02T00:00:00Z', - 'configuration': { - 'authorization': {'#apiKey': 'alpha'}, - 'parameters': {'query': 'nomatch'}, - }, - 'rows': [], - } - ], - [('test-config', [{'scope': 'authorization.#apiKey', 'patterns': ['alpha']}])], - ), - ( - ['alpha', 'beta'], - ('parameters',), - [ - { - 'id': 'test-config', - 'name': 'Test Config', - 'created': '2024-01-02T00:00:00Z', - 'configuration': { - 'parameters': {'query': 'alpha beta', 'query2': 'beta'}, - }, - 'rows': [], - } - ], - [ - ( - 'test-config', - [ - {'scope': 'parameters.query', 'patterns': ['alpha', 'beta']}, - {'scope': 'parameters.query2', 'patterns': ['beta']}, - ], - ) - ], - ), - ( - ['alpha', 'gamma'], - tuple(), - [ - { - 'id': 'test-config-a', - 'name': 'Test Config A', - 'created': '2024-01-02T00:00:00Z', - 'configuration': { - 'parameters': {'query': 'alpha'}, - 'storage': {'input': [{'source': 'beta'}]}, - }, - 'rows': [], - }, - { - 'id': 'test-config-b', - 'name': 'Test Config B', - 'created': '2024-01-03T00:00:00Z', - 'configuration': { - 'storage': {'output': [{'destination': 'gamma'}]}, - }, - 'rows': [], - }, - { - 'id': 'test-config-c', - 'name': 'Test Config C', - 'created': '2024-01-01T00:00:00Z', - 'configuration': { - 'parameters': {'query': 'nomatch'}, - }, - 'rows': [], - }, - ], - [ - ('test-config-b', [{'scope': 'storage.output[0].destination', 'patterns': ['gamma']}]), - ('test-config-a', [{'scope': 'parameters.query', 'patterns': ['alpha']}]), - ], - ), - ], - ids=[ - 'all_matches_in_scopes', - 'most_specific_scope_only', - 'scope_constrains_same_value_in_other_path', - 'hash_prefixed_scope_key_in_search_tool', - 'group_two_patterns_in_one_scope', - 'multiple_configurations_returned', - ], - ) - async def test_search_config_based_match_scopes( - self, - mocker: MockerFixture, - mcp_context_client: Context, - patterns: list[str], - scopes: tuple[str, ...], - component_configurations: list[dict[str, Any]], - expected_hits: list[tuple[str, list[dict[str, Any]]]], - ): - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - - keboola_client.storage_client.bucket_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.bucket_table_list = mocker.AsyncMock(return_value=[]) - keboola_client.storage_client.component_list = mocker.AsyncMock( - side_effect=lambda component_type, include=None: ( - [ - { - 'id': 'keboola.ex-db-mysql', - 'type': 'extractor', - 'configurations': component_configurations, - } - ] - if component_type == 'extractor' - else [] - ) - ) - keboola_client.storage_client.workspace_list = mocker.AsyncMock(return_value=[]) - - result = await search( - ctx=mcp_context_client, - patterns=patterns, - item_types=(cast(SearchItemType, 'configuration'),), - search_type='config-based', - scopes=scopes, - ) - - normalized_actual = [ - ( - hit.configuration_id, - sorted( - ({'scope': m.scope, 'patterns': sorted(m.patterns)} for m in hit.matches), - key=lambda x: x['scope'] or '', - ), - ) - for hit in result.hits - ] - normalized_expected = [ - ( - config_id, - sorted( - ({'scope': m['scope'], 'patterns': sorted(m['patterns'])} for m in matches), - key=lambda x: x['scope'] or '', - ), - ) - for config_id, matches in expected_hits - ] - assert normalized_actual == normalized_expected - - -def _global_search_item( - item_id: str, - name: str, - item_type: str, - *, - component_id: str | None = None, - full_path: dict[str, Any] | None = None, - created: str = '2024-01-01T00:00:00+00:00', -) -> JsonDict: - return { - 'id': item_id, - 'name': name, - 'type': item_type, - 'fullPath': full_path or {}, - 'componentId': component_id, - 'organizationId': 1, - 'projectId': 69420, - 'projectName': 'Test Project', - 'created': created, - } - - -def _global_search_response(*items: JsonDict) -> GlobalSearchResponse: - by_type: dict[str, int] = {} - for item in items: - item_type = cast(str, item['type']) - by_type[item_type] = by_type.get(item_type, 0) + 1 - return GlobalSearchResponse.model_validate( - {'all': len(items), 'items': list(items), 'byType': by_type, 'byProject': {'69420': 'Test Project'}} - ) - - -class TestGlobalTextualSearch: - """Test cases for the textual search backed by the SAPI global-search endpoint.""" - - @pytest.fixture(autouse=True) - def _enable_global_search(self, mocker: MockerFixture, mcp_context_client: Context): - """Enable the global-search feature so that textual search uses the server-side endpoint.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.is_enabled = mocker.AsyncMock(return_value=True) - - @pytest.mark.asyncio - async def test_search_maps_global_search_items(self, mocker: MockerFixture, mcp_context_client: Context): - """Items are mapped to SearchHits with IDs, branch info, re-typed flows and links.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - project_id = await keboola_client.storage_client.project_id() - - response = _global_search_response( - _global_search_item( - 'in.c-test-bucket.users', - 'users', - 'table', - full_path={'bucket': {'id': 'in.c-test-bucket'}, 'branch': {'id': 7, 'name': 'Main'}}, - created='2024-01-03T00:00:00+00:00', - ), - _global_search_item( - 'cfg-1', - 'Test MySQL Config', - 'configuration', - component_id='keboola.ex-db-mysql', - created='2024-01-02T00:00:00+00:00', - ), - _global_search_item( - 'flow-1', - 'My Flow', - 'configuration', - component_id='keboola.orchestrator', - created='2024-01-01T00:00:00+00:00', - ), - ) - keboola_client.storage_client.global_search = mocker.AsyncMock(return_value=response) - - result = await search(ctx=mcp_context_client, patterns=['test']) - - keboola_client.storage_client.global_search.assert_called_once_with( - query='test', types=[], limit=50, offset=0, branch_scope='current' - ) - assert isinstance(result, SearchOutput) - assert result.branch_scope == 'current-branch' - assert result.total == 3 - assert result.by_type == {'table': 1, 'configuration': 2} - assert result.hits == [ - SearchHit( - table_id='in.c-test-bucket.users', - bucket_id='in.c-test-bucket', - item_type='table', - updated='2024-01-03T00:00:00+00:00', - name='users', - branch_id='7', - branch_name='Main', - links=[ - Link( - type='ui-detail', - title='Table: users', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/storage/in.c-test-bucket/table/users' - ), - ) - ], - ), - SearchHit( - component_id='keboola.ex-db-mysql', - configuration_id='cfg-1', - item_type='configuration', - updated='2024-01-02T00:00:00+00:00', - name='Test MySQL Config', - links=[ - Link( - type='ui-detail', - title='Configuration: Test MySQL Config', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/components/keboola.ex-db-mysql/cfg-1' - ), - ) - ], - ), - SearchHit( - component_id='keboola.orchestrator', - configuration_id='flow-1', - item_type='flow', - updated='2024-01-01T00:00:00+00:00', - name='My Flow', - links=[ - Link( - type='ui-detail', - title='Flow: My Flow', - url=(f'https://connection.test.keboola.com/admin/projects/{project_id}' '/flows/flow-1'), - ) - ], - ), - ] - - @pytest.mark.asyncio - async def test_search_widens_to_all_branches_on_zero_hits(self, mocker: MockerFixture, mcp_context_client: Context): - """When the current branch context has no hits, the search retries across all branches.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - dev_branch_hit = _global_search_response( - _global_search_item( - 'in.c-dev-bucket.users', - 'users', - 'table', - full_path={'branch': {'id': 123, 'name': 'my-dev-branch'}}, - ), - ) - keboola_client.storage_client.global_search = mocker.AsyncMock( - side_effect=[_global_search_response(), dev_branch_hit] - ) - - result = await search(ctx=mcp_context_client, patterns=['users'], item_types=('table',)) - - assert keboola_client.storage_client.global_search.call_args_list == [ - call(query='users', types=['table'], limit=50, offset=0, branch_scope='current'), - call(query='users', types=['table'], limit=50, offset=0, branch_scope='all'), - ] - assert result.branch_scope == 'all-branches' - assert len(result.hits) == 1 - assert result.hits[0].table_id == 'in.c-dev-bucket.users' - assert result.hits[0].branch_id == '123' - assert result.hits[0].branch_name == 'my-dev-branch' - - @pytest.mark.asyncio - async def test_search_does_not_widen_when_paginating(self, mocker: MockerFixture, mcp_context_client: Context): - """An empty page with non-zero offset must not trigger the all-branches retry.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.global_search = mocker.AsyncMock(return_value=_global_search_response()) - - result = await search(ctx=mcp_context_client, patterns=['users'], offset=10) - - keboola_client.storage_client.global_search.assert_called_once_with( - query='users', types=[], limit=50, offset=10, branch_scope='current' - ) - assert result.hits == [] - assert result.branch_scope == 'current-branch' - - @pytest.mark.asyncio - async def test_search_falls_back_to_enumeration_on_zero_hits( - self, mocker: MockerFixture, mcp_context_client: Context - ): - """Zero hits (even after the all-branches retry) means the index may not be back-filled — - fall back to client-side enumeration so we never silently return nothing.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.global_search = mocker.AsyncMock(return_value=_global_search_response()) - fallback = SearchOutput( - hits=[SearchHit(table_id='in.c-main.users', item_type='table', updated='', name='users')], - total=1, - by_type={'table': 1}, - branch_scope='current-branch', - ) - enum_mock = mocker.patch( - 'keboola_mcp_server.tools.search._enumeration_search', - new=mocker.AsyncMock(return_value=fallback), - ) - - result = await search(ctx=mcp_context_client, patterns=['users'], item_types=('table',)) - - enum_mock.assert_awaited_once() - assert [h.table_id for h in result.hits] == ['in.c-main.users'] - - @pytest.mark.asyncio - async def test_search_falls_back_to_enumeration_on_error(self, mocker: MockerFixture, mcp_context_client: Context): - """A failing global-search request (e.g. a transient 5xx) falls back to enumeration.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.global_search = mocker.AsyncMock( - side_effect=RuntimeError('global-search exploded') - ) - fallback = SearchOutput( - hits=[SearchHit(table_id='in.c-main.users', item_type='table', updated='', name='users')], - total=1, - by_type={'table': 1}, - branch_scope='current-branch', - ) - enum_mock = mocker.patch( - 'keboola_mcp_server.tools.search._enumeration_search', - new=mocker.AsyncMock(return_value=fallback), - ) - - result = await search(ctx=mcp_context_client, patterns=['users'], item_types=('table',)) - - enum_mock.assert_awaited_once() - assert [h.table_id for h in result.hits] == ['in.c-main.users'] - - @pytest.mark.asyncio - async def test_search_multiple_patterns_merge_and_dedupe(self, mocker: MockerFixture, mcp_context_client: Context): - """Each pattern issues its own request; results are OR-merged and deduplicated by item.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - shared_item = _global_search_item('cfg-1', 'Shared Config', 'configuration', component_id='keboola.ex-db-mysql') - unique_item = _global_search_item('cfg-2', 'Unique Config', 'configuration', component_id='keboola.ex-db-mysql') - keboola_client.storage_client.global_search = mocker.AsyncMock( - side_effect=[ - _global_search_response(shared_item, unique_item), - _global_search_response(shared_item), - ] - ) - - result = await search(ctx=mcp_context_client, patterns=['shared', 'config']) - - assert keboola_client.storage_client.global_search.call_count == 2 - assert sorted(hit.configuration_id for hit in result.hits) == ['cfg-1', 'cfg-2'] - # The shared item is counted once per pattern in the server-side totals. - assert result.total == 3 - - @pytest.mark.asyncio - async def test_search_data_app_narrowing(self, mocker: MockerFixture, mcp_context_client: Context): - """Searching for data-apps over-fetches configurations and narrows them by component ID.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - response = _global_search_response( - _global_search_item('app-1', 'My Data App', 'configuration', component_id=DATA_APP_COMPONENT_ID), - _global_search_item('cfg-1', 'Regular Config', 'configuration', component_id='keboola.ex-db-mysql'), - ) - keboola_client.storage_client.global_search = mocker.AsyncMock(return_value=response) - - result = await search(ctx=mcp_context_client, patterns=['app'], item_types=('data-app',)) - - # Narrowing 'configuration' to data-apps is lossy, so the page is over-fetched up to the server max - # to avoid under-filling once the non-matching configurations are dropped client-side. - keboola_client.storage_client.global_search.assert_called_once_with( - query='app', types=['configuration'], limit=100, offset=0, branch_scope='current' - ) - assert len(result.hits) == 1 - assert result.hits[0].configuration_id == 'app-1' - assert result.hits[0].item_type == 'data-app' - - @pytest.mark.asyncio - async def test_search_overfetch_fills_page_after_narrowing( - self, mocker: MockerFixture, mcp_context_client: Context - ): - """A small user limit still over-fetches to the server max, then caps the narrowed page to that limit. - - Regular configurations preceding the data-apps would under-fill the page if the user limit were sent - verbatim; over-fetching ensures the page reaches the requested limit when enough data-apps exist. - """ - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - items = [ - _global_search_item('cfg-1', 'Regular One', 'configuration', component_id='keboola.ex-db-mysql'), - _global_search_item('cfg-2', 'Regular Two', 'configuration', component_id='keboola.ex-db-mysql'), - _global_search_item('app-1', 'Data App One', 'configuration', component_id=DATA_APP_COMPONENT_ID), - _global_search_item('app-2', 'Data App Two', 'configuration', component_id=DATA_APP_COMPONENT_ID), - _global_search_item('app-3', 'Data App Three', 'configuration', component_id=DATA_APP_COMPONENT_ID), - ] - keboola_client.storage_client.global_search = mocker.AsyncMock(return_value=_global_search_response(*items)) - - result = await search(ctx=mcp_context_client, patterns=['app'], item_types=('data-app',), limit=2) - - # The user limit (2) is below the server max, so the request over-fetches up to 100... - keboola_client.storage_client.global_search.assert_called_once_with( - query='app', types=['configuration'], limit=100, offset=0, branch_scope='current' - ) - # ...and the narrowed page is capped to the user limit, containing only data-apps. - assert len(result.hits) == 2 - assert all(hit.item_type == 'data-app' for hit in result.hits) - - @pytest.mark.asyncio - async def test_search_configuration_row_mapping(self, mocker: MockerFixture, mcp_context_client: Context): - """Row hits resolve their parent configuration from fullPath; unresolvable rows are skipped.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - response = _global_search_response( - _global_search_item( - 'row-1', - 'My Row', - 'configuration-row', - component_id='keboola.ex-db-mysql', - full_path={'configuration': {'id': 'cfg-1', 'name': 'Parent Config'}}, - ), - _global_search_item('row-2', 'Orphan Row', 'configuration-row', component_id='keboola.ex-db-mysql'), - ) - keboola_client.storage_client.global_search = mocker.AsyncMock(return_value=response) - - result = await search(ctx=mcp_context_client, patterns=['row'], item_types=('configuration-row',)) - - assert len(result.hits) == 1 - assert result.hits[0] == SearchHit( - component_id='keboola.ex-db-mysql', - configuration_id='cfg-1', - configuration_row_id='row-1', - item_type='configuration-row', - updated='2024-01-01T00:00:00+00:00', - name='My Row', - links=result.hits[0].links, - ) - - @pytest.mark.asyncio - async def test_search_regex_mode_rejected(self, mocker: MockerFixture, mcp_context_client: Context): - """Regex patterns are not supported by the server-side textual search.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - keboola_client.storage_client.global_search = mocker.AsyncMock() - - with pytest.raises(ToolError, match='Regex patterns are not supported for textual search'): - await search(ctx=mcp_context_client, patterns=['customer.*'], mode='regex') - - keboola_client.storage_client.global_search.assert_not_called() - - -@pytest.mark.parametrize( - ('spec_kwargs', 'texts', 'expected'), - [ - ( - { - 'patterns': ['foo.*', 'fo.*', 'olala'], - 'item_types': ('bucket',), - 'pattern_mode': 'regex', - 'return_all_matched_patterns': True, - }, - ['foo.*', 'foobar', 'olala'], - [ - {'scope': None, 'patterns': ['foo.*', 'fo.*']}, - {'scope': None, 'patterns': ['foo.*', 'fo.*']}, - {'scope': None, 'patterns': ['olala']}, - ], - ), - ( - { - 'patterns': ['foo.*', 'fo.*', 'olala'], - 'item_types': ('bucket',), - 'pattern_mode': 'regex', - 'return_all_matched_patterns': False, - }, - ['foo.*', 'foobar', 'olala'], - [{'scope': None, 'patterns': ['foo.*']}], - ), - ( - { - 'patterns': ['nomatch'], - 'item_types': ('bucket',), - 'return_all_matched_patterns': True, - }, - ['Foo baz', 'BAR qux'], - [], - ), - ( - { - 'patterns': ['bar'], - 'item_types': ('bucket',), - 'pattern_mode': 'literal', - 'case_sensitive': False, - 'return_all_matched_patterns': False, - }, - ['Foo baz', 'BAR qux', 'BARAndSomething'], - [ - {'scope': None, 'patterns': ['bar']}, - ], - ), - ( - { - 'patterns': ['bar'], - 'item_types': ('bucket',), - 'pattern_mode': 'literal', - 'case_sensitive': True, - 'return_all_matched_patterns': False, - }, - ['Foo baz', 'BAR qux', 'BARrAndSomething'], - [], - ), - ], - ids=[ - 'regex_all_matches', - 'regex_any_match', - 'regex_no_match', - 'literal_match_case_insensitive', - 'literal_match_case_sensitive', - ], -) -def test_match_texts(spec_kwargs: dict[str, Any], texts: list[str], expected: list[dict]): - spec = SearchSpec(**spec_kwargs) - matches = spec.match_texts(texts) - assert [match.model_dump() for match in matches] == expected - - -@pytest.mark.parametrize( - ('spec_kwargs', 'configuration', 'expected'), - [ - ( - # Scopes provided; each scope has one matching leaf – returns the exact leaf path. - { - 'patterns': ['alpha', 'beta'], - 'item_types': ('configuration',), - 'search_scopes': ('parameters', 'storage.input'), - 'return_all_matched_patterns': True, - }, - { - 'parameters': {'query': 'alpha'}, - 'storage': {'input': [{'source': 'beta'}], 'output': [{'destination': 'gamma'}]}, - }, - [ - {'scope': 'parameters.query', 'patterns': ['alpha']}, - {'scope': 'storage.input[0].source', 'patterns': ['beta']}, - ], - ), - ( - # Both patterns match across two leaves inside the same scope; each leaf gets its own entry. - { - 'patterns': ['alpha', 'beta'], - 'item_types': ('configuration',), - 'search_scopes': ('parameters', 'storage.input'), - 'return_all_matched_patterns': True, - }, - { - 'parameters': {'query': 'alpha'}, - 'storage': {'input': [{'source': 'beta'}, {'source': 'alpha'}], 'output': [{'destination': 'gamma'}]}, - }, - [ - {'scope': 'parameters.query', 'patterns': ['alpha']}, - {'scope': 'storage.input[0].source', 'patterns': ['beta']}, - {'scope': 'storage.input[1].source', 'patterns': ['alpha']}, - ], - ), - ( - # Pattern not present in any of the specified scopes → empty result. - { - 'patterns': ['gamma'], - 'item_types': ('configuration',), - 'search_scopes': ('parameters', 'storage.input'), - 'return_all_matched_patterns': True, - }, - { - 'parameters': {'query': 'alpha'}, - 'storage': {'input': [{'source': 'beta'}], 'output': [{'destination': 'gamma'}]}, - }, - [], - ), - ( - # No scopes → walk the whole config; can match parent nodes containing the searched fragment. - { - 'patterns': ['gamma'], - 'item_types': ('configuration',), - 'return_all_matched_patterns': True, - }, - { - 'parameters': {'query': 'alpha'}, - 'storage': {'input': [{'source': 'beta'}], 'output': [{'destination': 'gamma'}]}, - }, - [ - {'scope': 'storage', 'patterns': ['gamma']}, - {'scope': 'storage.output', 'patterns': ['gamma']}, - {'scope': 'storage.output[0].destination', 'patterns': ['gamma']}, - ], - ), - ( - # return_all_matched_patterns=False → stop after first matching leaf. - { - 'patterns': ['alpha', 'beta'], - 'item_types': ('configuration',), - 'search_scopes': ('parameters', 'storage.input'), - 'return_all_matched_patterns': False, - }, - { - 'parameters': {'query': 'alpha'}, - 'storage': {'input': [{'source': 'beta'}], 'output': [{'destination': 'gamma'}]}, - }, - [{'scope': 'parameters.query', 'patterns': ['alpha']}], - ), - ( - # Overlapping scopes should not return duplicate leaf hits. - { - 'patterns': ['alpha'], - 'item_types': ('configuration',), - 'search_scopes': ('parameters', 'parameters.query'), - 'return_all_matched_patterns': True, - }, - {'parameters': {'query': 'alpha'}}, - [{'scope': 'parameters.query', 'patterns': ['alpha']}], - ), - ( - # Scope pointing directly to scalar should still match (self-scope fallback). - { - 'patterns': ['wttr.in'], - 'item_types': ('configuration',), - 'search_scopes': ('parameters.api.baseUrl',), - 'return_all_matched_patterns': True, - }, - {'parameters': {'api': {'baseUrl': 'https://wttr.in'}}}, - [{'scope': 'parameters.api.baseUrl', 'patterns': ['wttr.in']}], - ), - ( - # Scope with #-prefixed key should be normalized and parsed correctly. - { - 'patterns': ['alpha'], - 'item_types': ('configuration',), - 'search_scopes': ('authorization.#apiKey',), - 'return_all_matched_patterns': True, - }, - {'authorization': {'#apiKey': 'alpha'}}, - [{'scope': 'authorization.#apiKey', 'patterns': ['alpha']}], - ), - ], - ids=[ - 'all_patterns_many_scopes', - 'two_patterns_in_one_scope', - 'no_patterns_in_scope', - 'all_patterns_no_scope', - 'any_patterns_return_first_match', - 'overlapping_scopes_deduplicated', - 'scalar_scope_matches_self', - 'hash_prefixed_scope_key_matches', - ], -) -def test_match_configuration_scopes(spec_kwargs: dict[str, Any], configuration: dict[str, Any], expected: list[dict]): - spec = SearchSpec(**spec_kwargs) - matches = spec.match_configuration_scopes(configuration) - assert [match.model_dump() for match in matches] == expected - - -@pytest.mark.asyncio -async def test_find_component_id(mocker: MockerFixture, mcp_context_client: Context): - """Test find_component_id returns suggested components.""" - keboola_client = KeboolaClient.from_state(mcp_context_client.session.state) - project_id = await keboola_client.storage_client.project_id() - - # Mock suggest_component to return a list of suggested components - expected_component_1 = SuggestedComponent(component_id='keboola.ex-salesforce', score=0.95, source='ai') - expected_component_2 = SuggestedComponent(component_id='keboola.ex-db-mysql', score=0.85, source='ai') - mock_response = ComponentSuggestionResponse(components=[expected_component_1, expected_component_2]) - keboola_client.ai_service_client.suggest_component = mocker.AsyncMock(return_value=mock_response) - - query = 'I am looking for a salesforce extractor component' - result = await find_component_id(ctx=mcp_context_client, query=query) - - assert isinstance(result, list) - assert result == [ - SuggestedComponentOutput( - component_id='keboola.ex-salesforce', - score=0.95, - links=[ - Link( - type='ui-dashboard', - title='Component "keboola.ex-salesforce" Configurations Dashboard', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/components/keboola.ex-salesforce' - ), - ) - ], - ), - SuggestedComponentOutput( - component_id='keboola.ex-db-mysql', - score=0.85, - links=[ - Link( - type='ui-dashboard', - title='Component "keboola.ex-db-mysql" Configurations Dashboard', - url=( - f'https://connection.test.keboola.com/admin/projects/{project_id}' - '/components/keboola.ex-db-mysql' - ), - ) - ], - ), - ] - keboola_client.ai_service_client.suggest_component.assert_called_once_with(query) diff --git a/tests/tools/test_sql.py b/tests/tools/test_sql.py deleted file mode 100644 index 256fb13e9..000000000 --- a/tests/tools/test_sql.py +++ /dev/null @@ -1,1317 +0,0 @@ -import asyncio -import contextlib -import json -from typing import Any -from unittest.mock import AsyncMock, MagicMock, Mock, call - -import pytest -from mcp.server.fastmcp import Context -from mcp.types import ProgressNotification - -from keboola_mcp_server.clients.client import KeboolaClient -from keboola_mcp_server.clients.query import QueryServiceClient -from keboola_mcp_server.tools.sql import QueryDataOutput, _watch_for_http_disconnect, query_data -from keboola_mcp_server.workspace import ( - JobSubmittedInfo, - QueryResult, - SqlSelectData, - TableFqn, - WorkspaceManager, - _BigQueryWorkspace, - _SnowflakeWorkspace, -) - - -def _truncate_data(qr: QueryResult, max_rows: int | None, max_chars: int | None) -> QueryResult: - rows = [] - total_chars = 0 - for row in qr.data.rows[: (max_rows or len(qr.data.rows))]: - chars = sum(len(str(v)) for v in row.values() if v is not None) - if max_chars is not None and total_chars + chars > max_chars: - break - total_chars += chars - rows.append(row) - - return QueryResult( - status=qr.status, - data=SqlSelectData(columns=qr.data.columns, rows=rows), - message=_SnowflakeWorkspace._SELECTED_ROWS_MSG.format(rows=len(rows), total=len(qr.data.rows)), - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ('query', 'query_name', 'result', 'expected_csv'), - [ - ( - 'select 1;', - 'Simple Count Query', - QueryResult(status='ok', data=SqlSelectData(columns=['a'], rows=[{'a': 1}]), message=None), - 'a\r\n1\r\n', # CSV - ), - ( - 'select id, name, email from user;', - 'User Details List', - QueryResult( - status='ok', - data=SqlSelectData( - columns=['id', 'name', 'email'], - rows=[ - {'id': 1, 'name': 'John', 'email': 'john@foo.com'}, - {'id': 2, 'name': 'Joe', 'email': 'joe@bar.com'}, - ], - ), - message=None, - ), - 'id,name,email\r\n1,John,john@foo.com\r\n2,Joe,joe@bar.com\r\n', # CSV - ), - ( - 'create table foo (id integer, name varchar);', - 'Create Table Operation', - QueryResult(status='ok', data=None, message='1 table created'), - 'message\r\n1 table created\r\n', # CSV - ), - ], -) -async def test_query_data( - query: str, query_name: str, result: QueryResult, expected_csv: str, mcp_context_client: Context, mocker -): - manager = mocker.AsyncMock(WorkspaceManager) - manager.execute_query.return_value = result - mcp_context_client.session.state[WorkspaceManager.STATE_KEY] = manager - - result = await query_data(query, query_name, mcp_context_client) - assert isinstance(result, QueryDataOutput) - assert result.query_name == query_name - assert result.csv_data == expected_csv - - -@pytest.mark.asyncio -async def test_query_data_emits_progress_notification_with_job_id(mcp_context_client: Context, mocker): - """When the client supplied a progressToken in the original tools/call, query_data must surface - the backend job id to the client by sending a `notifications/progress` whose `params._meta` - carries `keboola.queryJobId`, the backend name, and the absolute cancel URL. Without this, - clients cannot cancel a long-running query out-of-band against Query Service directly. - """ - info = JobSubmittedInfo( - job_id='job-xyz', - cancellation_url='https://query.keboola.com/api/v1/queries/job-xyz/cancel', - backend='snowflake', - ) - - async def fake_execute_query(sql_query, *, max_rows, max_chars, on_job_submitted=None): - if on_job_submitted is not None: - await on_job_submitted(info) - return QueryResult(status='ok', data=SqlSelectData(columns=['a'], rows=[{'a': 1}]), message=None) - - manager = mocker.AsyncMock(WorkspaceManager) - manager.execute_query.side_effect = fake_execute_query - mcp_context_client.session.state[WorkspaceManager.STATE_KEY] = manager - - mcp_context_client.request_context.meta = mocker.MagicMock() - mcp_context_client.request_context.meta.progressToken = 'tkn-1' - mcp_context_client.request_context.request_id = 'req-99' - mcp_context_client.session.send_notification = AsyncMock() - - await query_data('select 1;', 'q', mcp_context_client) - - # We bypass FastMCP's ctx.send_notification() because it does not set - # `related_request_id` -- and without that, the streamable_http router (mcp/server/ - # streamable_http.py:~1004) cannot route the notification to the originating - # tools/call SSE stream in stateless_http mode. The notification gets silently - # dropped onto GET_STREAM_KEY. Call the low-level session API with the request id - # so the notification reaches the correct response stream. - mcp_context_client.session.send_notification.assert_awaited_once() - call_args = mcp_context_client.session.send_notification.await_args - sent = call_args.args[0] - assert ( - call_args.kwargs.get('related_request_id') == 'req-99' - ), f'related_request_id missing or wrong: {call_args.kwargs!r}' - # The wrapper is ServerNotification(root=ProgressNotification(...)); both .root and - # the wrapper's model_dump should expose the progress notification shape. - progress = sent.root if hasattr(sent, 'root') else sent - assert isinstance(progress, ProgressNotification) - assert progress.params.progressToken == 'tkn-1' - on_wire = json.loads(progress.model_dump_json(by_alias=True, exclude_none=True)) - assert on_wire['method'] == 'notifications/progress' - assert on_wire['params']['_meta'] == { - 'keboola.queryJobId': 'job-xyz', - 'keboola.backend': 'snowflake', - 'keboola.cancellationUrl': 'https://query.keboola.com/api/v1/queries/job-xyz/cancel', - } - - -@pytest.mark.asyncio -async def test_query_data_skips_progress_when_no_token(mcp_context_client: Context, mocker): - """Per MCP spec, the server must only emit progress notifications when the client provided a - progressToken. Without one we must stay silent — sending unsolicited progress can break clients - that strictly validate the protocol.""" - - async def fake_execute_query(sql_query, *, max_rows, max_chars, on_job_submitted=None): - # The tool should not even hand us a callback when no token is set. - assert on_job_submitted is None - return QueryResult(status='ok', data=SqlSelectData(columns=['a'], rows=[{'a': 1}]), message=None) - - manager = mocker.AsyncMock(WorkspaceManager) - manager.execute_query.side_effect = fake_execute_query - mcp_context_client.session.state[WorkspaceManager.STATE_KEY] = manager - - # empty_context fixture defaults meta to None, which is the "no progressToken" shape. - assert mcp_context_client.request_context.meta is None - mcp_context_client.session.send_notification = AsyncMock() - - await query_data('select 1;', 'q', mcp_context_client) - - mcp_context_client.session.send_notification.assert_not_called() - - -@pytest.mark.asyncio -async def test_query_data_skips_progress_when_request_id_missing(mcp_context_client: Context, mocker, caplog): - """If the request id is unavailable, the streamable_http router has no way to attach the - notification to the originating tools/call SSE stream — it would silently fall through to - GET_STREAM_KEY (which doesn't exist in stateless_http) and be dropped. The notification - emitter must detect this and skip emitting, logging a warning so the failure mode is visible - instead of swallowed. The underlying query still completes normally. - - Note we null out `request_context.request_id` (not the `ctx.request_id` property, which raises - RuntimeError when the context is missing) — that is the field the emitter actually reads. - """ - info = JobSubmittedInfo(job_id='job-no-rid', cancellation_url='https://q/cancel', backend='snowflake') - - async def fake_execute_query(sql_query, *, max_rows, max_chars, on_job_submitted=None): - if on_job_submitted is not None: - await on_job_submitted(info) - return QueryResult(status='ok', data=SqlSelectData(columns=['a'], rows=[{'a': 1}]), message=None) - - manager = mocker.AsyncMock(WorkspaceManager) - manager.execute_query.side_effect = fake_execute_query - mcp_context_client.session.state[WorkspaceManager.STATE_KEY] = manager - - mcp_context_client.request_context.meta = mocker.MagicMock() - mcp_context_client.request_context.meta.progressToken = 'tkn-1' - mcp_context_client.request_context.request_id = None # the case under test - mcp_context_client.session.send_notification = AsyncMock() - - import logging - - with caplog.at_level(logging.WARNING, logger='keboola_mcp_server.tools.sql'): - result = await query_data('select 1;', 'q', mcp_context_client) - - # Query itself completes normally — guarding the notification must not abort the call. - assert isinstance(result, QueryDataOutput) - # No notification was sent (would have been dropped on GET_STREAM_KEY anyway). - mcp_context_client.session.send_notification.assert_not_called() - # The warning must mention the job id so the operator can correlate against Snowflake. - assert any( - 'job-no-rid' in r.getMessage() and 'request id is unavailable' in r.getMessage() for r in caplog.records - ), f'expected warning mentioning job_id and request id; got: {[r.getMessage() for r in caplog.records]}' - - -class TestWorkspaceManagerSnowflake: - - @pytest.fixture - def context(self, keboola_client: KeboolaClient, empty_context: Context, mocker) -> Context: - keboola_client.storage_client.workspace_list.return_value = [ - { - 'id': 1234, - 'connection': { - 'schema': 'workspace_1234', - 'backend': 'snowflake', - 'user': 'user_1234', - }, - 'readOnlyStorageAccess': True, - } - ] - - empty_context.session.state[KeboolaClient.STATE_KEY] = keboola_client - empty_context.session.state[WorkspaceManager.STATE_KEY] = WorkspaceManager( - client=keboola_client, workspace_schema='workspace_1234' - ) - - return empty_context - - @pytest.mark.asyncio - async def test_get_sql_dialect(self, context: Context): - m = WorkspaceManager.from_state(context.session.state) - assert await m.get_sql_dialect() == 'Snowflake' - - @pytest.mark.asyncio - async def test_get_quoted_name(self, context: Context): - m = WorkspaceManager.from_state(context.session.state) - assert await m.get_quoted_name('foo') == '"foo"' - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('table', 'expected'), - [ - ( - # table in.c-foo.bar in its own project — db_name from backendPath[0] - { - 'id': 'in.c-foo.bar', - 'name': 'bar', - 'bucket': {'id': 'in.c-foo', 'backendPath': ['db_xyz', 'in.c-foo']}, - }, - TableFqn(db_name='db_xyz', schema_name='in.c-foo', table_name='bar', quote_char='"'), - ), - ( - # linked (non-alias) table shared from project 153: Storage propagates the linked - # bucket's backendPath onto the table itself, so db_name/schema come from the table's - # own bucket.backendPath and the table name from the linked table. - { - 'id': 'in.c-acc.ccc', - 'name': 'ccc', - 'isAlias': True, - 'bucket': {'id': 'in.c-acc', 'backendPath': ['KBC_EUW3_153', 'out.c-acc']}, - 'sourceTable': {'id': 'out.c-acc.ccc', 'project': {'id': 153}, 'isAlias': False}, - }, - TableFqn(db_name='KBC_EUW3_153', schema_name='out.c-acc', table_name='ccc', quote_char='"'), - ), - ( - # storage-branches: backendPath present → db_name from backendPath[0] - { - 'id': 'out.c-model.customers', - 'name': 'customers', - 'bucket': {'id': 'out.c-model', 'backendPath': ['KBC_USE4_3047', '35403_out.c-model']}, - }, - TableFqn( - db_name='KBC_USE4_3047', schema_name='35403_out.c-model', table_name='customers', quote_char='"' - ), - ), - ( - # production bucket — db_name from backendPath[0] - { - 'id': 'in.c-shopify.orders', - 'name': 'orders', - 'bucket': {'id': 'in.c-shopify', 'backendPath': ['KBC_USE4_3047', 'in.c-shopify']}, - }, - TableFqn(db_name='KBC_USE4_3047', schema_name='in.c-shopify', table_name='orders', quote_char='"'), - ), - ( - # materialized alias from a linked bucket: the source table is an alias in the source - # project (sourceTable.isAlias=True), but with materialized aliases enabled it is - # physically present in the shared database. The table's own bucket.backendPath resolves - # to that shared db+schema, so the FQN is queryable — db_name/schema from backendPath, - # table name from the linked table itself. - { - 'id': 'in.c-acc.sample_customers_alias', - 'name': 'sample_customers_alias', - 'isAlias': True, - 'bucket': {'id': 'in.c-acc', 'backendPath': ['KBC_EUW3_153', 'out.c-acc']}, - 'sourceTable': { - 'id': 'out.c-acc.sample_customers_alias', - 'project': {'id': 153}, - 'isAlias': True, - }, - }, - TableFqn( - db_name='KBC_EUW3_153', - schema_name='out.c-acc', - table_name='sample_customers_alias', - quote_char='"', - ), - ), - ], - ) - async def test_get_table_fqn( - self, - table: dict[str, Any], - expected: TableFqn, - keboola_client: KeboolaClient, - context: Context, - mocker, - ): - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - mocker.patch.object(QueryServiceClient, 'create', side_effect=AssertionError('no SQL should be issued')) - - m = WorkspaceManager.from_state(context.session.state) - info = await m.get_table_info(table) - assert info is not None - assert info.fqn == expected - - @pytest.mark.asyncio - @pytest.mark.parametrize( - 'table', - [ - # no backendPath — returns None without any SQL - {'id': 'in.c-foo.bar', 'name': 'bar'}, - # linked alias table without its own bucket.backendPath — not reachable, no FQN - { - 'id': 'in.c-foo.bar', - 'name': 'bar', - 'sourceTable': {'project': {'id': '1234'}, 'id': 'out.c-baz.bam', 'isAlias': True}, - }, - # linked non-alias table without its own bucket.backendPath — not reachable, no FQN - { - 'id': 'in.c-foo.bar', - 'name': 'bar', - 'sourceTable': {'project': {'id': '1234'}, 'id': 'out.c-baz.bam', 'isAlias': False}, - }, - ], - ) - async def test_get_table_info_returns_none( - self, - table: dict[str, Any], - keboola_client: KeboolaClient, - context: Context, - mocker, - ): - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - mocker.patch.object(QueryServiceClient, 'create', side_effect=AssertionError('no SQL should be issued')) - - m = WorkspaceManager.from_state(context.session.state) - info = await m.get_table_info(table) - assert info is None - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('query', 'db_data', 'max_rows', 'max_chars'), - [ - ( - 'select id, name, email from user;', - QueryResult( - status='ok', - data=SqlSelectData( - columns=['id', 'name', 'email'], - rows=[ - {'id': 1, 'name': 'John', 'email': 'john@foo.com'}, - {'id': 2, 'name': 'Joe', 'email': 'joe@bar.com'}, - ], - ), - ), - None, - None, - ), - ( - 'select id, name, email from user;', - QueryResult( - status='ok', - data=SqlSelectData( - columns=['id', 'name', 'email'], - rows=[ - {'id': 1, 'name': 'John', 'email': 'john@foo.com'}, - {'id': 2, 'name': 'Joe', 'email': 'joe@bar.com'}, - ], - ), - ), - 1, - None, - ), - ( - 'select id, name, email from user;', - QueryResult( - status='ok', - data=SqlSelectData( - columns=['id', 'name', 'email'], - rows=[ - {'id': 1, 'name': 'John', 'email': 'john@foo.com'}, # 17 characters - {'id': 2, 'name': 'Joe', 'email': 'joe@bar.com'}, # 16 characters - ], - ), - ), - None, - 20, - ), - ( - 'create table foo (id integer, name varchar);', - QueryResult(status='ok', message='1 table created'), - None, - None, - ), - ('bla bla bla', QueryResult(status='error', message='Invalid SQL...'), None, None), - ], - ) - async def test_execute_query( - self, - query: str, - db_data: QueryResult, - max_rows: int | None, - max_chars: int | None, - keboola_client: KeboolaClient, - context: Context, - mocker, - ): - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-1234' - qsclient.get_job_status.return_value = { - 'status': 'completed', - 'statements': [{'id': 'qs-job-statement-1234', 'status': 'completed'}], - } - qsclient.get_job_results.return_value = { - 'status': 'completed' if db_data.is_ok else 'failed', - 'data': [list(row.values()) for row in db_data.data.rows] if db_data.data else [], - 'columns': [{'name': col_name} for col_name in db_data.data.columns] if db_data.data else [], - 'message': db_data.message, - 'numberOfRows': len(db_data.data.rows) if db_data.data else None, - } - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - - if db_data.data is not None: - expected = _truncate_data(db_data, max_rows, max_chars) - else: - expected = db_data - - m = WorkspaceManager.from_state(context.session.state) - actual = await m.execute_query(query, max_rows=max_rows, max_chars=max_chars) - - assert actual == expected - - keboola_client.storage_client.branches_list.assert_called_once() - qsclient.submit_job.assert_called_once() - qsclient.get_job_status.assert_called_once_with('qs-job-1234') - qsclient.get_job_results.assert_called_once_with( - 'qs-job-1234', 'qs-job-statement-1234', offset=0, limit=1000 if max_rows is None else max(max_rows, 100) - ) - - @pytest.mark.asyncio - async def test_execute_query_pagination(self, keboola_client: KeboolaClient, context: Context, mocker): - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-1234' - qsclient.get_job_status.return_value = { - 'status': 'completed', - 'statements': [{'id': 'qs-job-statement-1234', 'status': 'completed'}], - } - qsclient.get_job_results.side_effect = [ - { - 'status': 'completed', - 'data': [ - [1, 'John', 'john@foo.com'], - [2, 'Joe', 'joe@foo.com'], - [3, 'Jack', 'jack@foo.com'], - [4, 'Jerry', 'jerry@foo.com'], - ], - 'columns': [{'name': 'id'}, {'name': 'name'}, {'name': 'email'}], - 'message': None, - 'numberOfRows': 10, - }, - { - 'status': 'completed', - 'data': [ - [5, 'James', 'james@foo.com'], - [6, 'Julian', 'julian@foo.com'], - [7, 'Jordan', 'jordan@foo.com'], - [8, 'Jacob', 'jacob@foo.com'], - ], - 'columns': [{'name': 'id'}, {'name': 'name'}, {'name': 'email'}], - 'message': None, - 'numberOfRows': 10, - }, - { - 'status': 'completed', - 'data': [ - [9, 'Jagger', 'jagger@foo.com'], - [10, 'Jackson', 'jackson@foo.com'], - ], - 'columns': [{'name': 'id'}, {'name': 'name'}, {'name': 'email'}], - 'message': None, - 'numberOfRows': 10, - }, - ] - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - mocker.patch.object(_SnowflakeWorkspace, '_PAGE_SIZE', 4) - - m = WorkspaceManager.from_state(context.session.state) - actual = await m.execute_query('select id, name, email from user;') - assert actual == QueryResult( - status='ok', - data=SqlSelectData( - columns=['id', 'name', 'email'], - rows=[ - {'id': 1, 'name': 'John', 'email': 'john@foo.com'}, - {'id': 2, 'name': 'Joe', 'email': 'joe@foo.com'}, - {'id': 3, 'name': 'Jack', 'email': 'jack@foo.com'}, - {'id': 4, 'name': 'Jerry', 'email': 'jerry@foo.com'}, - {'id': 5, 'name': 'James', 'email': 'james@foo.com'}, - {'id': 6, 'name': 'Julian', 'email': 'julian@foo.com'}, - {'id': 7, 'name': 'Jordan', 'email': 'jordan@foo.com'}, - {'id': 8, 'name': 'Jacob', 'email': 'jacob@foo.com'}, - {'id': 9, 'name': 'Jagger', 'email': 'jagger@foo.com'}, - {'id': 10, 'name': 'Jackson', 'email': 'jackson@foo.com'}, - ], - ), - message='Returning 10 of 10 selected rows.', - ) - - keboola_client.storage_client.branches_list.assert_called_once() - qsclient.submit_job.assert_called_once() - qsclient.get_job_status.assert_called_once_with('qs-job-1234') - qsclient.get_job_results.assert_has_calls( - [ - call('qs-job-1234', 'qs-job-statement-1234', offset=0, limit=100), - call('qs-job-1234', 'qs-job-statement-1234', offset=4, limit=100), - call('qs-job-1234', 'qs-job-statement-1234', offset=8, limit=100), - ] - ) - - @pytest.mark.asyncio - async def test_execute_query_max_chars_stops_on_first_rejection( - self, keboola_client: KeboolaClient, context: Context, mocker - ): - """ - max_chars must yield a contiguous prefix across pages: once a row would exceed the - budget, pagination stops — later (smaller) rows that happen to fit must not sneak in. - """ - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-1234' - qsclient.get_job_status.return_value = { - 'status': 'completed', - 'statements': [{'id': 'qs-job-statement-1234', 'status': 'completed'}], - } - # Page 1: John (17 chars) fits, Joe (16 chars) does not (17+16=33 > 20) → pagination - # must stop here. The page 2 row (5 chars) would fit individually, but appending it - # would break the contiguous-prefix semantic and must not happen. - qsclient.get_job_results.side_effect = [ - { - 'status': 'completed', - 'data': [[1, 'John', 'john@foo.com'], [2, 'Joe', 'joe@bar.com']], - 'columns': [{'name': 'id'}, {'name': 'name'}, {'name': 'email'}], - 'message': None, - 'numberOfRows': 3, - }, - { - 'status': 'completed', - 'data': [[3, 'X', 'x@y']], - 'columns': [{'name': 'id'}, {'name': 'name'}, {'name': 'email'}], - 'message': None, - 'numberOfRows': 3, - }, - ] - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - mocker.patch.object(_SnowflakeWorkspace, '_PAGE_SIZE', 2) - - m = WorkspaceManager.from_state(context.session.state) - actual = await m.execute_query('select id, name, email from user;', max_chars=20) - - assert actual.data is not None - assert actual.data.rows == [{'id': 1, 'name': 'John', 'email': 'john@foo.com'}] - # Page 2 must not be fetched once page 1 hit the char budget. - qsclient.get_job_results.assert_called_once() - - -class TestWorkspaceManagerBigQuery: - @pytest.fixture - def context(self, keboola_client: KeboolaClient, empty_context: Context, mocker) -> Context: - keboola_client.storage_client.workspace_list.return_value = [ - { - 'id': 1234, - 'connection': { - 'schema': 'workspace_1234', - 'backend': 'bigquery', - 'user': json.dumps({'project_id': 'project_1234'}), - }, - 'readOnlyStorageAccess': True, - } - ] - - empty_context.session.state[KeboolaClient.STATE_KEY] = keboola_client - empty_context.session.state[WorkspaceManager.STATE_KEY] = WorkspaceManager( - client=keboola_client, workspace_schema='workspace_1234' - ) - - return empty_context - - @pytest.mark.asyncio - async def test_get_sql_dialect(self, context: Context): - m = WorkspaceManager.from_state(context.session.state) - assert await m.get_sql_dialect() == 'BigQuery' - - @pytest.mark.asyncio - async def test_get_quoted_name(self, context: Context): - m = WorkspaceManager.from_state(context.session.state) - assert await m.get_quoted_name('foo') == '`foo`' - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('table', 'expected', 'expected_identifier'), - [ - ( - # storage-branches: backendPath with branch-prefixed dataset name (1 element in BQ) - { - 'id': 'out.c-model.customers', - 'name': 'customers', - 'bucket': {'id': 'out.c-model', 'backendPath': ['35403_out_c_model']}, - }, - TableFqn( - db_name='', - schema_name='35403_out_c_model', - table_name='customers', - quote_char='`', - ), - '`35403_out_c_model`.`customers`', - ), - ( - # production bucket — backendPath is single dataset name - { - 'id': 'in.c-shopify.orders', - 'name': 'orders', - 'bucket': {'id': 'in.c-shopify', 'backendPath': ['in_c_shopify']}, - }, - TableFqn( - db_name='', - schema_name='in_c_shopify', - table_name='orders', - quote_char='`', - ), - '`in_c_shopify`.`orders`', - ), - ( - # linked (non-alias) bucket — data copied to destination dataset, queryable via backendPath FQN - { - 'id': 'in.c-abc.customers', - 'name': 'customers', - 'bucket': {'id': 'in.c-abc', 'backendPath': ['in_c_abc']}, - 'sourceTable': {'project': {'id': '9999'}, 'id': 'in.c-abc.customers', 'isAlias': False}, - }, - TableFqn( - db_name='', - schema_name='in_c_abc', - table_name='customers', - quote_char='`', - ), - '`in_c_abc`.`customers`', - ), - ], - ) - async def test_get_table_fqn( - self, - table: dict[str, Any], - expected: TableFqn, - expected_identifier: str, - keboola_client: KeboolaClient, - context: Context, - mocker, - ): - mocker.patch.object(QueryServiceClient, 'create', side_effect=AssertionError('no SQL should be issued')) - - m = WorkspaceManager.from_state(context.session.state) - info = await m.get_table_info(table) - assert info is not None - assert info.fqn == expected - # BigQuery FQN has no project/database tier — just dataset.table. - assert info.fqn.identifier == expected_identifier - - @pytest.mark.asyncio - @pytest.mark.parametrize( - 'table', - [ - # no backendPath — returns None without any SQL - {'id': 'in.c-foo.bar', 'name': 'bar'}, - # alias linked table (sourceTable.isAlias=True) — BigQuery has no cross-project sharing and - # does not materialize aliases, so it is never queryable even when a backendPath is present - { - 'id': 'in.c-foo.bar', - 'name': 'bar', - 'bucket': {'id': 'in.c-foo', 'backendPath': ['in_c_foo']}, - 'sourceTable': {'project': {'id': '1234'}, 'id': 'out.c-baz.bam', 'isAlias': True}, - }, - ], - ) - async def test_get_table_info_returns_none( - self, table: dict[str, Any], keboola_client: KeboolaClient, context: Context, mocker - ): - mocker.patch.object(QueryServiceClient, 'create', side_effect=AssertionError('no SQL should be issued')) - - m = WorkspaceManager.from_state(context.session.state) - info = await m.get_table_info(table) - assert info is None - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('query', 'db_data', 'max_rows', 'max_chars'), - [ - ( - 'select id, name, email from user;', - QueryResult( - status='ok', - data=SqlSelectData( - columns=['id', 'name', 'email'], - rows=[ - {'id': 1, 'name': 'John', 'email': 'john@foo.com'}, - {'id': 2, 'name': 'Joe', 'email': 'joe@bar.com'}, - ], - ), - ), - None, - None, - ), - ( - 'select id, name, email from user;', - QueryResult( - status='ok', - data=SqlSelectData( - columns=['id', 'name', 'email'], - rows=[ - {'id': 1, 'name': 'John', 'email': 'john@foo.com'}, - {'id': 2, 'name': 'Joe', 'email': 'joe@bar.com'}, - ], - ), - ), - 1, - None, - ), - ( - 'select id, name, email from user;', - QueryResult( - status='ok', - data=SqlSelectData( - columns=['id', 'name', 'email'], - rows=[ - {'id': 1, 'name': 'John', 'email': 'john@foo.com'}, # 17 characters - {'id': 2, 'name': 'Joe', 'email': 'joe@bar.com'}, # 16 characters - ], - ), - ), - None, - 20, - ), - ( - 'CREATE TABLE `foo` (id INT64, name STRING);', - QueryResult(status='ok', data=None, message='1 table created'), - None, - None, - ), - ('bla bla bla', QueryResult(status='error', data=None, message='400 Invalid SQL...'), None, None), - ], - ) - async def test_execute_query( - self, - query: str, - db_data: QueryResult, - max_rows: int | None, - max_chars: int | None, - keboola_client: KeboolaClient, - context: Context, - mocker, - ): - # BigQuery now runs queries through the backend-agnostic Query Service, just like Snowflake. - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-1234' - qsclient.get_job_status.return_value = { - 'status': 'completed', - 'statements': [{'id': 'qs-job-statement-1234', 'status': 'completed'}], - } - qsclient.get_job_results.return_value = { - 'status': 'completed' if db_data.is_ok else 'failed', - 'data': [list(row.values()) for row in db_data.data.rows] if db_data.data else [], - 'columns': [{'name': col_name} for col_name in db_data.data.columns] if db_data.data else [], - 'message': db_data.message, - 'numberOfRows': len(db_data.data.rows) if db_data.data else None, - } - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - - if db_data.data is not None: - expected = _truncate_data(db_data, max_rows, max_chars) - else: - expected = db_data - - m = WorkspaceManager.from_state(context.session.state) - actual = await m.execute_query(query, max_rows=max_rows, max_chars=max_chars) - - assert actual == expected - - keboola_client.storage_client.branches_list.assert_called_once() - qsclient.submit_job.assert_called_once() - qsclient.get_job_status.assert_called_once_with('qs-job-1234') - qsclient.get_job_results.assert_called_once_with( - 'qs-job-1234', 'qs-job-statement-1234', offset=0, limit=1000 if max_rows is None else max(max_rows, 100) - ) - - @pytest.mark.parametrize( - ('raw_message', 'expected'), - [ - # Query Service wraps BigQuery errors as a serialized error object; we extract `Message`. - ( - '{Location: "query"; Message: "Syntax error: Unexpected identifier \\"INVALID\\" at [1:1]"; ' - 'Reason: "invalidQuery"}', - 'Syntax error: Unexpected identifier "INVALID" at [1:1]', - ), - ( - '{Location: ""; Message: "Access Denied: Table foo: User does not have permission to query ' - 'table foo, or perhaps it does not exist."; Reason: "accessDenied"}', - 'Access Denied: Table foo: User does not have permission to query table foo, ' - 'or perhaps it does not exist.', - ), - # A plain message (no wrapper) is passed through unchanged. - ('400 Invalid SQL...', '400 Invalid SQL...'), - (None, None), - ], - ids=['syntax_error', 'access_denied', 'plain_message', 'none'], - ) - def test_format_error_message_unwraps_bigquery_error(self, raw_message: str | None, expected: str | None): - workspace = _BigQueryWorkspace( - workspace_id=1234, dataset_id='workspace_1234', project_id='project_1234', client=Mock(spec=KeboolaClient) - ) - assert workspace._format_error_message(raw_message) == expected - - -class TestQueryCancellation: - """Tests for query cancellation on timeout.""" - - @pytest.fixture - def snowflake_context(self, keboola_client: KeboolaClient, empty_context: Context) -> Context: - """Context with Snowflake workspace.""" - keboola_client.storage_client.workspace_list.return_value = [ - { - 'id': 1234, - 'connection': { - 'schema': 'workspace_1234', - 'backend': 'snowflake', - 'user': 'user_1234', - }, - 'readOnlyStorageAccess': True, - } - ] - empty_context.session.state[KeboolaClient.STATE_KEY] = keboola_client - empty_context.session.state[WorkspaceManager.STATE_KEY] = WorkspaceManager( - client=keboola_client, workspace_schema='workspace_1234' - ) - return empty_context - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('cancel_succeeds', 'final_cancel_status', 'expected_in_error'), - [ - # Successful cancellation - (True, 'canceled', 'has been cancelled'), - # Cancellation called but query still processing (timeout during polling) - (True, 'processing', 'could not be confirmed'), - # Cancellation API fails - (False, None, 'could not be confirmed'), - ], - ids=['cancel_success', 'cancel_timeout', 'cancel_fails'], - ) - async def test_timeout_triggers_cancellation( - self, - snowflake_context: Context, - keboola_client: KeboolaClient, - mocker, - cancel_succeeds: bool, - final_cancel_status: str | None, - expected_in_error: str, - ) -> None: - """Test that query timeout triggers cancellation with various outcomes.""" - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-timeout-123' - - # Simulate timeout by always returning 'processing' - status_responses = [ - {'status': 'processing', 'statements': [{'id': 'stmt-1'}]} - ] * 5 # Just enough for main query loop before timeout - - if cancel_succeeds: - qsclient.cancel_job.return_value = {} - # After cancel, change status - if final_cancel_status: - status_responses.append({'status': final_cancel_status}) - else: - qsclient.cancel_job.side_effect = Exception('Cancel API failed') - - qsclient.get_job_status.side_effect = status_responses - - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - mocker.patch.object(_SnowflakeWorkspace, '_QUERY_TIMEOUT', 2.0) # 2 second timeout for test - - m = WorkspaceManager.from_state(snowflake_context.session.state) - - with pytest.raises(RuntimeError, match=expected_in_error): - await m.execute_query('SELECT * FROM slow_table') - - # Verify cancellation was attempted - qsclient.cancel_job.assert_called_once() - call_args = qsclient.cancel_job.call_args - assert call_args.args[0] == 'qs-job-timeout-123' # job_id is first positional arg - assert 'timeout' in call_args.kwargs['reason'].lower() - - @pytest.mark.asyncio - async def test_cancel_job_method(self, mocker) -> None: - """Test QueryServiceClient.cancel_job() sends correct request.""" - from keboola_mcp_server.clients.base import RawKeboolaClient - - raw_client = mocker.AsyncMock(RawKeboolaClient) - raw_client.post.return_value = {'status': 'canceling'} - - qsclient = QueryServiceClient(raw_client=raw_client, branch_id='1234') - - result = await qsclient.cancel_job('job-abc-123', reason='Test cancellation') - - raw_client.post.assert_called_once_with( - endpoint='queries/job-abc-123/cancel', data={'reason': 'Test cancellation'}, params=None, timeout=None - ) - assert result == {'status': 'canceling'} - - @pytest.mark.asyncio - async def test_cancellation_polling_multiple_checks( - self, snowflake_context: Context, keboola_client: KeboolaClient, mocker - ) -> None: - """Verify cancellation polling makes multiple status checks until terminal state.""" - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-polling-123' - - # Simulate timeout by always returning 'processing' - status_responses = [{'status': 'processing', 'statements': [{'id': 'stmt-1'}]}] * 5 - - # After timeout, simulate cancellation polling: 'canceling' 3 times, then 'canceled' - cancel_status_responses = [ - {'status': 'canceling'}, - {'status': 'canceling'}, - {'status': 'canceling'}, - {'status': 'canceled'}, - ] - - qsclient.cancel_job.return_value = {} - qsclient.get_job_status.side_effect = status_responses + cancel_status_responses - - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - mocker.patch.object(_SnowflakeWorkspace, '_QUERY_TIMEOUT', 2.0) # 2 second timeout for test - - m = WorkspaceManager.from_state(snowflake_context.session.state) - - with pytest.raises(RuntimeError, match='has been cancelled'): - await m.execute_query('SELECT * FROM slow_table') - - # Verify cancellation was attempted - qsclient.cancel_job.assert_called_once() - - # Verify multiple status checks during cancellation polling - # We verify proper polling occurred by checking the total call count - # (status_responses during query + cancel_status_responses during cancellation) - assert qsclient.get_job_status.call_count == len(status_responses) + len(cancel_status_responses) - - @pytest.mark.asyncio - async def test_cancellation_polling_timeout( - self, snowflake_context: Context, keboola_client: KeboolaClient, mocker - ) -> None: - """Verify cancellation polling stops after 30s if job stays in 'canceling' state.""" - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-polling-timeout-123' - - # Simulate timeout by always returning 'processing' - status_responses = [{'status': 'processing', 'statements': [{'id': 'stmt-1'}]}] * 5 - - # After cancel, return 'canceling' indefinitely - cancel_status_responses = [{'status': 'canceling'}] * 1000 - - qsclient.cancel_job.return_value = {} - qsclient.get_job_status.side_effect = status_responses + cancel_status_responses - - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - mocker.patch.object(_SnowflakeWorkspace, '_QUERY_TIMEOUT', 2.0) # 2 second timeout for test - mocker.patch.object(_SnowflakeWorkspace, '_CANCELLATION_TIMEOUT', 1.0) # 1 second cancellation timeout - - m = WorkspaceManager.from_state(snowflake_context.session.state) - - with pytest.raises(RuntimeError, match='could not be confirmed'): - await m.execute_query('SELECT * FROM slow_table') - - # Verify cancellation was attempted - qsclient.cancel_job.assert_called_once() - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ('exception_type', 'exception_msg'), - [ - (ConnectionError, 'Network connection lost'), - (TimeoutError, 'API request timed out'), - (Exception, 'Unexpected API error'), - ], - ids=['connection_error', 'timeout_error', 'generic_error'], - ) - async def test_cancellation_polling_network_failure( - self, - snowflake_context: Context, - keboola_client: KeboolaClient, - mocker, - exception_type: type[Exception], - exception_msg: str, - ) -> None: - """Verify network failures during cancellation polling are handled gracefully.""" - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-network-fail-123' - - # Simulate timeout by always returning 'processing' - status_responses = [{'status': 'processing', 'statements': [{'id': 'stmt-1'}]}] * 5 - - qsclient.cancel_job.return_value = {} - # After cancel, get_job_status raises exception - qsclient.get_job_status.side_effect = status_responses + [exception_type(exception_msg)] - - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - mocker.patch.object(_SnowflakeWorkspace, '_QUERY_TIMEOUT', 2.0) # 2 second timeout for test - - m = WorkspaceManager.from_state(snowflake_context.session.state) - - with pytest.raises(RuntimeError, match='could not be confirmed'): - await m.execute_query('SELECT * FROM slow_table') - - # Verify cancellation was attempted - qsclient.cancel_job.assert_called_once() - - @pytest.mark.asyncio - async def test_query_completes_during_cancellation( - self, snowflake_context: Context, keboola_client: KeboolaClient, mocker - ) -> None: - """Test edge case where query completes successfully during cancellation polling.""" - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-completes-123' - - # Simulate timeout by always returning 'processing' - status_responses = [{'status': 'processing', 'statements': [{'id': 'stmt-1'}]}] * 5 - - # After cancel, status becomes 'completed', then one more completed (for break from loop) - cancel_status_responses = [ - {'status': 'completed', 'statements': [{'id': 'stmt-1'}]}, - {'status': 'completed', 'statements': [{'id': 'stmt-1'}]}, - ] - - qsclient.cancel_job.return_value = {} - qsclient.get_job_status.side_effect = status_responses + cancel_status_responses - qsclient.get_job_results.return_value = { - 'status': 'completed', - 'message': 'Query completed during cancellation', - 'columns': [{'name': 'col1'}], - 'data': [['value1']], - } - - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - mocker.patch.object(_SnowflakeWorkspace, '_QUERY_TIMEOUT', 2.0) # 2 second timeout for test - - m = WorkspaceManager.from_state(snowflake_context.session.state) - - # Should return results successfully (query completed, even though it exceeded timeout) - result = await m.execute_query('SELECT * FROM slow_table') - - # Verify query completed successfully - assert result.is_ok - assert result.data is not None - assert result.data.columns == ['col1'] - - # Verify cancellation was attempted - qsclient.cancel_job.assert_called_once() - - @pytest.mark.asyncio - @pytest.mark.parametrize( - 'terminal_status', - ['failed', 'canceled', 'cancelled'], - ids=['status_failed', 'status_canceled', 'status_cancelled'], - ) - async def test_cancellation_polling_terminal_statuses( - self, - snowflake_context: Context, - keboola_client: KeboolaClient, - mocker, - terminal_status: str, - ) -> None: - """Verify cancellation polling recognizes all terminal status types.""" - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-terminal-123' - - # Simulate timeout by always returning 'processing' - status_responses = [{'status': 'processing', 'statements': [{'id': 'stmt-1'}]}] * 5 - - # After cancel, return terminal_status - cancel_status_responses = [{'status': terminal_status}] - - qsclient.cancel_job.return_value = {} - qsclient.get_job_status.side_effect = status_responses + cancel_status_responses - - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - mocker.patch.object(_SnowflakeWorkspace, '_QUERY_TIMEOUT', 2.0) # 2 second timeout for test - - m = WorkspaceManager.from_state(snowflake_context.session.state) - - with pytest.raises(RuntimeError, match='has been cancelled'): - await m.execute_query('SELECT * FROM slow_table') - - # Verify cancellation was attempted - qsclient.cancel_job.assert_called_once() - - @pytest.mark.asyncio - async def test_query_data_cancels_on_http_disconnect(self, mcp_context_client: Context, mocker) -> None: - """When the underlying HTTP request disconnects mid-flight, `query_data` must - cancel the workspace task so its CancelledError branch can fire `cancel_job`.""" - - # Workspace task that never completes — simulates a long-running query. - async def never_returns(*_a, **_kw): - await asyncio.Event().wait() - - manager = AsyncMock(WorkspaceManager) - manager.execute_query.side_effect = never_returns - mcp_context_client.session.state[WorkspaceManager.STATE_KEY] = manager - - # Fake HTTP request: not disconnected for the first poll, then disconnected. - fake_request = MagicMock() - disconnect_states = iter([False, True]) - fake_request.is_disconnected = AsyncMock(side_effect=lambda: next(disconnect_states)) - mocker.patch('keboola_mcp_server.tools.sql.get_http_request_or_none', return_value=fake_request) - # Speed the poll up so the test doesn't have to wait a full second. - mocker.patch('keboola_mcp_server.tools.sql._DISCONNECT_POLL_INTERVAL', 0.01) - - # A disconnect surfaces as a plain ValueError (not CancelledError) so it stays on the - # `@tool_errors` path — logged as an error, not a success, and a response reaches the client. - with pytest.raises(ValueError, match='Query was cancelled'): - await query_data('SELECT 1', 'test', mcp_context_client) - - manager.execute_query.assert_called_once() - - @pytest.mark.asyncio - async def test_query_data_logs_when_disconnect_watcher_raises( - self, mcp_context_client: Context, mocker, caplog - ) -> None: - """If the disconnect watcher finishes with an exception (rather than returning on a - detected disconnect), `query_data` must still cancel the query and surface the watcher - error in the logs — never leak a "Task exception was never retrieved" warning.""" - - async def never_returns(*_a, **_kw): - await asyncio.Event().wait() - - manager = AsyncMock(WorkspaceManager) - manager.execute_query.side_effect = never_returns - mcp_context_client.session.state[WorkspaceManager.STATE_KEY] = manager - - async def boom(*_a, **_kw): - raise RuntimeError('watcher blew up') - - # An HTTP request must be bound for the watcher race to run at all. - mocker.patch('keboola_mcp_server.tools.sql.get_http_request_or_none', return_value=MagicMock()) - mocker.patch('keboola_mcp_server.tools.sql._watch_for_http_disconnect', side_effect=boom) - - with caplog.at_level('WARNING'), pytest.raises(ValueError, match='Query was cancelled'): - await query_data('SELECT 1', 'test', mcp_context_client) - - manager.execute_query.assert_called_once() - assert any('disconnect watcher' in r.message for r in caplog.records) - - @pytest.mark.asyncio - async def test_query_data_propagates_cancel_during_post_query_drain( - self, mcp_context_client: Context, mocker - ) -> None: - """If `query_data` is cancelled while draining the disconnect watcher after the query - completed, the `CancelledError` must propagate (not be swallowed by `_cancel_and_drain` - and cause a result to be returned), while the shielded drain still runs to completion.""" - manager = AsyncMock(WorkspaceManager) - manager.execute_query.return_value = QueryResult( - status='ok', - data=SqlSelectData(columns=['a'], rows=[{'a': 1}]), - message=None, - ) - mcp_context_client.session.state[WorkspaceManager.STATE_KEY] = manager - # HTTP mode with a request that never disconnects: the query wins the race, leaving the - # disconnect watcher as the pending task drained once the query completes. - fake_request = MagicMock() - fake_request.is_disconnected = AsyncMock(return_value=False) - mocker.patch('keboola_mcp_server.tools.sql.get_http_request_or_none', return_value=fake_request) - - drain_started = asyncio.Event() - drain_finished = asyncio.Event() - release = asyncio.Event() - - async def slow_drain(task: asyncio.Task) -> None: - task.cancel() - drain_started.set() - await release.wait() - # Mirror the real `_cancel_and_drain`: actually await the cancelled task so it does - # not linger as a pending task and emit "Task exception was never retrieved" warnings. - with contextlib.suppress(asyncio.CancelledError): - await task - drain_finished.set() - - mocker.patch('keboola_mcp_server.tools.sql._cancel_and_drain', side_effect=slow_drain) - - task = asyncio.create_task(query_data('SELECT 1', 'test', mcp_context_client)) - # Wait until query_data reaches the shielded post-query drain, then cancel it there. - await asyncio.wait_for(drain_started.wait(), timeout=1.0) - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - # The shield kept the drain alive: let it complete and confirm it finished despite the - # outer cancellation that already propagated out of query_data. - release.set() - await asyncio.wait_for(drain_finished.wait(), timeout=1.0) - - @pytest.mark.asyncio - async def test_query_data_no_disconnect_watcher_in_stdio_mode(self, mcp_context_client: Context, mocker) -> None: - """When there is no HTTP request bound (stdio transport), the disconnect race is skipped - entirely and the query runs directly to completion.""" - manager = AsyncMock(WorkspaceManager) - manager.execute_query.return_value = QueryResult( - status='ok', - data=SqlSelectData(columns=['a'], rows=[{'a': 1}]), - message=None, - ) - mcp_context_client.session.state[WorkspaceManager.STATE_KEY] = manager - - mocker.patch('keboola_mcp_server.tools.sql.get_http_request_or_none', return_value=None) - - result = await query_data('SELECT 1', 'test', mcp_context_client) - assert isinstance(result, QueryDataOutput) - assert result.csv_data == 'a\r\n1\r\n' - - @pytest.mark.asyncio - async def test_watch_for_http_disconnect_treats_errors_as_connected(self, mocker) -> None: - """A transient is_disconnected() failure must not be treated as a disconnect.""" - fake_request = MagicMock() - # First call errors (still treated as connected); second call signals disconnect. - fake_request.is_disconnected = AsyncMock(side_effect=[RuntimeError('asgi hiccup'), True]) - - await asyncio.wait_for(_watch_for_http_disconnect(fake_request, poll_interval=0.01), timeout=1.0) - assert fake_request.is_disconnected.await_count == 2 - - @pytest.mark.asyncio - async def test_query_completes_just_before_timeout( - self, snowflake_context: Context, keboola_client: KeboolaClient, mocker - ) -> None: - """Verify that a completed query is properly returned and no RuntimeError is raised.""" - keboola_client.storage_client.branches_list.return_value = [{'id': 1234, 'isDefault': True}] - - qsclient = mocker.AsyncMock(QueryServiceClient) - qsclient.submit_job.return_value = 'qs-job-success-123' - - # Return 'processing' a few times, then 'completed' before timeout - status_responses = [ - {'status': 'processing', 'statements': [{'id': 'stmt-1'}]}, - {'status': 'processing', 'statements': [{'id': 'stmt-1'}]}, - {'status': 'completed', 'statements': [{'id': 'stmt-1'}]}, - ] - - qsclient.cancel_job.return_value = {} - qsclient.get_job_status.side_effect = status_responses - qsclient.get_job_results.return_value = { - 'status': 'completed', - 'message': 'Query completed', - 'columns': [{'name': 'col1'}], - 'data': [['value1']], - } - - mocker.patch.object(QueryServiceClient, 'create', return_value=qsclient) - mocker.patch.object(_SnowflakeWorkspace, '_QUERY_TIMEOUT', 300.0) # Normal 5-minute timeout - - m = WorkspaceManager.from_state(snowflake_context.session.state) - - result = await m.execute_query('SELECT col1 FROM fast_table') - - # Query should succeed normally - assert result.is_ok - assert result.data is not None - assert result.data.columns == ['col1'] - - # Verify cancellation was NOT called - qsclient.cancel_job.assert_not_called() diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 000000000..6cddefc9a --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "@keboola/tsconfig/base.json", + "compilerOptions": { + "lib": ["ESNext"], + "types": ["node", "vitest/globals"], + "esModuleInterop": true, + "outDir": "dist", + "rootDir": ".", + "paths": { + "@/*": ["./src/*"] + }, + "noEmit": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts", "__tests__/**/*.ts", "integtests/**/*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/tsup.config.ts b/tsup.config.ts new file mode 100644 index 000000000..997808d77 --- /dev/null +++ b/tsup.config.ts @@ -0,0 +1,40 @@ +import { defineConfig } from 'tsup'; + +// Bundles the server into `dist/`. `index.ts` is the npx/bin entry. Third-party +// deps stay external (this ships as an npm package), but the `@/*` source alias +// is resolved at build time. +export default defineConfig({ + // `index` is the npx/bin server entry. `docs-build` (fixture seed) and `docs-crawl` + // (real crawl of the public help+dev docs → migrate + embed + seed) are docs-index + // CLIs emitted so they run in the production image (which has no tsx/scripts/) as + // `node dist/docs-build.js` / `node dist/docs-crawl.js` — used by the docker-compose + // docs-seed service and the kbc-stacks index-build CronJob. cheerio ships as an + // optionalDependency (installed by `npm ci --omit=dev`), so the crawl runs in-image. + entry: { + index: 'src/index.ts', + 'docs-build': 'scripts/docs-build.ts', + 'docs-crawl': 'scripts/docs-crawl.ts', + }, + format: ['esm'], + target: 'node22', + platform: 'node', + splitting: false, + dts: true, + sourcemap: true, + clean: true, + outDir: 'dist', + // Shebang so `npx @keboola/mcp-server` runs directly. + banner: { js: '#!/usr/bin/env node' }, + external: [/^[^.@]/, /^@(?!\/)/], + // Force-bundle @keboola/api-client (+ its dayjs dependency). Its published ESM uses + // extensionless subpath imports (e.g. `import 'dayjs/plugin/utc'`) that Node 22's strict + // ESM resolver rejects at runtime — so if it stayed external the built `dist/index.js` + // would crash on boot (`ERR_MODULE_NOT_FOUND: dayjs/plugin/utc`). Bundling lets esbuild + // resolve those imports at build time. noExternal takes precedence over `external`. + noExternal: [/^@keboola\/api-client/, 'dayjs'], + // Resource files (flow schema/examples, system prompt, data-app code templates) + // are read from disk at runtime via `@/resource-path` (resolves to dist/resources + // in the bundle). Copy the tree into dist so it ships in the image and on npm. + // Idempotent copy: rm first so a re-run can't nest into dist/resources/resources. + onSuccess: 'rm -rf dist/resources && cp -R src/resources dist/resources', +}); diff --git a/uv.lock b/uv.lock deleted file mode 100644 index ec6a2a02d..000000000 --- a/uv.lock +++ /dev/null @@ -1,2910 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.10" -resolution-markers = [ - "python_full_version >= '3.13'", - "python_full_version >= '3.11' and python_full_version < '3.13'", - "python_full_version < '3.11'", -] - -[options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. -exclude-newer-span = "P7D" - -[[package]] -name = "aiofile" -version = "3.9.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -dependencies = [ - { name = "caio", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/67/e2/d7cb819de8df6b5c1968a2756c3cb4122d4fa2b8fc768b53b7c9e5edb646/aiofile-3.9.0.tar.gz", hash = "sha256:e5ad718bb148b265b6df1b3752c4d1d83024b93da9bd599df74b9d9ffcf7919b", size = 17943, upload-time = "2024-10-08T10:39:35.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/50/25/da1f0b4dd970e52bf5a36c204c107e11a0c6d3ed195eba0bfbc664c312b2/aiofile-3.9.0-py3-none-any.whl", hash = "sha256:ce2f6c1571538cbdfa0143b04e16b208ecb0e9cb4148e528af8a640ed51cc8aa", size = 19539, upload-time = "2024-10-08T10:39:32.955Z" }, -] - -[[package]] -name = "aiofile" -version = "3.11.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.13'", - "python_full_version >= '3.11' and python_full_version < '3.13'", -] -dependencies = [ - { name = "caio", marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/41/2fea7e193e061ce54eacc3b7bc0e6a99e4fcff43c78cf0a76dd781ed8334/aiofile-3.11.1.tar.gz", hash = "sha256:1f91912c6643d2a4e49ca4ae3514f0bf3867ce948a36d99a6411b8f4755f4cf9", size = 19342, upload-time = "2026-05-16T08:18:33.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/cd/0d76dfc5de72bde52f55f53e925c7d152d9c7906634ec1e0cbc7e8d4ad93/aiofile-3.11.1-py3-none-any.whl", hash = "sha256:ce77d14ac07f77bc2b757834a5c129321f3f705c474593deed5ab209079a52c9", size = 20446, upload-time = "2026-05-16T08:18:32.051Z" }, -] - -[[package]] -name = "annotated-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, -] - -[[package]] -name = "anyio" -version = "4.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "idna" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, -] - -[[package]] -name = "attrs" -version = "26.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, -] - -[[package]] -name = "authlib" -version = "1.7.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "joserfc" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/36/98/7d93f30d029643c0275dbc0bd6d5a6f670661ee6c9a94d93af7ab4887600/authlib-1.7.2.tar.gz", hash = "sha256:2cea25fefcd4e7173bdf1372c0afc265c8034b23a8cd5dcb6a9164b826c64231", size = 176511, upload-time = "2026-05-06T08:10:23.116Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/95/adcb68e20c34162e9135f370d6e31737719c2b6f94bc953fe7ed1f10fe21/authlib-1.7.2-py2.py3-none-any.whl", hash = "sha256:3e1faedc9d87e7d56a164eca3ccb6ace0d61b94abe83e92242f8dc8bba9b4a9f", size = 259548, upload-time = "2026-05-06T08:10:21.436Z" }, -] - -[[package]] -name = "azure-core" -version = "1.41.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" }, -] - -[[package]] -name = "azure-storage-blob" -version = "12.29.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-core" }, - { name = "cryptography" }, - { name = "isodate" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/59/25/fdcf1e381922dbab8ba23d6fd78d397fe6cbac6b480310218834b7bc91fe/azure_storage_blob-12.29.0.tar.gz", hash = "sha256:2824ddd7ebc9056034ebc76b17971a38e9aa5835abb0d565b9700493f2a6c657", size = 611359, upload-time = "2026-05-15T03:34:59.865Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/2c/6ddee6a3e42d0236ba9259e4df7fa97fdc415ff0802b736c634baaf4b285/azure_storage_blob-12.29.0-py3-none-any.whl", hash = "sha256:ccf8a1bcd5e49df83ab85aab793b579e5ba2eeea2ad8900b2f62ca3a37dc391f", size = 434823, upload-time = "2026-05-15T03:35:01.837Z" }, -] - -[[package]] -name = "backports-asyncio-runner" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/ff/70dca7d7cb1cbc0edb2c6cc0c38b65cba36cccc491eca64cabd5fe7f8670/backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162", size = 69893, upload-time = "2025-07-02T02:27:15.685Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, -] - -[[package]] -name = "backports-tarfile" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991", size = 86406, upload-time = "2024-05-28T17:01:54.731Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181, upload-time = "2024-05-28T17:01:53.112Z" }, -] - -[[package]] -name = "beartype" -version = "0.22.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c7/94/1009e248bbfbab11397abca7193bea6626806be9a327d399810d523a07cb/beartype-0.22.9.tar.gz", hash = "sha256:8f82b54aa723a2848a56008d18875f91c1db02c32ef6a62319a002e3e25a975f", size = 1608866, upload-time = "2025-12-13T06:50:30.72Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/71/cc/18245721fa7747065ab478316c7fea7c74777d07f37ae60db2e84f8172e8/beartype-0.22.9-py3-none-any.whl", hash = "sha256:d16c9bbc61ea14637596c5f6fbff2ee99cbe3573e46a716401734ef50c3060c2", size = 1333658, upload-time = "2025-12-13T06:50:28.266Z" }, -] - -[[package]] -name = "black" -version = "26.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "mypy-extensions" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "platformdirs" }, - { name = "pytokens" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/37/5628dd55bf2b34257fc7603f0fe97c40e3aaf24265f416a9c85c95ca1436/black-26.5.1.tar.gz", hash = "sha256:dd321f668053961824bcc1be1cc1df748b2d7e4fa28086b08331e577b0100a73", size = 679439, upload-time = "2026-05-18T16:53:36.107Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/84/b3f55026206a9e8820a91503308075ca48eadc515e436731ca01dbe043b3/black-26.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9942db8888e06943c5dde66ca0037dcff82a2a4ec1ad0ada9e0d2ee9d9823893", size = 1987719, upload-time = "2026-05-18T17:05:02.757Z" }, - { url = "https://files.pythonhosted.org/packages/c6/34/7db312c5e5783d6e76cffd9d5ac8972a32badae4c6e3288dac0eed8d3bed/black-26.5.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:89c93167a74d3a75dfaa38a5c7cca015537d5820dd7f17d63267d674a61cae90", size = 1810083, upload-time = "2026-05-18T17:05:04.302Z" }, - { url = "https://files.pythonhosted.org/packages/33/e2/e0101e73c2c8727634e2efcb35e2b34bd23ad70dfa673789f5773a591b21/black-26.5.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22f2cd76d069cc54c71f10360744ba8983fbb616903b4304a85b734915c8e1b4", size = 1860633, upload-time = "2026-05-18T17:05:06.391Z" }, - { url = "https://files.pythonhosted.org/packages/b0/4c/e15c0c5b23cf3651035fe5addcce90e283af3548a3f91bb03d81b83106ab/black-26.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:87ed5c6f450580a2f6790bc7cbfb016dfc73bc750249762268a3695361315eef", size = 1477886, upload-time = "2026-05-18T17:05:07.96Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3f/59d43ade98d2ce5c8dc34a4e46cbecd177e6d55d7d4092969c6003ccc655/black-26.5.1-cp310-cp310-win_arm64.whl", hash = "sha256:58b4bd92cf88aacf83d88479c8f9caee044b1ec55f2451a337354a7ea2590a22", size = 1277111, upload-time = "2026-05-18T17:05:09.473Z" }, - { url = "https://files.pythonhosted.org/packages/4b/96/3c3e09f09f44a37aac36b178a279cd19aa7001bd796187a7b162a294c81f/black-26.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:96ae2c733b2aabdd9986e2c5df628ff3473676cd1c5faded1ff496cf6d74083c", size = 1970639, upload-time = "2026-05-18T17:05:11.461Z" }, - { url = "https://files.pythonhosted.org/packages/83/ea/5ad117b9ee3ecd933c712bcbae610006e5b7cc9f41c526cd7ed3b6c4124c/black-26.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0e48b87e03bf109288e55cfceadcfa15ff5470aca2851a851950ed2926f450d7", size = 1792130, upload-time = "2026-05-18T17:05:12.983Z" }, - { url = "https://files.pythonhosted.org/packages/06/3a/7c448bc623fcdfa96672531beb5a616ea5e64f6975955254d7731ffb0ad9/black-26.5.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5119fa92ae61f786e8c3662fd60aece1d0a2dd5cca5d0c79417a95e7a4272a59", size = 1846134, upload-time = "2026-05-18T17:05:14.506Z" }, - { url = "https://files.pythonhosted.org/packages/a1/5b/0b39b3a5917f0657ac014ad2edb58c139553a478adfe7f817abf1622ff6e/black-26.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:30d3c14661f2792e9142cce3eeeb1cbc175b3eb5f733be0c8eeb99651e52b0c3", size = 1478883, upload-time = "2026-05-18T17:05:16.542Z" }, - { url = "https://files.pythonhosted.org/packages/4c/48/dc222692e0f95030db1bbfb6c857e76858bad09058221ea7aae815255327/black-26.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:1ef92b76f7733f282fd096ea406200b5a286c42947412b0eaff3a74e3616cefe", size = 1277776, upload-time = "2026-05-18T17:05:18.029Z" }, - { url = "https://files.pythonhosted.org/packages/24/99/7744b906703228264ef73bdd534df88ec1ef3de45c4e78f6d31b9e32d0c9/black-26.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4ad6fa01f941920f54f2bbb35f3df7673428a0ef98a0b0840c2eaef3b110efa8", size = 2012518, upload-time = "2026-05-18T17:05:20.108Z" }, - { url = "https://files.pythonhosted.org/packages/b7/c0/c5a3b1636dfd09c42534f2b3cf33506814f6d3e066fb0879ffa16c1ae860/black-26.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3915f256e75a2d7cf88d8953d37f780455dc586cc72dee059c528fe77f581217", size = 1816016, upload-time = "2026-05-18T17:05:21.84Z" }, - { url = "https://files.pythonhosted.org/packages/1f/0e/36044316b65ca471d3bb6d3703fd06fb50c6b727c3562f6a5a3153634f88/black-26.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d98d4137277c75dfb898ec8d846c4fd68ba1e9cf77f95e2865c203dc18f4c3d", size = 1884150, upload-time = "2026-05-18T17:05:23.546Z" }, - { url = "https://files.pythonhosted.org/packages/b3/33/dafc5808c2af43672912111d7c3354af1615f7e2be3bed7a878461abbe4d/black-26.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:a1dca32d9f1784af512a13410ec204c6f7f0aa9797a111c42e1c03449821c264", size = 1486825, upload-time = "2026-05-18T17:05:25.004Z" }, - { url = "https://files.pythonhosted.org/packages/82/14/b965ee6ad2a311f28bdbf692def3ee9848d2ae289dab28b27657fcee3e78/black-26.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1037d5ac7b7b310b2632ad867ec8d0e4c4819dcdb0b820f63135da746a24e418", size = 1288646, upload-time = "2026-05-18T17:05:26.477Z" }, - { url = "https://files.pythonhosted.org/packages/3f/5c/c384363980e11e25ca6b93205949bb331fbf35f4e0dbec376dfa6326cec8/black-26.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2b36cf2ddf5566e205f6535f782a62194a184d33e175b64ae8c40b1737522be3", size = 2009020, upload-time = "2026-05-18T17:05:28.132Z" }, - { url = "https://files.pythonhosted.org/packages/0b/df/9f31c5e0babbfed77d505fc5d120beb98b21b33feaeded3924ea941fe360/black-26.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f7ea64ebfa01b50f693508fc39f875e264446d3b097088f84f203b9d09618a0", size = 1813335, upload-time = "2026-05-18T17:05:31.266Z" }, - { url = "https://files.pythonhosted.org/packages/fb/24/8e7b9a2fa61b0afd82209efe937557d180a1fa055bd7f6161eb9defc3719/black-26.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecb3e624844c798144e9bd986954e0adc81d8911a1f30f375e1252fe26e8c294", size = 1881614, upload-time = "2026-05-18T17:05:32.718Z" }, - { url = "https://files.pythonhosted.org/packages/49/ad/b4e0d9365ba8ac34f6bbab62a4b1b2dd5d618fac3fa1b8db968c844201b5/black-26.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:e1a26503279b6b310669fb0b219c39e4820b77e8189fe80f522bb511f247db0a", size = 1488925, upload-time = "2026-05-18T17:05:34.259Z" }, - { url = "https://files.pythonhosted.org/packages/a1/4b/652b859bf5df88a751c30451b09338f7fd26a77d1271c666992f836b7711/black-26.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c34b25da232ead53a6f335b76dbea124f4d152ad568b9080d6f944bc2b34b52", size = 1289883, upload-time = "2026-05-18T17:05:36.019Z" }, - { url = "https://files.pythonhosted.org/packages/a6/16/a8da8eb208c51c7f4ce74609a45d0dcc6d8a2141e45e81ee5289d1bb0d59/black-26.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e88976690a64b0af98312ca958415849cb42423423c5f2ee74af4b49a97a2168", size = 2004800, upload-time = "2026-05-18T17:05:38.182Z" }, - { url = "https://files.pythonhosted.org/packages/11/8a/a479296a19e383b70a725882a6cf3d786540601ff03cabbaaf1cce864c5a/black-26.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32d5ea7f6c8bdfa6e648326ebca1f02b0764e2a029edc6f8dce2627e19d468c3", size = 1815576, upload-time = "2026-05-18T17:05:40.309Z" }, - { url = "https://files.pythonhosted.org/packages/81/6b/cfaf3d39f25132c156a068f6b805576c9103a84086019507c70e1911ee7d/black-26.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea8d16dc41655aa113cd64665e7219446cd7e4ff2248d7178eaa905190c86b18", size = 1877927, upload-time = "2026-05-18T17:05:42.463Z" }, - { url = "https://files.pythonhosted.org/packages/66/76/302e313964bcff7e28df329d39f84f5270095730d85ff0acc260610a0d82/black-26.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:577f21094ea469ef92ec1adaf2c9441a226d2144d01a5be2fa823cecf6543e50", size = 1511860, upload-time = "2026-05-18T17:05:43.943Z" }, - { url = "https://files.pythonhosted.org/packages/27/4e/a3827e35e0e567f9f9ee59e2a0ab979267dca98718f25547ca8c6733afd4/black-26.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:ed1a20af114c301a0269bf01163d51dbef72737fd65f850001e7cbe7f3c7abae", size = 1316632, upload-time = "2026-05-18T17:05:45.521Z" }, - { url = "https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl", hash = "sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2", size = 213693, upload-time = "2026-05-18T16:53:33.964Z" }, -] - -[[package]] -name = "boto3" -version = "1.43.22" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, - { name = "jmespath" }, - { name = "s3transfer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/31/32388d5ec332ffe81d8f3650860f94b66294009172d188c390cad58c6f5f/boto3-1.43.22.tar.gz", hash = "sha256:2a7fe12d8e0731bb8aa7c1e59b4ccc770fda031b8659c2f6f497393bdcec3051", size = 113203, upload-time = "2026-06-03T19:33:13.39Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/6b/c2fb3b91e849882df5426e68cc15eb2b5ba6ac28325ab2aa3a1065da5884/boto3-1.43.22-py3-none-any.whl", hash = "sha256:0597fb9fe1613e636ac55219a5a54ad0fcb7c15e6be32c799301f7fb53ff04e1", size = 140536, upload-time = "2026-06-03T19:33:10.939Z" }, -] - -[[package]] -name = "botocore" -version = "1.43.22" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jmespath" }, - { name = "python-dateutil" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/cf/840f1b8db16d45e3807c23d1ea779723eed1cd9cf3b6c49e16f372d2a777/botocore-1.43.22.tar.gz", hash = "sha256:b00de525e538289ed4a7a85263f1be4e47473c124cec87be6b23be49356bf745", size = 15458781, upload-time = "2026-06-03T19:33:02.882Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/6b/576a1b0f915871e35f14a33104f2bcae635f19c6a72486ba639db0d1fc70/botocore-1.43.22-py3-none-any.whl", hash = "sha256:ceec9f81d0891abe7b28ca2b2ee47e32de7b3360ad11e80d351470f015217379", size = 15141375, upload-time = "2026-06-03T19:32:58.324Z" }, -] - -[[package]] -name = "cachetools" -version = "6.2.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/39/91/d9ae9a66b01102a18cd16db0cf4cd54187ffe10f0865cc80071a4104fbb3/cachetools-6.2.6.tar.gz", hash = "sha256:16c33e1f276b9a9c0b49ab5782d901e3ad3de0dd6da9bf9bcd29ac5672f2f9e6", size = 32363, upload-time = "2026-01-27T20:32:59.956Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/45/f458fa2c388e79dd9d8b9b0c99f1d31b568f27388f2fdba7bb66bbc0c6ed/cachetools-6.2.6-py3-none-any.whl", hash = "sha256:8c9717235b3c651603fff0076db52d6acbfd1b338b8ed50256092f7ce9c85bda", size = 11668, upload-time = "2026-01-27T20:32:58.527Z" }, -] - -[[package]] -name = "caio" -version = "0.9.25" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/88/b8527e1b00c1811db339a1df8bd1ae49d146fcea9d6a5c40e3a80aaeb38d/caio-0.9.25.tar.gz", hash = "sha256:16498e7f81d1d0f5a4c0ad3f2540e65fe25691376e0a5bd367f558067113ed10", size = 26781, upload-time = "2025-12-26T15:21:36.501Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/80/ea4ead0c5d52a9828692e7df20f0eafe8d26e671ce4883a0a146bb91049e/caio-0.9.25-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ca6c8ecda611478b6016cb94d23fd3eb7124852b985bdec7ecaad9f3116b9619", size = 36836, upload-time = "2025-12-26T15:22:04.662Z" }, - { url = "https://files.pythonhosted.org/packages/17/b9/36715c97c873649d1029001578f901b50250916295e3dddf20c865438865/caio-0.9.25-cp310-cp310-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db9b5681e4af8176159f0d6598e73b2279bb661e718c7ac23342c550bd78c241", size = 79695, upload-time = "2025-12-26T15:22:18.818Z" }, - { url = "https://files.pythonhosted.org/packages/0b/ab/07080ecb1adb55a02cbd8ec0126aa8e43af343ffabb6a71125b42670e9a1/caio-0.9.25-cp310-cp310-manylinux_2_34_aarch64.whl", hash = "sha256:bf61d7d0c4fd10ffdd98ca47f7e8db4d7408e74649ffaf4bef40b029ada3c21b", size = 79457, upload-time = "2026-03-04T22:08:16.024Z" }, - { url = "https://files.pythonhosted.org/packages/88/95/dd55757bb671eb4c376e006c04e83beb413486821f517792ea603ef216e9/caio-0.9.25-cp310-cp310-manylinux_2_34_x86_64.whl", hash = "sha256:ab52e5b643f8bbd64a0605d9412796cd3464cb8ca88593b13e95a0f0b10508ae", size = 77705, upload-time = "2026-03-04T22:08:17.202Z" }, - { url = "https://files.pythonhosted.org/packages/ec/90/543f556fcfcfa270713eef906b6352ab048e1e557afec12925c991dc93c2/caio-0.9.25-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d6956d9e4a27021c8bd6c9677f3a59eb1d820cc32d0343cea7961a03b1371965", size = 36839, upload-time = "2025-12-26T15:21:40.267Z" }, - { url = "https://files.pythonhosted.org/packages/51/3b/36f3e8ec38dafe8de4831decd2e44c69303d2a3892d16ceda42afed44e1b/caio-0.9.25-cp311-cp311-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf84bfa039f25ad91f4f52944452a5f6f405e8afab4d445450978cd6241d1478", size = 80255, upload-time = "2025-12-26T15:22:20.271Z" }, - { url = "https://files.pythonhosted.org/packages/df/ce/65e64867d928e6aff1b4f0e12dba0ef6d5bf412c240dc1df9d421ac10573/caio-0.9.25-cp311-cp311-manylinux_2_34_aarch64.whl", hash = "sha256:ae3d62587332bce600f861a8de6256b1014d6485cfd25d68c15caf1611dd1f7c", size = 80052, upload-time = "2026-03-04T22:08:20.402Z" }, - { url = "https://files.pythonhosted.org/packages/46/90/e278863c47e14ec58309aa2e38a45882fbe67b4cc29ec9bc8f65852d3e45/caio-0.9.25-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:fc220b8533dcf0f238a6b1a4a937f92024c71e7b10b5a2dfc1c73604a25709bc", size = 78273, upload-time = "2026-03-04T22:08:21.368Z" }, - { url = "https://files.pythonhosted.org/packages/d3/25/79c98ebe12df31548ba4eaf44db11b7cad6b3e7b4203718335620939083c/caio-0.9.25-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fb7ff95af4c31ad3f03179149aab61097a71fd85e05f89b4786de0359dffd044", size = 36983, upload-time = "2025-12-26T15:21:36.075Z" }, - { url = "https://files.pythonhosted.org/packages/a3/2b/21288691f16d479945968a0a4f2856818c1c5be56881d51d4dac9b255d26/caio-0.9.25-cp312-cp312-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:97084e4e30dfa598449d874c4d8e0c8d5ea17d2f752ef5e48e150ff9d240cd64", size = 82012, upload-time = "2025-12-26T15:22:20.983Z" }, - { url = "https://files.pythonhosted.org/packages/03/c4/8a1b580875303500a9c12b9e0af58cb82e47f5bcf888c2457742a138273c/caio-0.9.25-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:4fa69eba47e0f041b9d4f336e2ad40740681c43e686b18b191b6c5f4c5544bfb", size = 81502, upload-time = "2026-03-04T22:08:22.381Z" }, - { url = "https://files.pythonhosted.org/packages/d1/1c/0fe770b8ffc8362c48134d1592d653a81a3d8748d764bec33864db36319d/caio-0.9.25-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:6bebf6f079f1341d19f7386db9b8b1f07e8cc15ae13bfdaff573371ba0575d69", size = 80200, upload-time = "2026-03-04T22:08:23.382Z" }, - { url = "https://files.pythonhosted.org/packages/31/57/5e6ff127e6f62c9f15d989560435c642144aa4210882f9494204bc892305/caio-0.9.25-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d6c2a3411af97762a2b03840c3cec2f7f728921ff8adda53d7ea2315a8563451", size = 36979, upload-time = "2025-12-26T15:21:35.484Z" }, - { url = "https://files.pythonhosted.org/packages/a3/9f/f21af50e72117eb528c422d4276cbac11fb941b1b812b182e0a9c70d19c5/caio-0.9.25-cp313-cp313-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0998210a4d5cd5cb565b32ccfe4e53d67303f868a76f212e002a8554692870e6", size = 81900, upload-time = "2025-12-26T15:22:21.919Z" }, - { url = "https://files.pythonhosted.org/packages/9c/12/c39ae2a4037cb10ad5eb3578eb4d5f8c1a2575c62bba675f3406b7ef0824/caio-0.9.25-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:1a177d4777141b96f175fe2c37a3d96dec7911ed9ad5f02bac38aaa1c936611f", size = 81523, upload-time = "2026-03-04T22:08:25.187Z" }, - { url = "https://files.pythonhosted.org/packages/22/59/f8f2e950eb4f1a5a3883e198dca514b9d475415cb6cd7b78b9213a0dd45a/caio-0.9.25-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:9ed3cfb28c0e99fec5e208c934e5c157d0866aa9c32aa4dc5e9b6034af6286b7", size = 80243, upload-time = "2026-03-04T22:08:26.449Z" }, - { url = "https://files.pythonhosted.org/packages/69/ca/a08fdc7efdcc24e6a6131a93c85be1f204d41c58f474c42b0670af8c016b/caio-0.9.25-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fab6078b9348e883c80a5e14b382e6ad6aabbc4429ca034e76e730cf464269db", size = 36978, upload-time = "2025-12-26T15:21:41.055Z" }, - { url = "https://files.pythonhosted.org/packages/5e/6c/d4d24f65e690213c097174d26eda6831f45f4734d9d036d81790a27e7b78/caio-0.9.25-cp314-cp314-manylinux2010_x86_64.manylinux2014_x86_64.manylinux_2_12_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:44a6b58e52d488c75cfaa5ecaa404b2b41cc965e6c417e03251e868ecd5b6d77", size = 81832, upload-time = "2025-12-26T15:22:22.757Z" }, - { url = "https://files.pythonhosted.org/packages/87/a4/e534cf7d2d0e8d880e25dd61e8d921ffcfe15bd696734589826f5a2df727/caio-0.9.25-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:628a630eb7fb22381dd8e3c8ab7f59e854b9c806639811fc3f4310c6bd711d79", size = 81565, upload-time = "2026-03-04T22:08:27.483Z" }, - { url = "https://files.pythonhosted.org/packages/3f/ed/bf81aeac1d290017e5e5ac3e880fd56ee15e50a6d0353986799d1bc5cfd5/caio-0.9.25-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:0ba16aa605ccb174665357fc729cf500679c2d94d5f1458a6f0d5ca48f2060a7", size = 80071, upload-time = "2026-03-04T22:08:28.751Z" }, - { url = "https://files.pythonhosted.org/packages/86/93/1f76c8d1bafe3b0614e06b2195784a3765bbf7b0a067661af9e2dd47fc33/caio-0.9.25-py3-none-any.whl", hash = "sha256:06c0bb02d6b929119b1cfbe1ca403c768b2013a369e2db46bfa2a5761cf82e40", size = 19087, upload-time = "2025-12-26T15:22:00.221Z" }, -] - -[[package]] -name = "certifi" -version = "2026.5.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, -] - -[[package]] -name = "cffi" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/d7/516d984057745a6cd96575eea814fe1edd6646ee6efd552fb7b0921dec83/cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", size = 184283, upload-time = "2025-09-08T23:22:08.01Z" }, - { url = "https://files.pythonhosted.org/packages/9e/84/ad6a0b408daa859246f57c03efd28e5dd1b33c21737c2db84cae8c237aa5/cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", size = 180504, upload-time = "2025-09-08T23:22:10.637Z" }, - { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" }, - { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" }, - { url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" }, - { url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" }, - { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" }, - { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" }, - { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" }, - { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" }, - { url = "https://files.pythonhosted.org/packages/e2/cc/027d7fb82e58c48ea717149b03bcadcbdc293553edb283af792bd4bcbb3f/cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", size = 172184, upload-time = "2025-09-08T23:22:23.328Z" }, - { url = "https://files.pythonhosted.org/packages/33/fa/072dd15ae27fbb4e06b437eb6e944e75b068deb09e2a2826039e49ee2045/cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", size = 182790, upload-time = "2025-09-08T23:22:24.752Z" }, - { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, - { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, - { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, - { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, - { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, - { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, - { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, - { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, - { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, - { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, - { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, - { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, - { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, - { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, - { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, - { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, - { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, - { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, - { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, - { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, - { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, - { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, - { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, - { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, - { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, - { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, - { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, - { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, - { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, - { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, - { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, - { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, - { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, - { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, - { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, - { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, - { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, - { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, - { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, - { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, - { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, - { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, - { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, - { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, - { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, - { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, - { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, -] - -[[package]] -name = "chardet" -version = "7.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/19/b6/9df434a8eeba2e6628c465a1dfa31034228ef79b26f76f46278f4ef7e49d/chardet-7.4.3.tar.gz", hash = "sha256:cc1d4eb92a4ec1c2df3b490836ffa46922e599d34ce0bb75cf41fd2bf6303d56", size = 784800, upload-time = "2026-04-13T21:33:39.803Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/1b/7f73766c119a1344eb69e31890ede7c5825ce03d69a9d29292d1bd1cfa1b/chardet-7.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0c79b13c9908ac7dfe0a74116ebc9a0f28b2319d23c32f3dfcdfbe1279c7eaf", size = 874121, upload-time = "2026-04-13T21:32:47.065Z" }, - { url = "https://files.pythonhosted.org/packages/8b/02/b677c8203d34dad6c2af48287bb1f8c5dff63db2094636fbe634b555b7fb/chardet-7.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bba8bea1b28d927b3e99e47deafe53658d34497c0a891d95ff1ba8ff6663f01c", size = 856900, upload-time = "2026-04-13T21:32:48.893Z" }, - { url = "https://files.pythonhosted.org/packages/c4/4b/1361a485a999d97cac4c895e615326f69a639532a52ef365a468bd09bad1/chardet-7.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23163921dccf3103ce59540b0443c106d2c0a0ff2e0503e05196f5e6fdea453f", size = 876634, upload-time = "2026-04-13T21:32:50.238Z" }, - { url = "https://files.pythonhosted.org/packages/87/23/e31c8ad33aa448f0845fd58af5fc22da1626407616d09df4973b2b34f477/chardet-7.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cfb54563fe5f130da17c44c6a4e2e8052ba628e5ab4eab7ef8190f736f0f8f72", size = 886497, upload-time = "2026-04-13T21:32:52.111Z" }, - { url = "https://files.pythonhosted.org/packages/18/ef/ea4edec8c87f7e6eda02673acc68fe48725e564fc5a1865782efb53d5598/chardet-7.4.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3990fffcc6a6045f2234ab72752ad037e3b2d48c72037f244d42738db397eb75", size = 881061, upload-time = "2026-04-13T21:32:53.755Z" }, - { url = "https://files.pythonhosted.org/packages/f2/11/fc10600da98541777d720ad9e6bc040c0e0af1adb92e27142e35158957cb/chardet-7.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c7116b0452994734ccff35e154b44240090eb0f4f74b9106292668133557c175", size = 942533, upload-time = "2026-04-13T21:32:55.134Z" }, - { url = "https://files.pythonhosted.org/packages/19/52/505c207f334d51e937cbaa27ff95776e16e2d120e13cbe491cd7b3a70b50/chardet-7.4.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:25a862cddc6a9ac07023e808aedd297115345fbaabc2690479481ddc0f980e09", size = 870747, upload-time = "2026-04-13T21:32:56.916Z" }, - { url = "https://files.pythonhosted.org/packages/14/4b/d3c79495dee4831b8bebca2790e72cb90f0c5849c940570a7c7e5b70b952/chardet-7.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7005c88da26fd95d8abb8acbe6281d833e9a9181b03cf49b4546c4555389bd97", size = 853210, upload-time = "2026-04-13T21:32:58.309Z" }, - { url = "https://files.pythonhosted.org/packages/b9/99/f6a822ad1bde25a4c38dc3e770485e78e0893dfd871cd6e18ed3ea3a795e/chardet-7.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc50f28bad067393cce0af9091052c3b8df7a23115afd8ba7b2e0947f0cef1f8", size = 873625, upload-time = "2026-04-13T21:32:59.606Z" }, - { url = "https://files.pythonhosted.org/packages/b1/10/31932775c94a86814f76b41c4a772b52abfb0e6125324f32c6da1196c297/chardet-7.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c3da294de1a681097848ab58bd3f2771a674f8039d2d87a5538b28856b815e9", size = 883436, upload-time = "2026-04-13T21:33:01.351Z" }, - { url = "https://files.pythonhosted.org/packages/6c/63/0f43e3acf2c436fdb32a0f904aeb03a2904d2126eed34a042a194d235926/chardet-7.4.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:93c45e116dd51b66226a53ade3f9f635e870de5399b90e00ce45dcc311093bf4", size = 876589, upload-time = "2026-04-13T21:33:02.636Z" }, - { url = "https://files.pythonhosted.org/packages/5d/a6/e9b8f8a3e99602792b01fa7d0a731737615ab56d8bfd0b52935a0ef88b85/chardet-7.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:ccc1f83ab4bcfb901cf39e0c4ba6bc6e726fc6264735f10e24ceb5cb47387578", size = 941866, upload-time = "2026-04-13T21:33:04.282Z" }, - { url = "https://files.pythonhosted.org/packages/61/33/29de185079e6675c3f375546e30a559b7ddc75ce972f18d6e566cd9ea4eb/chardet-7.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:75d3c65cc16bddf40b8da1fd25ba84fca5f8070f2b14e86083653c1c85aee971", size = 874870, upload-time = "2026-04-13T21:33:05.977Z" }, - { url = "https://files.pythonhosted.org/packages/9c/2f/4c5af01fd1a7506a1d5375403d68925eac70289229492db5aa68b58103d8/chardet-7.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:29af5999f654e8729d251f1724a62b538b1262d9292cccaefddf8a02aae1ef6a", size = 854859, upload-time = "2026-04-13T21:33:07.381Z" }, - { url = "https://files.pythonhosted.org/packages/36/21/edb36ad5dfa48d7f8eed97ab43931ecdaa8c15166c21b1d614967e49d681/chardet-7.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:626f00299ad62dfe937058a09572beed442ccc7b58f87aa667949b20fd3db235", size = 875032, upload-time = "2026-04-13T21:33:08.741Z" }, - { url = "https://files.pythonhosted.org/packages/e5/59/a32a241d861cf180853a11c8e5a67641cb1b2af13c3a5ccce83ec07e2c9f/chardet-7.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a4904dd5f071b7a7d7f50b4a67a86db3c902d243bf31708f1d5cde2f68239cb", size = 888283, upload-time = "2026-04-13T21:33:10.213Z" }, - { url = "https://files.pythonhosted.org/packages/87/2e/e1ee6a77abf3782c00e05b89c4d4328c8353bf9500661c4348df1dd68614/chardet-7.4.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d2879598bc220689e8ce509fe9c3f37ad2fca53a36be9c9bd91abdd91dd364f", size = 879974, upload-time = "2026-04-13T21:33:11.448Z" }, - { url = "https://files.pythonhosted.org/packages/32/60/fca69c534602a7ced04280c952a246ad1edde2a6ca3a164f65d32ac41fe7/chardet-7.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:4b2799bd58e7245cfa8d4ab2e8ad1d76a5c3a5b1f32318eb6acca4c69a3e7101", size = 943973, upload-time = "2026-04-13T21:33:12.756Z" }, - { url = "https://files.pythonhosted.org/packages/7c/43/79ac9b4db5bc87020c9dbc419125371d80882d1d197e9c4765ba8682b605/chardet-7.4.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a9e4486df251b8962e86ea9f139ca235aa6e0542a00f7844c9a04160afb99aa9", size = 873769, upload-time = "2026-04-13T21:33:14.002Z" }, - { url = "https://files.pythonhosted.org/packages/55/5f/25bdec773905bff0ff6cf35ca73b17bd05593b4f87bd8c5fa43705f7167d/chardet-7.4.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4fbff1907925b0c5a1064cffb5e040cd5e338585c9c552625f30de6bc2f3107a", size = 853991, upload-time = "2026-04-13T21:33:15.564Z" }, - { url = "https://files.pythonhosted.org/packages/b4/07/a29380ee0b215d23d77733b5ad60c5c0c7969650e080c667acdf9462040d/chardet-7.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:365135eaf37ba65a828f8e668eb0a8c38c479dcbec724dc25f4dfd781049c357", size = 874024, upload-time = "2026-04-13T21:33:16.915Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b1/3338e121cbd4c8a126b8ccb1061170c2ce51a53f678c502793ea49c6fd6d/chardet-7.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfc134b70c846c21ead8e43ada3ae1a805fff732f6922f8abcf2ff27b8f6493d", size = 887410, upload-time = "2026-04-13T21:33:18.368Z" }, - { url = "https://files.pythonhosted.org/packages/63/1c/44a9a9e0c59c185a5d307ceaeee8768afa1558f0a24f7a4b5fa11b67586b/chardet-7.4.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9acd9988a93e09390f3cd231201ea7166c415eb8da1b735928990ffc05cb9fbb", size = 879269, upload-time = "2026-04-13T21:33:20.377Z" }, - { url = "https://files.pythonhosted.org/packages/1b/b3/5d0e77ea774bd3224321c248880ea0c0379000ac5c2bb6d77609549de247/chardet-7.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:e1b98790c284ff813f18f7cf7de5f05ea2435a080030c7f1a8318f3a4f80b131", size = 944155, upload-time = "2026-04-13T21:33:21.694Z" }, - { url = "https://files.pythonhosted.org/packages/70/a8/bf0811d859e13801279a2ae64f37a408027b282f2047bc0001c75dd356ad/chardet-7.4.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d892d3dcd652fdef53e3d6327d39b17c0df40a899dfc919abaeb64c974497531", size = 872887, upload-time = "2026-04-13T21:33:23.328Z" }, - { url = "https://files.pythonhosted.org/packages/51/ac/b9d68ebddfe1b02c77af5bf81120e12b036b4432dc6af7a303d90e2bc38b/chardet-7.4.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:acc46d1b8b7d5783216afe15db56d1c179b9a40e5a1558bc13164c4fd20674c4", size = 853964, upload-time = "2026-04-13T21:33:24.724Z" }, - { url = "https://files.pythonhosted.org/packages/2a/81/17fa103ea9caf5d325a5e4051ab2ba65996fd66baa60b81ee41af1f54e10/chardet-7.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ac3bf11c645734a1701a3804e43eabd98851838192267d08c353a834ab79fea", size = 876006, upload-time = "2026-04-13T21:33:26.098Z" }, - { url = "https://files.pythonhosted.org/packages/c2/20/193faab46a68ea550587331a698c3dca8099f8901d10937c4443135c7ed9/chardet-7.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e3bd9f936e04bae89c254262af08d9e5b98f805175ba1e29d454e6cba3107b7", size = 887680, upload-time = "2026-04-13T21:33:27.49Z" }, - { url = "https://files.pythonhosted.org/packages/40/c6/94a3c673327392652ee8bdea9a45bc8a5f5365197a7387d68f0eed007115/chardet-7.4.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:27cc23da03630cdecc9aa81a895aa86629c211f995cd57651f0fbc280717bf93", size = 879865, upload-time = "2026-04-13T21:33:29.052Z" }, - { url = "https://files.pythonhosted.org/packages/b1/2c/cad8b5e3623a987f3c930b68e2bdd06cfc388cd91cd42ed05f1227701b73/chardet-7.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:b95c934b9ad59e2ba8abb9be49df70d3ad1b0d95d864b9fdb7588d4fa8bd921c", size = 939594, upload-time = "2026-04-13T21:33:31.391Z" }, - { url = "https://files.pythonhosted.org/packages/33/e0/d06e42fd6f02a58e5e227e5106587751cb38adcff0aaf949add744b78b6e/chardet-7.4.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c77867f0c1cb8bd819502249fcdc500364aedb07881e11b743726fa2148e7b6e", size = 889714, upload-time = "2026-04-13T21:33:32.772Z" }, - { url = "https://files.pythonhosted.org/packages/d4/ed/40d091954d48abea037baae6be8fb79905e5f78d34d12ea955132c7d8011/chardet-7.4.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cf1efeaf65a6ef2f5b9cc3a1df6f08ba2831b369ccaa4c7018eaf90aa757bb11", size = 872319, upload-time = "2026-04-13T21:33:34.427Z" }, - { url = "https://files.pythonhosted.org/packages/bb/77/82a46821dbfbdfe062710d2bf2ede13426304e3567a23c57d919c0c31630/chardet-7.4.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f3504c139a2ad544077dd2d9e412cd08b01786843d76997cd43bb6de311723c", size = 892021, upload-time = "2026-04-13T21:33:35.766Z" }, - { url = "https://files.pythonhosted.org/packages/49/57/42d30c562bda5b4a839766c1aad8d5856b798ad2a1c3247b72a679afec94/chardet-7.4.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457f619882ba66327d4d8d14c6c342269bdb1e4e1c38e8117df941d14d351b04", size = 902509, upload-time = "2026-04-13T21:33:37.096Z" }, - { url = "https://files.pythonhosted.org/packages/8c/6c/0a40afdb50a0fe041ab95553b835a8160b6cf0e81edf2ae2fe9f5224cbf9/chardet-7.4.3-py3-none-any.whl", hash = "sha256:1173b74051570cf08099d7429d92e4882d375ad4217f92a6e5240ccfb26f231e", size = 626562, upload-time = "2026-04-13T21:33:38.559Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d", size = 315182, upload-time = "2026-04-02T09:25:40.673Z" }, - { url = "https://files.pythonhosted.org/packages/24/47/b192933e94b546f1b1fe4df9cc1f84fcdbf2359f8d1081d46dd029b50207/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8", size = 209329, upload-time = "2026-04-02T09:25:42.354Z" }, - { url = "https://files.pythonhosted.org/packages/c2/b4/01fa81c5ca6141024d89a8fc15968002b71da7f825dd14113207113fabbd/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790", size = 231230, upload-time = "2026-04-02T09:25:44.281Z" }, - { url = "https://files.pythonhosted.org/packages/20/f7/7b991776844dfa058017e600e6e55ff01984a063290ca5622c0b63162f68/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc", size = 225890, upload-time = "2026-04-02T09:25:45.475Z" }, - { url = "https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393", size = 216930, upload-time = "2026-04-02T09:25:46.58Z" }, - { url = "https://files.pythonhosted.org/packages/e2/ab/b18f0ab31cdd7b3ddb8bb76c4a414aeb8160c9810fdf1bc62f269a539d87/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153", size = 202109, upload-time = "2026-04-02T09:25:48.031Z" }, - { url = "https://files.pythonhosted.org/packages/82/e5/7e9440768a06dfb3075936490cb82dbf0ee20a133bf0dd8551fa096914ec/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af", size = 214684, upload-time = "2026-04-02T09:25:49.245Z" }, - { url = "https://files.pythonhosted.org/packages/71/94/8c61d8da9f062fdf457c80acfa25060ec22bf1d34bbeaca4350f13bcfd07/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34", size = 212785, upload-time = "2026-04-02T09:25:50.671Z" }, - { url = "https://files.pythonhosted.org/packages/66/cd/6e9889c648e72c0ab2e5967528bb83508f354d706637bc7097190c874e13/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1", size = 203055, upload-time = "2026-04-02T09:25:51.802Z" }, - { url = "https://files.pythonhosted.org/packages/92/2e/7a951d6a08aefb7eb8e1b54cdfb580b1365afdd9dd484dc4bee9e5d8f258/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752", size = 232502, upload-time = "2026-04-02T09:25:53.388Z" }, - { url = "https://files.pythonhosted.org/packages/58/d5/abcf2d83bf8e0a1286df55cd0dc1d49af0da4282aa77e986df343e7de124/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53", size = 214295, upload-time = "2026-04-02T09:25:54.765Z" }, - { url = "https://files.pythonhosted.org/packages/47/3a/7d4cd7ed54be99973a0dc176032cba5cb1f258082c31fa6df35cff46acfc/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616", size = 227145, upload-time = "2026-04-02T09:25:55.904Z" }, - { url = "https://files.pythonhosted.org/packages/1d/98/3a45bf8247889cf28262ebd3d0872edff11565b2a1e3064ccb132db3fbb0/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a", size = 218884, upload-time = "2026-04-02T09:25:57.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/80/2e8b7f8915ed5c9ef13aa828d82738e33888c485b65ebf744d615040c7ea/charset_normalizer-3.4.7-cp310-cp310-win32.whl", hash = "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374", size = 148343, upload-time = "2026-04-02T09:25:58.199Z" }, - { url = "https://files.pythonhosted.org/packages/35/1b/3b8c8c77184af465ee9ad88b5aea46ea6b2e1f7b9dc9502891e37af21e30/charset_normalizer-3.4.7-cp310-cp310-win_amd64.whl", hash = "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943", size = 159174, upload-time = "2026-04-02T09:25:59.322Z" }, - { url = "https://files.pythonhosted.org/packages/be/c1/feb40dca40dbb21e0a908801782d9288c64fc8d8e562c2098e9994c8c21b/charset_normalizer-3.4.7-cp310-cp310-win_arm64.whl", hash = "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008", size = 147805, upload-time = "2026-04-02T09:26:00.756Z" }, - { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, - { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, - { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, - { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742, upload-time = "2026-04-02T09:26:06.36Z" }, - { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061, upload-time = "2026-04-02T09:26:08.347Z" }, - { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239, upload-time = "2026-04-02T09:26:09.823Z" }, - { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173, upload-time = "2026-04-02T09:26:10.953Z" }, - { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841, upload-time = "2026-04-02T09:26:12.142Z" }, - { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304, upload-time = "2026-04-02T09:26:13.711Z" }, - { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455, upload-time = "2026-04-02T09:26:14.941Z" }, - { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036, upload-time = "2026-04-02T09:26:16.478Z" }, - { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739, upload-time = "2026-04-02T09:26:17.751Z" }, - { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277, upload-time = "2026-04-02T09:26:18.981Z" }, - { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819, upload-time = "2026-04-02T09:26:20.295Z" }, - { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281, upload-time = "2026-04-02T09:26:21.74Z" }, - { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843, upload-time = "2026-04-02T09:26:22.901Z" }, - { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, - { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, - { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, - { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, - { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, - { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, - { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, - { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, - { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, - { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, - { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, - { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, - { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, - { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, - { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, - { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, - { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, - { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, - { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, - { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, - { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, - { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, - { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, - { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, - { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, - { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, - { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, - { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, - { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, - { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, - { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, - { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, - { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, - { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, - { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, - { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, - { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, - { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, - { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, - { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, - { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, - { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, - { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, - { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, - { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, - { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, - { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, - { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, - { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, - { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, - { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, - { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, - { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, - { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, - { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, - { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, - { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, - { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, -] - -[[package]] -name = "click" -version = "8.4.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, -] - -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, -] - -[[package]] -name = "coverage" -version = "7.14.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/fd/0ab2772530e946e1be1abd0bc09e647ec9b02e88f0867857601fefca8953/coverage-7.14.1.tar.gz", hash = "sha256:30c08f7d90415aa98b3c990385dea2939b0da55f38515e5b369b83655f8523be", size = 920132, upload-time = "2026-05-26T20:41:36.783Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/69/0d2ef01ff4b8fcecd4cba920d11e92fa4f96ae412441d3b56a90a258e69b/coverage-7.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3e3680291c4a1d0dadfa84a2c459576a4af5133abb617905714339a0c73138cf", size = 219722, upload-time = "2026-05-26T20:38:14.002Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ae/9afdeaa31b9d9ce98124b6abf8bb49119bf71aecae04f8567c189d91299f/coverage-7.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a5274669f37f2343635a347b91a60777621341ab3378e9c6ac9335eee704bddf", size = 220240, upload-time = "2026-05-26T20:38:17.424Z" }, - { url = "https://files.pythonhosted.org/packages/51/69/c998589871df7ea7dba865cc5ee32b5a3e1d47ba6c68ef91104c7c46fa5e/coverage-7.14.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cfe5a5fec635799ef33428f1e5e61bafa45a92a96190ba731561ba558ccc214d", size = 246981, upload-time = "2026-05-26T20:38:19.266Z" }, - { url = "https://files.pythonhosted.org/packages/fc/10/1c7d04c13040dac531d21b712bbe08f902e6dd9b58f5d77875c4d030f8f2/coverage-7.14.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:62a9f70b52e0b5a95cfef4a5c5641b06983cadc5e538a3feeb5c00211f523ac2", size = 248812, upload-time = "2026-05-26T20:38:20.75Z" }, - { url = "https://files.pythonhosted.org/packages/c1/65/2a38a4607ef27cadcfbcee034dba5830ae2569f90144a0f4c7dbf47d30b0/coverage-7.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c18ebc343e15be53049b3a2dce38fe82d58f37e20ab9094b3a39c0aa4f6bb47", size = 250675, upload-time = "2026-05-26T20:38:22.159Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a2/a446ed9752a4a59b79e0fb6cbb319f6facb2183045c0725462625e66f87e/coverage-7.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b84ffdf877644e7096aa936991efeed873f7f3df57b9cd001312b7668ab08550", size = 252590, upload-time = "2026-05-26T20:38:23.63Z" }, - { url = "https://files.pythonhosted.org/packages/9e/fd/e81fbd7ba752365546e9842b1cbdaad3d6919d2a522c590aef16a281ec5e/coverage-7.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e854312c4103f2ad4c0dc023b69b77ebfd2c89db5f86c4c94dc2353f9a92167e", size = 247691, upload-time = "2026-05-26T20:38:25.057Z" }, - { url = "https://files.pythonhosted.org/packages/53/35/f3c26fdaae9ea937d154ca4d372e5ea0a4167ff70d36c6074ac2eacb2f83/coverage-7.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c643734307300234fafa36bf2a040a7235f8f177ea1fd6ec1423aea6fb7b929f", size = 248716, upload-time = "2026-05-26T20:38:26.406Z" }, - { url = "https://files.pythonhosted.org/packages/2e/14/940b6c49551fd343e8507ee2b0ba7af5d0aa04ed5bf768285cb7c72a9884/coverage-7.14.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:84ac9499e48700399a5dd0ea7085b5091961fec52c68d66b4ec0d3cf7f4441b1", size = 246721, upload-time = "2026-05-26T20:38:28.282Z" }, - { url = "https://files.pythonhosted.org/packages/aa/2c/40fc0634186c28292a662dff578866b3913983d6c375a3c2a74020938719/coverage-7.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:7f02d09f70776579b926d889a4c9c235070a1f47c40458aeaca563fae5acfdb5", size = 250533, upload-time = "2026-05-26T20:38:29.753Z" }, - { url = "https://files.pythonhosted.org/packages/de/e3/2c26bf1e811f9df991ff2a9bdddebdd13ee0665d564df7d05979f9146297/coverage-7.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:ce66d8e46da2bb5ee313a745cbd2e391d319176c1f7a9451bfcd3a2fb920859b", size = 246990, upload-time = "2026-05-26T20:38:31.516Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b0/060260ef56bd92363ebdce0c7095ce422b06e69aae71828efeca473ab1ca/coverage-7.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c912c259304cfb5ee584481cfb7ce1ff932b4d61e6c9140b8f19cb7b5ed82332", size = 247593, upload-time = "2026-05-26T20:38:33.065Z" }, - { url = "https://files.pythonhosted.org/packages/63/f3/501502046efeb0d6d94b5ca54941d95f1184183dd6bdb7f283985783bb4a/coverage-7.14.1-cp310-cp310-win32.whl", hash = "sha256:1238cb94638e610e972c60dac68e813f868dc7d6e982535270558443058d9d59", size = 222330, upload-time = "2026-05-26T20:38:35.36Z" }, - { url = "https://files.pythonhosted.org/packages/a0/5d/1bf99f2c558f128faf7906817ccbdb576ba815d3b41ce2ac1719b70a3663/coverage-7.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:fc459e5d73be2d6332fcfe8dbf3d8994671fe33c700f4565988ecfa511547253", size = 223261, upload-time = "2026-05-26T20:38:37.196Z" }, - { url = "https://files.pythonhosted.org/packages/7d/d7/477ad149490e6cb849f28abea1dabb9c823cea72e7500c81b4240ce619c0/coverage-7.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:478b5bcd63c2e1357c5c7e16c070690df7b07f676b1c114d7b93e533c664309f", size = 219848, upload-time = "2026-05-26T20:38:38.715Z" }, - { url = "https://files.pythonhosted.org/packages/91/82/a5eb47257c50601bb7b9a9d2857c67b7a3a85ad74180eb2c98bb1fbe0ce5/coverage-7.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a24a81f9715ee42ef59a316cc11611c98fe23920f7c81861315c9f3ff4a230f4", size = 220354, upload-time = "2026-05-26T20:38:40.232Z" }, - { url = "https://files.pythonhosted.org/packages/43/8b/78419b5391a5cb706b6544390507e469d83ffc9a8248b02c4011aceb9365/coverage-7.14.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:196a13319ad88d6d8ef5ab489ec4f44ddde2143c0c7d5b27786f6c3ffd56a7e1", size = 250771, upload-time = "2026-05-26T20:38:41.782Z" }, - { url = "https://files.pythonhosted.org/packages/77/63/e77aaacd491182210d639636b7a8bba23ffffa9b82aa3762da9431855fa9/coverage-7.14.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d452fd08b5c72c5167c93e6867b5c08500bd40f2a21e1e854a500550b6cc36f", size = 252683, upload-time = "2026-05-26T20:38:43.305Z" }, - { url = "https://files.pythonhosted.org/packages/65/1c/a022e3cfbec2ac241640003cb3a817e161d9c7f5aa9b49173756cdc03204/coverage-7.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23bf7fa51ac02e07fc7c96849b82946da47ae862dc8f86d183b2a4864fc38129", size = 254791, upload-time = "2026-05-26T20:38:45.361Z" }, - { url = "https://files.pythonhosted.org/packages/61/d6/967e408aca4c1ceb88cb0cc677169110ae7f5995fb5eaf5fb1f5a1bb8f5d/coverage-7.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcaa50684dcaadfa599ac48f81103c756d791cfd85c97203d2217c593d48b860", size = 256748, upload-time = "2026-05-26T20:38:46.91Z" }, - { url = "https://files.pythonhosted.org/packages/b8/be/869188f7fe28638078ec479331ace6dc5f7b40b7153eb616f47ab79404d8/coverage-7.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4ea1c034f95c9b056e856b794630b17f9fa3d57e4800ff1e503d3be0f9c9078c", size = 250907, upload-time = "2026-05-26T20:38:48.493Z" }, - { url = "https://files.pythonhosted.org/packages/07/aa/adb7d3b4278d690e68703abcd76ab1b948242e3668d921711551b78f9ddb/coverage-7.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7e057326434e441306226fbeb5d1aaf14a2637efe97ba668306635835f32ad7", size = 252483, upload-time = "2026-05-26T20:38:50.074Z" }, - { url = "https://files.pythonhosted.org/packages/43/61/331c74103c62dcb0c4b9b3a0de9a61aca016208b0a90f109592a9f9ecc28/coverage-7.14.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:59baf88468dbc8d63b1887afd92bda52e40bb1561696e5819670601403810cec", size = 250545, upload-time = "2026-05-26T20:38:51.613Z" }, - { url = "https://files.pythonhosted.org/packages/f6/b6/c5dae3c104d89be04828f61810e6b3473825482e4c288cc4ed04553e08ae/coverage-7.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d34d75f892b3ab73ba11cab5442cce7b3e168fd64162b16f0e1e0d09c508edef", size = 254310, upload-time = "2026-05-26T20:38:53.503Z" }, - { url = "https://files.pythonhosted.org/packages/ad/a1/2b9d5863e3b83c01ad8199e3c597802fbb3a9dc90b058885804c20296d31/coverage-7.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3a56abc20a472baf0304c455721bc601477440d28ecfde8a03dde79ede07e0df", size = 250266, upload-time = "2026-05-26T20:38:55.414Z" }, - { url = "https://files.pythonhosted.org/packages/7f/5e/0e511fbdb269359be26fe678a1c3fa1f2aa2a01573cc3f54268c8d6d4797/coverage-7.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6a3cb83d1552c0cd1b4906655b6a33fd4a8473229633a901c6b73bf86914dee9", size = 251174, upload-time = "2026-05-26T20:38:57.141Z" }, - { url = "https://files.pythonhosted.org/packages/85/10/e55307b622b3dd9671cb321824502dc10f93e72f2802b9946159a8edadeb/coverage-7.14.1-cp311-cp311-win32.whl", hash = "sha256:10274a1fbeb8ec5d72966e17bb198a3104257aca4ac09d98667c5f8aca8c8548", size = 222354, upload-time = "2026-05-26T20:38:58.727Z" }, - { url = "https://files.pythonhosted.org/packages/71/cf/107421693cfb71e4f1ca5bf70443f64d4161878068d07a3e51c7ad21d17b/coverage-7.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:87ebdf787d4888e3f3f2d523eadc6e18c6d18c6d0eb173801a189641627fb37e", size = 223290, upload-time = "2026-05-26T20:39:00.413Z" }, - { url = "https://files.pythonhosted.org/packages/b8/1d/3e3644585eb29e9dafefb19555078529a4d7cce12bd21929664eea989277/coverage-7.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:dd34767fa19848d35659ffc0a75314f58c7af3f1cd87ec521e8292a1238398a3", size = 221953, upload-time = "2026-05-26T20:39:02.159Z" }, - { url = "https://files.pythonhosted.org/packages/3d/b7/bdbb725ba02c5b42825b200c940f38b7a54fcad24627b7192f78f8110d76/coverage-7.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a06c76364a9360e33d6d23769aefdf7f66f38e2ffb60ceb1baaa4989d83b695c", size = 220022, upload-time = "2026-05-26T20:39:03.702Z" }, - { url = "https://files.pythonhosted.org/packages/72/81/fdc0898a55c6219223291ec1a1fe89966ef212ce82276aa0899df84b5de0/coverage-7.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fad54e871165f6ec2f536063ac74c3104508a12963e64072ba44bd822de52b0c", size = 220379, upload-time = "2026-05-26T20:39:05.381Z" }, - { url = "https://files.pythonhosted.org/packages/de/72/de048c4a25e13bce59ac6a339351c10bdf2515e07459afcdaf04dc3143a2/coverage-7.14.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:84b535f00655ecafe1d929d1fb00ed5d6fa3051ea643ab2c161a3887b86f294b", size = 251888, upload-time = "2026-05-26T20:39:07.367Z" }, - { url = "https://files.pythonhosted.org/packages/28/30/300c343f68beb9d4cbb64ec81e58c5b6b80b56927f72d2b38654ac26e013/coverage-7.14.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6b6b0853b895fe0e98cbfc580d1ec3393d9302b4b1e96a77b3f5c91fdab899e6", size = 254624, upload-time = "2026-05-26T20:39:09.037Z" }, - { url = "https://files.pythonhosted.org/packages/b1/ed/7b25642496e8170b6bac14adce00537c6e5fa2d586159401a4de3e8b49e6/coverage-7.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:442cc9c952b2df400cda54bb04ab87330cf2cd08a8692cbbea36773531eb6f37", size = 255739, upload-time = "2026-05-26T20:39:10.889Z" }, - { url = "https://files.pythonhosted.org/packages/7f/a2/abd210b8c4e29c24e4624916db97bb519097a91034aaeb767f937e7da794/coverage-7.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8270544c361ed405a27a060dbc9ed2c124b084d96dfdc2d9a2510482aef981ad", size = 257998, upload-time = "2026-05-26T20:39:12.722Z" }, - { url = "https://files.pythonhosted.org/packages/7f/24/7c50beed3792fe62f6ce0545c6686ce83379719e2c0276179333d97eae92/coverage-7.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:48b283b1dd6372e8de2a7a9a4c4d5dc06f4d4fd209b876f3c88a7a205a0c8f84", size = 252296, upload-time = "2026-05-26T20:39:14.259Z" }, - { url = "https://files.pythonhosted.org/packages/15/05/0f874628ebcbfc77ead559ff210281ef06a97db08481832e7dd39274a135/coverage-7.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5b0c99ba93a07d56f6df340bb79be53202a082b2fdb81bfe6190b741a3470d54", size = 253658, upload-time = "2026-05-26T20:39:15.923Z" }, - { url = "https://files.pythonhosted.org/packages/99/6f/ca6ad067364b337ef997802115e7ecad2abd2248b05471464b0dea02b4d4/coverage-7.14.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e471bc5769ff073b058cfadb0d736b56ce067c8560eabeb0da88462df98c23e7", size = 251803, upload-time = "2026-05-26T20:39:17.537Z" }, - { url = "https://files.pythonhosted.org/packages/c0/30/b9b4d377cd9f40baf228068f5a81faf8450c6228503011bd499708483a50/coverage-7.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f497a1ea81d4cd7c10ddcaa685135b9aabd291af3d55775a9ddf3cb7a364cdd9", size = 255873, upload-time = "2026-05-26T20:39:19.414Z" }, - { url = "https://files.pythonhosted.org/packages/3c/21/7c721a9e5e6bb88547d30a787aefb97512d3f54c1324c7488d9b3743f7f9/coverage-7.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2222be86d0b54f5dd5a38f45f17f315f737245e857bf0bdedc70734f84a13c02", size = 251372, upload-time = "2026-05-26T20:39:21.169Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/f8ae5a2200130e1503cd7661a6cd3b2b7bacef98277fbf3571fb13f8b766/coverage-7.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:85e85586565842f6932abebd4c18bcb1074223dc0b3576e7d173ca710622813a", size = 253245, upload-time = "2026-05-26T20:39:23.097Z" }, - { url = "https://files.pythonhosted.org/packages/34/62/70a9024672a5f6910517d9628c52c9afbdd3cf8f46426af52bb148a56fff/coverage-7.14.1-cp312-cp312-win32.whl", hash = "sha256:4a28fd227808366b196a75476dced2eb35b351d6766ba9c858dc93319e87f4f1", size = 222567, upload-time = "2026-05-26T20:39:24.868Z" }, - { url = "https://files.pythonhosted.org/packages/f6/81/8b7cd386839b039ebe1855733b9f9449a8dec5d79564018234f185a7fa70/coverage-7.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:54acdb6674a4661768d7bf7db32dfb9f46ab1d764f8aba6df75ce1a6a088724e", size = 223372, upload-time = "2026-05-26T20:39:26.603Z" }, - { url = "https://files.pythonhosted.org/packages/ae/ba/b44d472022f620d289d95fa830143235c0c36461c6f2437ea8d51e5481ed/coverage-7.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:99cd41ff91afd94896fea3bc002706b6ae4ce95727d06e4a0f39c0a8d8bd8b1a", size = 221989, upload-time = "2026-05-26T20:39:28.242Z" }, - { url = "https://files.pythonhosted.org/packages/8a/9e/5f6d56327c62b185225d145191c607e07515294a0aa6338e58805cd4a5ac/coverage-7.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:be9f2c802dcfce3f71298303aa5dad0dce440a76c52f2f60dacd8656dab78793", size = 220044, upload-time = "2026-05-26T20:39:29.902Z" }, - { url = "https://files.pythonhosted.org/packages/75/92/e82aca356744cbbc0f77a0b623e38918c1872361963413a3bab5d0340393/coverage-7.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6223a72fd0e4c7156353ec0f08a5f93623e1d3034d0e2683b9bb8ea674131b1d", size = 220412, upload-time = "2026-05-26T20:39:31.561Z" }, - { url = "https://files.pythonhosted.org/packages/27/c9/385bde0bf7ed0f4bf3a7ee5367060a86b5d218718cfd6fb943c0f836b34f/coverage-7.14.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7279d2110a28cebc738b6459ecda2771735a4c18465fbbd36b3288fe5ed92247", size = 251412, upload-time = "2026-05-26T20:39:33.337Z" }, - { url = "https://files.pythonhosted.org/packages/51/8c/23faf6a2343a0d17f960a4bd56c43bc7eb4cf312f774dd6ceebd82c7d8fc/coverage-7.14.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9eeb3fcbc13ba40dfbdb22d01d196a28e9cef9ed4c29b60061a1e0e823a9929d", size = 254008, upload-time = "2026-05-26T20:39:35.009Z" }, - { url = "https://files.pythonhosted.org/packages/42/06/36f4aa9ca8a815e6036156e80706a67828bb97bd826948244f6996dda957/coverage-7.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f0cfc27c539f07cf5c0a4cfe211d0b6cae039f8f40526dbaa71944e64b50a7b", size = 255241, upload-time = "2026-05-26T20:39:36.71Z" }, - { url = "https://files.pythonhosted.org/packages/ca/79/95266316352f90f6b1c6736bb413302edfde2453fb32422d3911642691b3/coverage-7.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:221c70f316241a78e77e607c227cefc8808d4e08f28d99c04f35694690e940be", size = 257373, upload-time = "2026-05-26T20:39:38.412Z" }, - { url = "https://files.pythonhosted.org/packages/e3/9c/58316d1f66c488b5fca8a0eb3e98348807813efa8a0d0833b9021be27488/coverage-7.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da028256b04ec30e5e0114b6f76172938c313991f0a2d3d894271315cf5d5e43", size = 251635, upload-time = "2026-05-26T20:39:40.268Z" }, - { url = "https://files.pythonhosted.org/packages/ef/5a/ca2398a568e16fed7bb713e84ba3603a7164fb65779abe645c565ec890d5/coverage-7.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76a085d7005236a767e3426148b2c407e53ad61695c562f8a81da2d373324901", size = 253373, upload-time = "2026-05-26T20:39:42.145Z" }, - { url = "https://files.pythonhosted.org/packages/6e/2c/0396562c32deaebe7be51d865b3a41e9a87d7561acafe1a28f53b07e019a/coverage-7.14.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b553d04b5e778a8e56d57eb134aff42a92718ecba45e79c4764ecfa40efd92ff", size = 251341, upload-time = "2026-05-26T20:39:43.907Z" }, - { url = "https://files.pythonhosted.org/packages/fd/8f/a94f9221184c9cae1ee115820e3798e48b6b17777a9f19e46fb9a0c8dc74/coverage-7.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:46f714d2fb8ae2f4f29f23ada7f1e79b759fff5a70f94a1dac23af204c3ec9e4", size = 255497, upload-time = "2026-05-26T20:39:46.166Z" }, - { url = "https://files.pythonhosted.org/packages/71/69/505d70e47db1eaebcd002c39759707621ef184cd6b1ae084d9f41293f323/coverage-7.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1896f5e19ff3f0431c7ce2172adc54890fd97f86b59ced8ca1649145d9ffe35d", size = 251159, upload-time = "2026-05-26T20:39:48.03Z" }, - { url = "https://files.pythonhosted.org/packages/e0/aa/58681c383aa33a9d2ed40a02d7a22fbf780d1fa4d575396365777828198c/coverage-7.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:62fd185ef9df3c33d1c8178c5af105f762afbad96038de9a4ae100aa6297ca33", size = 252934, upload-time = "2026-05-26T20:39:49.872Z" }, - { url = "https://files.pythonhosted.org/packages/eb/fd/11c928cd6bdffc7074bb5965c173d9ebf517fb00205e1da524b98d29ef92/coverage-7.14.1-cp313-cp313-win32.whl", hash = "sha256:ab4af6352741a604c431c6072fce5bee33bf0f20dc7a56618d6bf6bb89e9810c", size = 222584, upload-time = "2026-05-26T20:39:51.68Z" }, - { url = "https://files.pythonhosted.org/packages/6f/92/fb416fc26d340dcba19518c418d6048e913186e17243982c5e435e41fa7a/coverage-7.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:7af486dabe8954d03b087f0021540897afe084f04e16ff5579e08cc46f871416", size = 223394, upload-time = "2026-05-26T20:39:53.472Z" }, - { url = "https://files.pythonhosted.org/packages/73/c6/02d56e3867972f77d5036de924643f26c056e848f00452cafb4dbc3c29b4/coverage-7.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:2224f89ffd0c5605ccce1ed7a584da162bc7c55f601ab1c946bc9de31a486b42", size = 222015, upload-time = "2026-05-26T20:39:55.374Z" }, - { url = "https://files.pythonhosted.org/packages/4d/9e/fcc77914050df73f7662fa1f00902774c79c075a8388ab334074574bf77e/coverage-7.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de286598cc65d2b489411174b1faec2f5a7775fb3201fd925db2a76b4030f37d", size = 220733, upload-time = "2026-05-26T20:39:57.189Z" }, - { url = "https://files.pythonhosted.org/packages/f7/67/2963cbdaf5cbadec44efa3a1e39eaa1f02df4079585f05387607a221e126/coverage-7.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:042c46ded7c288aeb07cf14a28b6c1e10b78fcba40171c3fa1e939377eeef0b5", size = 221086, upload-time = "2026-05-26T20:39:59.019Z" }, - { url = "https://files.pythonhosted.org/packages/c8/c5/8701645574e11881f2f47d8930f98bc48b5d43b25eb5b4430dfc4a2f9f48/coverage-7.14.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4ddbe407477f04c45115d1a4e5bc480f753553b534d338d4c3358b1cdd0ea52", size = 262381, upload-time = "2026-05-26T20:40:00.822Z" }, - { url = "https://files.pythonhosted.org/packages/7c/28/7a64d73598263e0c5abd5084211a8474488d31b3c552ff531c719dfcff62/coverage-7.14.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d13e6725992e2d2fd7d81d4f5241952d13740121dfd501da09201be39b2c003a", size = 264458, upload-time = "2026-05-26T20:40:02.506Z" }, - { url = "https://files.pythonhosted.org/packages/fa/d8/4969179db9f7eb4df218e69540adf829d1c835f59452513d065d15446802/coverage-7.14.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f747dc8edcfe740130f28f32f3995e955494285717e86ee25af51db2219df08a", size = 266884, upload-time = "2026-05-26T20:40:04.421Z" }, - { url = "https://files.pythonhosted.org/packages/a6/78/a45d5794dbc9bafd97afc96a4377c86c7820d78b6cf51b89bc1d4e919275/coverage-7.14.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced2f09ef276fd58611a1ef502164ad266d2b75174e5a40cabbdb4033f9f6cf2", size = 268022, upload-time = "2026-05-26T20:40:06.298Z" }, - { url = "https://files.pythonhosted.org/packages/21/cb/4f5e354e9e3e67af96bd4e57113e6db6b22298c7168b13eec408a549903d/coverage-7.14.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b84800013769a78ccb9ef4659402e26d06867e337b61ec365f77ad008adea80e", size = 261631, upload-time = "2026-05-26T20:40:08.226Z" }, - { url = "https://files.pythonhosted.org/packages/ec/49/eced49af4cb996d5d8b7e94e736175c513e4facd3398507b89892b4326d8/coverage-7.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ea8cd6ca0ee9f616aaef3afc6882e32c2cbf18b00d96313ffd76af650574034d", size = 264443, upload-time = "2026-05-26T20:40:10.137Z" }, - { url = "https://files.pythonhosted.org/packages/f1/d8/5603a88a7c5913a6b54f6cb1a8c46f7b39cbb30f27cd3f492908da09b2d7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:aa5e304a873fabddc11e484e9b6b738bd38bd7bed17b09aa84eecf5332e8b8bb", size = 262069, upload-time = "2026-05-26T20:40:11.999Z" }, - { url = "https://files.pythonhosted.org/packages/f0/59/2ae3cb79da554a06c8619d6c88ea19dd1e4aed4b834b6a83bb1fa243bdc5/coverage-7.14.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5a1c5215be81035e629d5bc756650634d0bf31991038db7a0eccb90f025ce16d", size = 265780, upload-time = "2026-05-26T20:40:13.858Z" }, - { url = "https://files.pythonhosted.org/packages/af/5f/b130c1dc999031f2648bd25317fbce505ad8d5562079b4ed81e736a84967/coverage-7.14.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:79058c47dae6788504b5effb319961bcd72d7240551464b91d474bc0ed186d69", size = 260970, upload-time = "2026-05-26T20:40:16.142Z" }, - { url = "https://files.pythonhosted.org/packages/87/d1/ec13ccddeb48ec963bdfa72a11224bac2584bd045ba13beca82f8113e9c7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:370c5afae3fa0658e11694a32b24c2778f6bc2d17718121f94ee185e69f26b54", size = 263157, upload-time = "2026-05-26T20:40:18.382Z" }, - { url = "https://files.pythonhosted.org/packages/cf/c2/cd91ead503045161092d3845f7bb95ea2f25131ce96d3e314dd835d91b9c/coverage-7.14.1-cp313-cp313t-win32.whl", hash = "sha256:3758dd0a7f1fa57365ef2e781df0f0731d38b6e3772259d13dae4bd8a958d4b1", size = 223259, upload-time = "2026-05-26T20:40:20.381Z" }, - { url = "https://files.pythonhosted.org/packages/71/9f/1e28d97e6bd2c76b07f38b7c02870f1371255ff6717f54eca578fcbbdd0e/coverage-7.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:6ff665fb023a77386fe11685190cee1f60a7d635994a30d9b0a061533d470fce", size = 224320, upload-time = "2026-05-26T20:40:22.316Z" }, - { url = "https://files.pythonhosted.org/packages/a9/e0/d936e908f0e1efa55e52b91e01b52f1055cef5e1ab2718493390ed8e2fb8/coverage-7.14.1-cp313-cp313t-win_arm64.whl", hash = "sha256:17a5a241e5997621a956a7f402a7433ef4221e5152809b785bec79e2323799f1", size = 222577, upload-time = "2026-05-26T20:40:24.894Z" }, - { url = "https://files.pythonhosted.org/packages/d6/34/fc2f101b151af3799a101f0550b0454aa008afdc0add677394ec4aa8ea10/coverage-7.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d5ed429d0b8edaac649e889b4ffcedb6c80b06629a3f93050e3dddfb99235bee", size = 220091, upload-time = "2026-05-26T20:40:27.249Z" }, - { url = "https://files.pythonhosted.org/packages/3d/a7/1ebae2ab5b961b5c79bb09fe7b3ac99edb190d8be4a8c510b2cf66f46468/coverage-7.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8011224a62280e50dab346960c03cf47aca1a1e09e608c0fb33fd6e0cc8e9500", size = 220421, upload-time = "2026-05-26T20:40:30.084Z" }, - { url = "https://files.pythonhosted.org/packages/5e/90/92aca9cf0acc95123c96cd1eb1f08917897a7f5dee01e15738922971ec31/coverage-7.14.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:12c42ec1e14f553c4f817e989365982e646e27211f10a0f717855b94a79c8906", size = 251466, upload-time = "2026-05-26T20:40:32.542Z" }, - { url = "https://files.pythonhosted.org/packages/26/2b/78048cbe3b999f6cbf9cc0d90abba6a88a3e0863a8c1c6cbc762f3f8802f/coverage-7.14.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:06144cd511cf2624873a035c5069cf297144f6e77a73ee3d7a55b605ec5efb42", size = 253973, upload-time = "2026-05-26T20:40:34.473Z" }, - { url = "https://files.pythonhosted.org/packages/8e/21/c2e33b29d1cfde484a19d437afc343c6cd30b08d78cbbf9f5aff14e57b2b/coverage-7.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a311d8e1da24be5c1ccf85cbfb06315dbaa1703d5a1eab3f6432c72b837917c8", size = 255318, upload-time = "2026-05-26T20:40:38.154Z" }, - { url = "https://files.pythonhosted.org/packages/8e/ee/aad2f108d63b769121005302f16bf66db8625c88ceaba466942e09a2607e/coverage-7.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c79cead5b5bc584d9c71451cb984d0e3a84e0c0937379c8efcbf27c8d661b851", size = 257633, upload-time = "2026-05-26T20:40:40.164Z" }, - { url = "https://files.pythonhosted.org/packages/c2/f8/11a2c29b4fd76d9849f81d0bb812ec0017a9396df3217214e38934a8c837/coverage-7.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dcbf65f1f66a26cdd88c35cf68fb4729c5d1cd2e88added72420541dfb212034", size = 251488, upload-time = "2026-05-26T20:40:42.631Z" }, - { url = "https://files.pythonhosted.org/packages/c9/b8/9a5820de4b8ac2b71d85e3b5fb49108d7469c665f0e2ad0dd7569023e305/coverage-7.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fd86572566fb40189a8260446158235159bc7a82dfbc87a3b39cf4fb57fcec1c", size = 253329, upload-time = "2026-05-26T20:40:45.208Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ff/f33e4823667e27548e8fd8df44217515303f9808d0ff29817db56f87d990/coverage-7.14.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:7771b601718fdde84832c3a434ca9bbf4ae9adbc49d84198b4110700c3c77c36", size = 251291, upload-time = "2026-05-26T20:40:47.502Z" }, - { url = "https://files.pythonhosted.org/packages/68/9b/489db0ebb209054766b90a9014a45f6d26eb724c02ec21311c3733b5a644/coverage-7.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:39b21e212c55af06fa375e3dbf90a8a8e38792f3a910c580066d23563830ddd5", size = 255564, upload-time = "2026-05-26T20:40:49.372Z" }, - { url = "https://files.pythonhosted.org/packages/27/b5/16bc2d4c2409b23c7737edb68c83bc89e345f378050549fe1d75ac7d34d5/coverage-7.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f2302660e32562a532b442480121aef8aa61a5bdb20b30bf0adab29f10a5a4b4", size = 251107, upload-time = "2026-05-26T20:40:51.677Z" }, - { url = "https://files.pythonhosted.org/packages/7d/0c/2629997469a00cd069d588a41c9dc887610f2775ae89d250c4791e65272a/coverage-7.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03a6f93c1ec3b7f2e77b5dbcc5573a2c21f12529a5c6bbe0f16f72303cc2fa4d", size = 252764, upload-time = "2026-05-26T20:40:54.267Z" }, - { url = "https://files.pythonhosted.org/packages/d2/ee/f78d63c8f079e0d7211c7e2401fa17e311514534ba61bae03e4b287ce4ab/coverage-7.14.1-cp314-cp314-win32.whl", hash = "sha256:8a3ce026d73290f42f08dafecbd82c193a74df280461fbf97300fec51fd133ee", size = 222837, upload-time = "2026-05-26T20:40:56.496Z" }, - { url = "https://files.pythonhosted.org/packages/dc/b9/be539854f93a70dfbeec69117f33ec70dc42ff0b65b5b07ab8d40d04228e/coverage-7.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:114c95ef29302423b87d159075805f4ab973254a2638a5d7d046c94887cc87d7", size = 223650, upload-time = "2026-05-26T20:40:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/fe/9e/24e2842fef40f35ac82ba3a7719c8023d011bf3bf652d0675316a9d088a1/coverage-7.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:a07891c3f4805442b31b71e84ba3cf29ed1aa9a428284e06deeb4b23e5b46343", size = 222218, upload-time = "2026-05-26T20:41:00.321Z" }, - { url = "https://files.pythonhosted.org/packages/0a/1d/ac0a9df5fe31c1e8bdd658074905fc12844a05c1a7e3fdb8417e97c31e23/coverage-7.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1101a5ebb083aecb625ebb6209d4105b58f647b093cb2dc8122d7b33f743cfe1", size = 220822, upload-time = "2026-05-26T20:41:02.281Z" }, - { url = "https://files.pythonhosted.org/packages/32/cf/f964fd9aff20323f9f1a726c97135f8a76bcd87b92dad141a456a43f3c64/coverage-7.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:851b9e1e4e8a4608e77c79714b2e77c0970d2ed7202a05e92ae407817481887b", size = 221084, upload-time = "2026-05-26T20:41:04.593Z" }, - { url = "https://files.pythonhosted.org/packages/d8/5e/7e5ef2aba844de2b80d678619fcf0841b42e3f37f16411226f3fe4c1016f/coverage-7.14.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d5b89cdfb2ee051b71e8c3c70bd81a9eff81100f736a269136fe1a68efe00474", size = 262454, upload-time = "2026-05-26T20:41:06.641Z" }, - { url = "https://files.pythonhosted.org/packages/64/62/75809bded87015cc4935524218a2a8ed8dd1a8498bfed30a2f4f7a4b4d34/coverage-7.14.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0177614a0370f227888b4e436a7c55686d6a9f90eb1ade2b624ba685a1686e86", size = 264578, upload-time = "2026-05-26T20:41:08.556Z" }, - { url = "https://files.pythonhosted.org/packages/f3/42/d33392dc14633525012d2d504fa1a33b05538bf535f5c1d64675e5754b78/coverage-7.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d69af5dea2de76fc485a83032a630523f985198b7e25be901ec60181587b01e", size = 266981, upload-time = "2026-05-26T20:41:10.824Z" }, - { url = "https://files.pythonhosted.org/packages/2a/49/0157c4428c2aca7f1e09d5565930586fd5ae36f1655f08b0daa7cf1fcae1/coverage-7.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35ab22d91de736e8966b980dc355cbcdd2c6dbbcfe275f9a2991bc8a91b3df65", size = 268112, upload-time = "2026-05-26T20:41:12.966Z" }, - { url = "https://files.pythonhosted.org/packages/96/26/86b9ce71f4092b1ed325ce1421698081df1286b833400b6836912834d6e0/coverage-7.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:357d4e32935c36588aaba057d734fa32428c360c9fc2e4442afbf1b646beee6e", size = 261558, upload-time = "2026-05-26T20:41:15Z" }, - { url = "https://files.pythonhosted.org/packages/20/4c/c311210c5472cf5401d8422b0d7812cdd520f24417673afabda6c323faca/coverage-7.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:51bd64741cc6fa065abd300ede1afe5a5291ece9c31da8b24884deda48bcc3f8", size = 264447, upload-time = "2026-05-26T20:41:17.369Z" }, - { url = "https://files.pythonhosted.org/packages/fb/71/59513f8710ed3e6b0ac0a050a5b7e977bb9c9e880354863b5d00d8809256/coverage-7.14.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:9132cd363a68a4c3daa7c8704a654b1e39d3360f6f5b8ddd470608a945236c07", size = 262048, upload-time = "2026-05-26T20:41:19.309Z" }, - { url = "https://files.pythonhosted.org/packages/84/8d/bceed32dc494f5bbf50f775cd2e78ca814953942b5ea28d3c1c3ac316f14/coverage-7.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07c6290b1697b862c0478eab545eec949a0d0e4d6d03497f446d706da3b4f2de", size = 265781, upload-time = "2026-05-26T20:41:21.559Z" }, - { url = "https://files.pythonhosted.org/packages/e7/c5/9348fe40dbfd4991aaf78df2c6c3098bfb2cc834d1fd362a64b4efef855a/coverage-7.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5ea0c297e27133853b4d8a3eb799bff5a2dbd9f2f41537a240d337ac9b4df890", size = 260896, upload-time = "2026-05-26T20:41:23.428Z" }, - { url = "https://files.pythonhosted.org/packages/ca/92/1ea0f03929da7cf87206b1fa24f4c8e9c158be0455481af29ec0a1f3503f/coverage-7.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:01b7733daad0237daa01ef80fe2dfceffc911e6a17fa7b55d14aa8214eaaaecd", size = 263214, upload-time = "2026-05-26T20:41:25.419Z" }, - { url = "https://files.pythonhosted.org/packages/f6/a9/b2493c054c0e01a643266742ab45e15744e60743f9260cd930c7142b1124/coverage-7.14.1-cp314-cp314t-win32.whl", hash = "sha256:6adc5a36984624a70bf11d7184e20fa0a49aa7c47ffab43804106a1a695ea22e", size = 223624, upload-time = "2026-05-26T20:41:27.795Z" }, - { url = "https://files.pythonhosted.org/packages/fc/bd/3e1e6a57fccd2d7c83fcdf338e93ba98eb85c6e877dd34731ac585375490/coverage-7.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ddf799247318f34dbcd2efa8c95a8d0642674e926bb1774cf9b63dfd2a389d1c", size = 224728, upload-time = "2026-05-26T20:41:30.098Z" }, - { url = "https://files.pythonhosted.org/packages/bb/d7/31066cf1d2f0c6c797fce911bcfa01dd35642dc6da992a950256097c5860/coverage-7.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:145986fe66647eb489f18d9a997567a3fd358584c4b5a808769113abc07466af", size = 222752, upload-time = "2026-05-26T20:41:32.123Z" }, - { url = "https://files.pythonhosted.org/packages/8a/3c/1a983b9a745d7f83d53f057bcc5bf79ba6a2bbc08266b3f0c7d6fe630c9b/coverage-7.14.1-py3-none-any.whl", hash = "sha256:a252f21c27e38347e60111a3266b03827422a7d5525951aceee313aa68bab1d2", size = 211815, upload-time = "2026-05-26T20:41:34.078Z" }, -] - -[package.optional-dependencies] -toml = [ - { name = "tomli", marker = "python_full_version <= '3.11'" }, -] - -[[package]] -name = "cryptography" -version = "48.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984, upload-time = "2026-05-04T22:59:38.133Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587, upload-time = "2026-05-04T22:57:36.803Z" }, - { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433, upload-time = "2026-05-04T22:57:40.373Z" }, - { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620, upload-time = "2026-05-04T22:57:42.935Z" }, - { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283, upload-time = "2026-05-04T22:57:45.294Z" }, - { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573, upload-time = "2026-05-04T22:57:47.933Z" }, - { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677, upload-time = "2026-05-04T22:57:50.067Z" }, - { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808, upload-time = "2026-05-04T22:57:52.301Z" }, - { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941, upload-time = "2026-05-04T22:57:54.603Z" }, - { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579, upload-time = "2026-05-04T22:57:57.207Z" }, - { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326, upload-time = "2026-05-04T22:57:59.535Z" }, - { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672, upload-time = "2026-05-04T22:58:01.996Z" }, - { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574, upload-time = "2026-05-04T22:58:03.968Z" }, - { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868, upload-time = "2026-05-04T22:58:06.467Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107, upload-time = "2026-05-04T22:58:08.845Z" }, - { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", size = 7986556, upload-time = "2026-05-04T22:58:11.172Z" }, - { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", size = 4684776, upload-time = "2026-05-04T22:58:13.712Z" }, - { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", size = 4698121, upload-time = "2026-05-04T22:58:16.448Z" }, - { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", size = 4690042, upload-time = "2026-05-04T22:58:18.544Z" }, - { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", size = 5282526, upload-time = "2026-05-04T22:58:20.75Z" }, - { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", size = 4733116, upload-time = "2026-05-04T22:58:23.627Z" }, - { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", size = 4316030, upload-time = "2026-05-04T22:58:25.581Z" }, - { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", size = 4689640, upload-time = "2026-05-04T22:58:27.747Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", size = 5237657, upload-time = "2026-05-04T22:58:29.848Z" }, - { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", size = 4732362, upload-time = "2026-05-04T22:58:32.009Z" }, - { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", size = 4819580, upload-time = "2026-05-04T22:58:34.254Z" }, - { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", size = 4963283, upload-time = "2026-05-04T22:58:36.376Z" }, - { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", size = 3270954, upload-time = "2026-05-04T22:58:38.791Z" }, - { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", size = 3797313, upload-time = "2026-05-04T22:58:40.746Z" }, - { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482, upload-time = "2026-05-04T22:58:43.769Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266, upload-time = "2026-05-04T22:58:46.064Z" }, - { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228, upload-time = "2026-05-04T22:58:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097, upload-time = "2026-05-04T22:58:50.9Z" }, - { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582, upload-time = "2026-05-04T22:58:53.017Z" }, - { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479, upload-time = "2026-05-04T22:58:55.611Z" }, - { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481, upload-time = "2026-05-04T22:58:57.607Z" }, - { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713, upload-time = "2026-05-04T22:59:00.077Z" }, - { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165, upload-time = "2026-05-04T22:59:02.317Z" }, - { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947, upload-time = "2026-05-04T22:59:05.255Z" }, - { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059, upload-time = "2026-05-04T22:59:07.802Z" }, - { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575, upload-time = "2026-05-04T22:59:09.851Z" }, - { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117, upload-time = "2026-05-04T22:59:12.019Z" }, - { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100, upload-time = "2026-05-04T22:59:14.884Z" }, - { url = "https://files.pythonhosted.org/packages/be/d2/024b5e06be9d44cb021fb0e1a03d34d63989cf56a0fe62f3dfbab695b9b4/cryptography-48.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:84cf79f0dc8b36ac5da873481716e87aef31fcfa0444f9e1d8b4b2cece142855", size = 3950391, upload-time = "2026-05-04T22:59:17.415Z" }, - { url = "https://files.pythonhosted.org/packages/bc/17/3861e17c56fa0fd37491a14a8673fdb77c57fc5693cafe745ea8b06dba75/cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:fdfef35d751d510fcef5252703621574364fec16418c4a1e5e1055248401054b", size = 4637126, upload-time = "2026-05-04T22:59:20.197Z" }, - { url = "https://files.pythonhosted.org/packages/f0/0a/7e226dbff530f21480727eb764973a7bff2b912f8e15cd4f129e71b56d1d/cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0890f502ddf7d9c6426129c3f49f5c0a39278ed7cd6322c8755ffca6ee675a13", size = 4667270, upload-time = "2026-05-04T22:59:22.647Z" }, - { url = "https://files.pythonhosted.org/packages/3b/f2/5a72274ca9f1b2a8b44a662ee0bf1b435909deb473d6f97bcd035bcdbc71/cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:ecde28a596bead48b0cfd2a1b4416c3d43074c2d785e3a398d7ec1fc4d0f7fbb", size = 4636797, upload-time = "2026-05-04T22:59:24.912Z" }, - { url = "https://files.pythonhosted.org/packages/b4/e1/48cedb2fe63626e91ded1edad159e2a4fb8b6906c4425eb7749673077ce7/cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:4defde8685ae324a9eb9d818717e93b4638ef67070ac9bc15b8ca85f63048355", size = 4666800, upload-time = "2026-05-04T22:59:27.474Z" }, - { url = "https://files.pythonhosted.org/packages/a2/ca/7e8365deec19afb2b2c7be7c1c0aa8f99633b54e90c570999acda93260fc/cryptography-48.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:db63bf618e5dea46c07de12e900fe1cdd2541e6dc9dbae772a70b7d4d4765f6a", size = 3739536, upload-time = "2026-05-04T22:59:29.61Z" }, -] - -[[package]] -name = "cyclopts" -version = "4.16.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "docstring-parser" }, - { name = "rich" }, - { name = "rich-rst" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/07/bf61d13de86d96a4c46aff00c9ca0eced44bcc8c3e16280605c1253e5720/cyclopts-4.16.1.tar.gz", hash = "sha256:8aa47bf92a5fb33abca5af05e576eecdb0d2f79893ad29238046df78370fc4a8", size = 181196, upload-time = "2026-05-25T15:29:08.518Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/8d/7f362c2fb8ef4decd2160bc24d4292c6ca658cc6d9a161b89ca5122bbdbf/cyclopts-4.16.1-py3-none-any.whl", hash = "sha256:617795392c4113a2c2cc7af716f20244900e87f23daa05442d1268d81472a592", size = 219020, upload-time = "2026-05-25T15:29:09.646Z" }, -] - -[[package]] -name = "distlib" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/86/b2/d6fc3f2347f43dada79e5ff118493e8109c98400a0e29a1d5264a3aa479b/distlib-0.4.1.tar.gz", hash = "sha256:c3804d0d2d4b5fcd44036eb860cb6660485fcdf5c2aba53dc324d805837ea65b", size = 610526, upload-time = "2026-06-02T11:17:40.691Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/25/18/3497c4fa83a76dcb154923fd2075522e8dd6995ecee4093c00ae18160046/distlib-0.4.1-py2.py3-none-any.whl", hash = "sha256:9c2c552c68cbadc619f2d0ed3a69e27c351a3f4c9baa9ffb7df9e9cdc3d19a97", size = 469216, upload-time = "2026-06-02T11:17:38.779Z" }, -] - -[[package]] -name = "dnspython" -version = "2.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, -] - -[[package]] -name = "docstring-parser" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, -] - -[[package]] -name = "email-validator" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dnspython" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, -] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, -] - -[[package]] -name = "fastmcp" -version = "3.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "fastmcp-slim", extra = ["client", "server"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/29/18/46beaec18c9f86a599ae3f9cdf6677dd6b50240cfd844d18233710b47f13/fastmcp-3.4.2.tar.gz", hash = "sha256:b468722946fc467c3796a6572f7a14d93d48c014cf8fea12910245220cbbe4e1", size = 28756849, upload-time = "2026-06-06T01:30:35.694Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/4d/8b1ba42251160e11ca34686344572121432c23a082d56ef6bbdec5888fc1/fastmcp-3.4.2-py3-none-any.whl", hash = "sha256:c87a62b029f0c5400ada85f683629345d2466c39169f0cb853e487b2f7308c08", size = 8018, upload-time = "2026-06-06T01:30:38.118Z" }, -] - -[[package]] -name = "fastmcp-slim" -version = "3.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "platformdirs" }, - { name = "pydantic", extra = ["email"] }, - { name = "pydantic-settings" }, - { name = "python-dotenv" }, - { name = "rich" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a3/2e/d627b28b7403ecc526991ef732921b08bde010006e6148635f053fd29f4c/fastmcp_slim-3.4.2.tar.gz", hash = "sha256:290646e0955a516235a317151034559aa48336cb843d3f006131aedad8759bb4", size = 576291, upload-time = "2026-06-06T01:30:12.553Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/58/22afebf18df7260b09148199cbeb90cdcc4b3a4e1b5d7460e3591c3a7add/fastmcp_slim-3.4.2-py3-none-any.whl", hash = "sha256:bdc72492212681ca502755fa8acc0457f559295da1fc3dfc0599adc1c04b82f3", size = 749195, upload-time = "2026-06-06T01:30:11.22Z" }, -] - -[package.optional-dependencies] -client = [ - { name = "authlib" }, - { name = "exceptiongroup" }, - { name = "httpx" }, - { name = "mcp" }, - { name = "opentelemetry-api" }, - { name = "py-key-value-aio", extra = ["filetree", "keyring", "memory"] }, - { name = "starlette" }, -] -server = [ - { name = "authlib" }, - { name = "cyclopts" }, - { name = "exceptiongroup" }, - { name = "griffelib" }, - { name = "httpx" }, - { name = "joserfc" }, - { name = "jsonref" }, - { name = "jsonschema-path" }, - { name = "mcp" }, - { name = "openapi-pydantic" }, - { name = "opentelemetry-api" }, - { name = "packaging" }, - { name = "py-key-value-aio", extra = ["filetree", "keyring", "memory"] }, - { name = "pyperclip" }, - { name = "python-multipart" }, - { name = "pyyaml" }, - { name = "starlette" }, - { name = "uncalled-for" }, - { name = "uvicorn" }, - { name = "watchfiles" }, - { name = "websockets" }, -] - -[[package]] -name = "filelock" -version = "3.29.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/f9/f38573ed5844586db374d085911740a501ccfa373b455fc9413f09f85237/filelock-3.29.1.tar.gz", hash = "sha256:d97e6b1b9757569626c58caa07dc4beb1613f4a2938b1e8cc81afca398906c9e", size = 59335, upload-time = "2026-06-03T15:19:04.053Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/a0/614c5fe402fd88951df45f4dda2fa3b4e17a99ecd92340771929169b3b95/filelock-3.29.1-py3-none-any.whl", hash = "sha256:85199dfd706869641b72b2e8955d5416a4b2b7dc4b0e8e6d97b4cc1299a6983b", size = 40750, upload-time = "2026-06-03T15:19:02.959Z" }, -] - -[[package]] -name = "flake8" -version = "7.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mccabe" }, - { name = "pycodestyle" }, - { name = "pyflakes" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9b/af/fbfe3c4b5a657d79e5c47a2827a362f9e1b763336a52f926126aa6dc7123/flake8-7.3.0.tar.gz", hash = "sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872", size = 48326, upload-time = "2025-06-20T19:31:35.838Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/56/13ab06b4f93ca7cac71078fbe37fcea175d3216f31f85c3168a6bbd0bb9a/flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e", size = 57922, upload-time = "2025-06-20T19:31:34.425Z" }, -] - -[[package]] -name = "flake8-bugbear" -version = "25.11.29" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "flake8" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ec/20/2a996e2fca7810bd1b031901d65fc4292630895afcb946ebd00568bdc669/flake8_bugbear-25.11.29.tar.gz", hash = "sha256:b5d06710f3d26e595541ad303ad4d5cb52578bd4bccbb2c2c0b2c72e243dafc8", size = 84896, upload-time = "2025-11-29T20:51:57.75Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/42/c18f199780d99a6f6a64c4a36f4ad28a445d9e11968a6025b21d0c8b6802/flake8_bugbear-25.11.29-py3-none-any.whl", hash = "sha256:9bf15e2970e736d2340da4c0a70493db964061c9c38f708cfe1f7b2d87392298", size = 37861, upload-time = "2025-11-29T20:51:56.439Z" }, -] - -[[package]] -name = "flake8-colors" -version = "0.1.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "flake8" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/41/9b/d9431c1daca8cc0cd660e1a07272f896a38565207032985a600b9e08d492/flake8-colors-0.1.9.tar.gz", hash = "sha256:35a5483a7d156d0438b402faea2fefe45b411571ce5dc93ba28670fd9429cc46", size = 2882, upload-time = "2020-11-16T09:38:40.417Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/1c/9773d449a82b32d3e1677d2e081b1dadde514b749f62bea35bebd6c6a343/flake8_colors-0.1.9-py3-none-any.whl", hash = "sha256:e80ed1839dc151730adc51207e632823aa1f393d6db32897ffd0e60dceecfd9f", size = 3981, upload-time = "2020-11-16T09:38:39.087Z" }, -] - -[[package]] -name = "flake8-isort" -version = "7.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "flake8" }, - { name = "isort" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/57/a4/4b170983fd2e9b5ecc40c88738db6dfb77e069c71be9a81f9893cdb8a0cc/flake8_isort-7.0.0.tar.gz", hash = "sha256:a677199d1197f826eb69084e7ac272f208f4583363285f43111c34272abe7e5d", size = 17796, upload-time = "2025-10-25T13:31:08.768Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/7d/907ef4135f6ede5187930d9ddd1f36564e07c6cdcd15ae8fb9849c9517e0/flake8_isort-7.0.0-py3-none-any.whl", hash = "sha256:c301a0e55fc77582348e636194b84b1a0baf0dfdaa6eddf3b0eeea75f8be7f36", size = 18383, upload-time = "2025-10-25T13:31:06.914Z" }, -] - -[[package]] -name = "flake8-plugin-utils" -version = "1.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/51/14/53727a2bc5bbda1e1f7e266e0e2d2718e5eb6c943a1e8cc2b33e5af002e0/flake8-plugin-utils-1.3.3.tar.gz", hash = "sha256:39f6f338d038b301c6fd344b06f2e81e382b68fa03c0560dff0d9b1791a11a2c", size = 10459, upload-time = "2023-06-26T16:42:20.946Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/da/a7/23c012c9becaacb24e9ba8e359e48db5f96982d505e38a3e7003902c5b9f/flake8_plugin_utils-1.3.3-py3-none-any.whl", hash = "sha256:e4848c57d9d50f19100c2d75fa794b72df068666a9041b4b0409be923356a3ed", size = 9664, upload-time = "2023-06-26T16:42:23.939Z" }, -] - -[[package]] -name = "flake8-pyproject" -version = "1.2.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "flake8" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/6a/cdee9ff7f2b7c6ddc219fd95b7c70c0a3d9f0367a506e9793eedfc72e337/flake8_pyproject-1.2.4-py3-none-any.whl", hash = "sha256:ea34c057f9a9329c76d98723bb2bb498cc6ba8ff9872c4d19932d48c91249a77", size = 5694, upload-time = "2025-11-28T21:40:01.309Z" }, -] - -[[package]] -name = "flake8-pytest-style" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "flake8-plugin-utils" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4a/18/b5115b91db2eca4d287d578f2f97c5e99e713ce4cb747774076f5a42fe1c/flake8_pytest_style-2.2.0.tar.gz", hash = "sha256:d23a33294bccfb9f1b11aaf5212256727b299b9c9b17cf21e230c52c1095a468", size = 17759, upload-time = "2025-10-20T07:52:25.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/de/36523c4defc0d02f1617de66e23cd18ad74b869d401269bba6d790abc95e/flake8_pytest_style-2.2.0-py3-none-any.whl", hash = "sha256:d01c4198a6c4e0ab759a92a0fa7710f10d83ec28e32a50ab6fb2e10f973a2f36", size = 22370, upload-time = "2025-10-20T07:52:23.82Z" }, -] - -[[package]] -name = "flake8-quotes" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "flake8" }, - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dd/57/a173e3eb86072b7ee77650aca496b15d6886367d257f58ea9de5276e330a/flake8-quotes-3.4.0.tar.gz", hash = "sha256:aad8492fb710a2d3eabe68c5f86a1428de650c8484127e14c43d0504ba30276c", size = 14107, upload-time = "2024-02-10T21:58:22.357Z" } - -[[package]] -name = "flake8-typing-imports" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "flake8" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/96/9d16adf5752cbf4ad8e47957c36a13749451aa60221ad350510bdfa966a0/flake8_typing_imports-1.17.0.tar.gz", hash = "sha256:ac7a328eca24ad5662bab58994cd1440412a5ca768cdf2d0705fbac767243ab2", size = 7509, upload-time = "2025-10-09T19:20:37.878Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/94/ab8f9ba92f7434c9195ac3ad77726536e79b156410bc0308542a713d1608/flake8_typing_imports-1.17.0-py2.py3-none-any.whl", hash = "sha256:4ec816ce0f4772ffd0b812a0bfd22883f699ad88dc945a67694246ca2ad50e19", size = 7820, upload-time = "2025-10-09T19:20:36.913Z" }, -] - -[[package]] -name = "google-api-core" -version = "2.31.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "googleapis-common-protos" }, - { name = "proto-plus" }, - { name = "protobuf" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c6/22/155cadf1d49272a9cf48f3168c0f3874fa13397297e611a5ea00cd093880/google_api_core-2.31.0.tar.gz", hash = "sha256:2be84ee0f584c48e6bde1b36766e23348b361fb7e55e56135fc76ce1c397f9c2", size = 176492, upload-time = "2026-06-03T14:52:17.257Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/40/9bdbb60b03a332bd45acb8703da08bbc27d991d35286b62e42acc86d243a/google_api_core-2.31.0-py3-none-any.whl", hash = "sha256:ef79fb3784c71cbac89cbd03301ba0c8fb8ad2aa95d7f9204dd9628f7adf59ab", size = 173102, upload-time = "2026-06-03T14:51:26.729Z" }, -] - -[[package]] -name = "google-auth" -version = "2.43.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cachetools" }, - { name = "pyasn1-modules" }, - { name = "rsa" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ff/ef/66d14cf0e01b08d2d51ffc3c20410c4e134a1548fc246a6081eae585a4fe/google_auth-2.43.0.tar.gz", hash = "sha256:88228eee5fc21b62a1b5fe773ca15e67778cb07dc8363adcb4a8827b52d81483", size = 296359, upload-time = "2025-11-06T00:13:36.587Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/d1/385110a9ae86d91cc14c5282c61fe9f4dc41c0b9f7d423c6ad77038c4448/google_auth-2.43.0-py2.py3-none-any.whl", hash = "sha256:af628ba6fa493f75c7e9dbe9373d148ca9f4399b5ea29976519e0a3848eddd16", size = 223114, upload-time = "2025-11-06T00:13:35.209Z" }, -] - -[[package]] -name = "google-cloud-core" -version = "2.6.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a8/dd/1eef226e470369b26824a505c34482c0b493bc35fe8e0c6b003b5feca21a/google_cloud_core-2.6.0.tar.gz", hash = "sha256:e76149739f90fac1fc6757c09f47eaccb3145b54adbd7759b0f7c4b235f46c83", size = 36001, upload-time = "2026-05-07T08:04:04.124Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/4a/98da8930ab109c73d9a5d13782a9ebb81ea8c111f6d534a567b71d23e52b/google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e", size = 29390, upload-time = "2026-05-07T08:02:34.672Z" }, -] - -[[package]] -name = "google-cloud-storage" -version = "3.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-crc32c" }, - { name = "google-resumable-media" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/98/c0c6d10f893509585c755a6567689e914df3501ae269f46b0d67d7e7c70a/google_cloud_storage-3.5.0.tar.gz", hash = "sha256:10b89e1d1693114b3e0ca921bdd28c5418701fd092e39081bb77e5cee0851ab7", size = 17242207, upload-time = "2025-11-05T12:41:02.715Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/81/a567236070e7fe79a17a11b118d7f5ce4adefe2edd18caf1824d7e29a30a/google_cloud_storage-3.5.0-py3-none-any.whl", hash = "sha256:e28fd6ad8764e60dbb9a398a7bc3296e7920c494bc329057d828127e5f9630d3", size = 289998, upload-time = "2025-11-05T12:41:01.212Z" }, -] - -[[package]] -name = "google-crc32c" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/ac/6f7bc93886a823ab545948c2dd48143027b2355ad1944c7cf852b338dc91/google_crc32c-1.8.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff", size = 31296, upload-time = "2025-12-16T00:19:07.261Z" }, - { url = "https://files.pythonhosted.org/packages/f7/97/a5accde175dee985311d949cfcb1249dcbb290f5ec83c994ea733311948f/google_crc32c-1.8.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288", size = 30870, upload-time = "2025-12-16T00:29:17.669Z" }, - { url = "https://files.pythonhosted.org/packages/3d/63/bec827e70b7a0d4094e7476f863c0dbd6b5f0f1f91d9c9b32b76dcdfeb4e/google_crc32c-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d", size = 33214, upload-time = "2025-12-16T00:40:19.618Z" }, - { url = "https://files.pythonhosted.org/packages/63/bc/11b70614df04c289128d782efc084b9035ef8466b3d0a8757c1b6f5cf7ac/google_crc32c-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092", size = 33589, upload-time = "2025-12-16T00:40:20.7Z" }, - { url = "https://files.pythonhosted.org/packages/3e/00/a08a4bc24f1261cc5b0f47312d8aebfbe4b53c2e6307f1b595605eed246b/google_crc32c-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733", size = 34437, upload-time = "2025-12-16T00:35:19.437Z" }, - { url = "https://files.pythonhosted.org/packages/5d/ef/21ccfaab3d5078d41efe8612e0ed0bfc9ce22475de074162a91a25f7980d/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8", size = 31298, upload-time = "2025-12-16T00:20:32.241Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b8/f8413d3f4b676136e965e764ceedec904fe38ae8de0cdc52a12d8eb1096e/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7", size = 30872, upload-time = "2025-12-16T00:33:58.785Z" }, - { url = "https://files.pythonhosted.org/packages/f6/fd/33aa4ec62b290477181c55bb1c9302c9698c58c0ce9a6ab4874abc8b0d60/google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15", size = 33243, upload-time = "2025-12-16T00:40:21.46Z" }, - { url = "https://files.pythonhosted.org/packages/71/03/4820b3bd99c9653d1a5210cb32f9ba4da9681619b4d35b6a052432df4773/google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a", size = 33608, upload-time = "2025-12-16T00:40:22.204Z" }, - { url = "https://files.pythonhosted.org/packages/7c/43/acf61476a11437bf9733fb2f70599b1ced11ec7ed9ea760fdd9a77d0c619/google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2", size = 34439, upload-time = "2025-12-16T00:35:20.458Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" }, - { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" }, - { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" }, - { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" }, - { url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" }, - { url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" }, - { url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" }, - { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, - { url = "https://files.pythonhosted.org/packages/21/88/8ecf3c2b864a490b9e7010c84fd203ec8cf3b280651106a3a74dd1b0ca72/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697", size = 31301, upload-time = "2025-12-16T00:24:48.527Z" }, - { url = "https://files.pythonhosted.org/packages/36/c6/f7ff6c11f5ca215d9f43d3629163727a272eabc356e5c9b2853df2bfe965/google_crc32c-1.8.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651", size = 30868, upload-time = "2025-12-16T00:48:12.163Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/c25671c7aad70f8179d858c55a6ae8404902abe0cdcf32a29d581792b491/google_crc32c-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2", size = 33381, upload-time = "2025-12-16T00:40:26.268Z" }, - { url = "https://files.pythonhosted.org/packages/42/fa/f50f51260d7b0ef5d4898af122d8a7ec5a84e2984f676f746445f783705f/google_crc32c-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21", size = 33734, upload-time = "2025-12-16T00:40:27.028Z" }, - { url = "https://files.pythonhosted.org/packages/08/a5/7b059810934a09fb3ccb657e0843813c1fee1183d3bc2c8041800374aa2c/google_crc32c-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2", size = 34878, upload-time = "2025-12-16T00:35:23.142Z" }, - { url = "https://files.pythonhosted.org/packages/52/c5/c171e4d8c44fec1422d801a6d2e5d7ddabd733eeda505c79730ee9607f07/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93", size = 28615, upload-time = "2025-12-16T00:40:29.298Z" }, - { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, -] - -[[package]] -name = "google-resumable-media" -version = "2.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-crc32c" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/f8/1ca5781d6be9cb9f73f7d40f4958c4bd1226a60598e3e39e1d6aaf838c4b/google_resumable_media-2.10.0.tar.gz", hash = "sha256:e324bc9d0fdae4c52a08ae90456edc4e71ece858399e1217ac0eb3a51d6bc6ee", size = 2164570, upload-time = "2026-06-03T16:14:26.103Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/d8/00c6854ac1512bb9eaf13bd3f8f28222f7674947fc510a4ff7616f2efc80/google_resumable_media-2.10.0-py3-none-any.whl", hash = "sha256:88152884bee37b2bf36a0ab81ad8c7fd12212c9803dd981d77c1b35b02d34e7c", size = 81533, upload-time = "2026-06-03T16:13:12.51Z" }, -] - -[[package]] -name = "googleapis-common-protos" -version = "1.75.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, -] - -[[package]] -name = "griffelib" -version = "2.0.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/82/74f4a3310cdabfbb10da554c3a672847f1ed33c6f61dd472681ce7f1fe67/griffelib-2.0.2.tar.gz", hash = "sha256:3cf20b3bc470e83763ffbf236e0076b1211bac1bc67de13daf494640f2de707e", size = 166461, upload-time = "2026-03-27T11:34:51.091Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/8c/c9138d881c79aa0ea9ed83cbd58d5ca75624378b38cee225dcf5c42cc91f/griffelib-2.0.2-py3-none-any.whl", hash = "sha256:925c857658fb1ba40c0772c37acbc2ab650bd794d9c1b9726922e36ea4117ea1", size = 142357, upload-time = "2026-03-27T11:34:46.275Z" }, -] - -[[package]] -name = "h11" -version = "0.16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, -] - -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - -[[package]] -name = "httpx-retries" -version = "0.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/f5/046cac13877ce9b55aebdbb3999e0e45b19b989a95c5fd1040fa04bd1f92/httpx_retries-0.5.0.tar.gz", hash = "sha256:d8c8e1e0852d84be3837aba0bcf78aeb89a4b77db95e8cc988c8c058830b3044", size = 15647, upload-time = "2026-04-20T01:21:47.154Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/a8/aadeaa9a28510727d538636ee8688f0782a98523147852b29404ce696f1b/httpx_retries-0.5.0-py3-none-any.whl", hash = "sha256:d3124592979a9dc6197e666d1f02e9ab996a0c58fce59fad8db6201a6a87304e", size = 8908, upload-time = "2026-04-20T01:21:46.157Z" }, -] - -[[package]] -name = "httpx-sse" -version = "0.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, -] - -[[package]] -name = "idna" -version = "3.18" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, -] - -[[package]] -name = "importlib-metadata" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, -] - -[[package]] -name = "iniconfig" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, -] - -[[package]] -name = "isodate" -version = "0.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, -] - -[[package]] -name = "isort" -version = "8.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ef/7c/ec4ab396d31b3b395e2e999c8f46dec78c5e29209fac49d1f4dace04041d/isort-8.0.1.tar.gz", hash = "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d", size = 769592, upload-time = "2026-02-28T10:08:20.685Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/95/c7c34aa53c16353c56d0b802fba48d5f5caa2cdee7958acbcb795c830416/isort-8.0.1-py3-none-any.whl", hash = "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75", size = 89733, upload-time = "2026-02-28T10:08:19.466Z" }, -] - -[[package]] -name = "jaraco-classes" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "more-itertools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, -] - -[[package]] -name = "jaraco-context" -version = "6.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "backports-tarfile", marker = "python_full_version < '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, -] - -[[package]] -name = "jaraco-functools" -version = "4.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "more-itertools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/36/cf/ea4ef2920830dea3f5ab2ea4da6fb67724e6dca80ee2553788c3607243d0/jaraco_functools-4.5.0.tar.gz", hash = "sha256:3bb5665ea4a020cf78a7040e89154c77edadb3ca74f366479669c5999aa70b03", size = 20272, upload-time = "2026-05-15T21:34:10.025Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/9a/982e48afcffcd727a9144506720ffd4224b6b7e355c98641866f38b7c043/jaraco_functools-4.5.0-py3-none-any.whl", hash = "sha256:79ce39246eddbde4b3a03b77ea5f0f7878dc669b166a66cf3fa8e266aa3fa2f4", size = 10594, upload-time = "2026-05-15T21:34:08.595Z" }, -] - -[[package]] -name = "jeepney" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, -] - -[[package]] -name = "jmespath" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, -] - -[[package]] -name = "joserfc" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d3/c3/2f590052b55cbdd0ace470ee7ee1f685f6882051be93a9374891005623e2/joserfc-1.7.0.tar.gz", hash = "sha256:4aced6ab0c47846f0a531402aec2419a874b91e918df9c4c9da8a82fb559d6c4", size = 232967, upload-time = "2026-06-02T09:59:34.506Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/83/b6b62a66a06ce872d9429a5eb5ee20b2002fd9c331b953c94381c1f7c9f9/joserfc-1.7.0-py3-none-any.whl", hash = "sha256:17e5d7a5a35e65442b05efc435a3d5d46696ffa2c8a2ed0eea6f63fc268e3224", size = 70387, upload-time = "2026-06-02T09:59:33.264Z" }, -] - -[[package]] -name = "json-log-formatter" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/ef/324f4a28ed0152a32b80685b26316b604218e4ac77487ea82719c3c28bc6/json_log_formatter-1.1.1.tar.gz", hash = "sha256:0815e3b4469e5c79cf3f6dc8a0613ba6601f4a7464f85ba03655cfa6e3e17d10", size = 5896, upload-time = "2025-02-27T22:56:15.643Z" } - -[[package]] -name = "jsonpath-ng" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/32/58/250751940d75c8019659e15482d548a4aa3b6ce122c515102a4bfdac50e3/jsonpath_ng-1.8.0.tar.gz", hash = "sha256:54252968134b5e549ea5b872f1df1168bd7defe1a52fed5a358c194e1943ddc3", size = 74513, upload-time = "2026-02-24T14:42:06.182Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/99/33c7d78a3fb70d545fd5411ac67a651c81602cc09c9cf0df383733f068c5/jsonpath_ng-1.8.0-py3-none-any.whl", hash = "sha256:b8dde192f8af58d646fc031fac9c99fe4d00326afc4148f1f043c601a8cfe138", size = 67844, upload-time = "2026-02-28T00:53:19.637Z" }, -] - -[[package]] -name = "jsonref" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/aa/0d/c1f3277e90ccdb50d33ed5ba1ec5b3f0a242ed8c1b1a85d3afeb68464dca/jsonref-1.1.0.tar.gz", hash = "sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552", size = 8814, upload-time = "2023-01-16T16:10:04.455Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/ec/e1db9922bceb168197a558a2b8c03a7963f1afe93517ddd3cf99f202f996/jsonref-1.1.0-py3-none-any.whl", hash = "sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9", size = 9425, upload-time = "2023-01-16T16:10:02.255Z" }, -] - -[[package]] -name = "jsonschema" -version = "4.26.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "jsonschema-specifications" }, - { name = "referencing" }, - { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "rpds-py", version = "2026.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, -] - -[[package]] -name = "jsonschema-path" -version = "0.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "pathable" }, - { name = "pyyaml" }, - { name = "referencing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/79/cd02a4df6d9270efdc7d3feefe6edd730b0820c39eeaa107a2faee8322d5/jsonschema_path-0.5.0.tar.gz", hash = "sha256:493b156ba895c97602655b620a8456caa2ce08c1aa389f5a7addec065e6e855c", size = 19597, upload-time = "2026-05-19T20:45:00.971Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/2c/9e69d73c4297508be9e3b64a970ea3971b3eb8db64ffc5802d40bd25981f/jsonschema_path-0.5.0-py3-none-any.whl", hash = "sha256:2790a070bc7abb08ea3dbe4d340ece4efadf639223001f020c7503229ba068e2", size = 24077, upload-time = "2026-05-19T20:44:59.225Z" }, -] - -[[package]] -name = "jsonschema-specifications" -version = "2025.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "referencing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, -] - -[[package]] -name = "kbcstorage" -version = "0.9.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "azure-storage-blob" }, - { name = "boto3" }, - { name = "google-auth" }, - { name = "google-cloud-storage" }, - { name = "python-dotenv" }, - { name = "requests" }, - { name = "responses" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/49/e0d0f476ba1b44c29c790f7e95514fcdf3097f10a54c53f17fcb1ddf505a/kbcstorage-0.9.5.tar.gz", hash = "sha256:3848d73f839e7bfd980bd06707ed0b4512b5b76f132ce643db8b5522b9531cf4", size = 38706, upload-time = "2026-01-15T10:53:16.046Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/5c/5aeb25d9b257a577ee8da1bb8beebffd178cf10a6120b5afb732fae13421/kbcstorage-0.9.5-py3-none-any.whl", hash = "sha256:4c3e1587969fc7e7e6b55f44d8cf0dd31698bb6e09ac1851f109df14d5a9c729", size = 26821, upload-time = "2026-01-15T10:53:15.119Z" }, -] - -[[package]] -name = "keboola-mcp-server" -version = "1.72.8" -source = { editable = "." } -dependencies = [ - { name = "cryptography" }, - { name = "fastmcp" }, - { name = "httpx" }, - { name = "httpx-retries" }, - { name = "json-log-formatter" }, - { name = "jsonpath-ng" }, - { name = "jsonschema" }, - { name = "mcp" }, - { name = "pydantic" }, - { name = "pyjwt" }, - { name = "pyyaml" }, - { name = "sqlglot" }, - { name = "toon-format" }, -] - -[package.optional-dependencies] -codestyle = [ - { name = "black" }, - { name = "flake8" }, - { name = "flake8-bugbear" }, - { name = "flake8-colors" }, - { name = "flake8-isort" }, - { name = "flake8-pyproject" }, - { name = "flake8-pytest-style" }, - { name = "flake8-quotes" }, - { name = "flake8-typing-imports" }, - { name = "isort" }, - { name = "pep8-naming" }, -] -dev = [ - { name = "tox" }, -] -integtests = [ - { name = "kbcstorage" }, - { name = "requests" }, -] -tests = [ - { name = "pytest" }, - { name = "pytest-asyncio" }, - { name = "pytest-cov" }, - { name = "pytest-datadir" }, - { name = "pytest-mock" }, - { name = "python-dateutil" }, - { name = "python-dotenv" }, -] - -[package.metadata] -requires-dist = [ - { name = "black", marker = "extra == 'codestyle'", specifier = "~=26.3" }, - { name = "cryptography", specifier = "~=48.0" }, - { name = "fastmcp", specifier = "==3.4.2" }, - { name = "flake8", marker = "extra == 'codestyle'", specifier = "~=7.3" }, - { name = "flake8-bugbear", marker = "extra == 'codestyle'", specifier = "~=25.11" }, - { name = "flake8-colors", marker = "extra == 'codestyle'", specifier = "~=0.1" }, - { name = "flake8-isort", marker = "extra == 'codestyle'", specifier = "~=7.0" }, - { name = "flake8-pyproject", marker = "extra == 'codestyle'", specifier = "~=1.2" }, - { name = "flake8-pytest-style", marker = "extra == 'codestyle'", specifier = "~=2.2" }, - { name = "flake8-quotes", marker = "extra == 'codestyle'", specifier = "~=3.4" }, - { name = "flake8-typing-imports", marker = "extra == 'codestyle'", specifier = "~=1.17" }, - { name = "httpx", specifier = "~=0.28" }, - { name = "httpx-retries", specifier = "~=0.5" }, - { name = "isort", marker = "extra == 'codestyle'", specifier = "~=8.0" }, - { name = "json-log-formatter", specifier = "~=1.1" }, - { name = "jsonpath-ng", specifier = "~=1.8" }, - { name = "jsonschema", specifier = "~=4.26" }, - { name = "kbcstorage", marker = "extra == 'integtests'", specifier = "~=0.9" }, - { name = "mcp", specifier = "==1.27.2" }, - { name = "pep8-naming", marker = "extra == 'codestyle'", specifier = "~=0.15" }, - { name = "pydantic", specifier = "~=2.13.0" }, - { name = "pyjwt", specifier = "~=2.13" }, - { name = "pytest", marker = "extra == 'tests'", specifier = "~=9.0" }, - { name = "pytest-asyncio", marker = "extra == 'tests'", specifier = "~=1.4" }, - { name = "pytest-cov", marker = "extra == 'tests'", specifier = "~=7.0" }, - { name = "pytest-datadir", marker = "extra == 'tests'", specifier = "~=1.8" }, - { name = "pytest-mock", marker = "extra == 'tests'", specifier = "~=3.15" }, - { name = "python-dateutil", marker = "extra == 'tests'", specifier = "~=2.9" }, - { name = "python-dotenv", marker = "extra == 'tests'", specifier = "~=1.2" }, - { name = "pyyaml", specifier = "~=6.0" }, - { name = "requests", marker = "extra == 'integtests'", specifier = "~=2.34" }, - { name = "sqlglot", specifier = "~=30.0" }, - { name = "toon-format", specifier = "~=0.9.0b1" }, - { name = "tox", marker = "extra == 'dev'", specifier = "~=4.35" }, -] -provides-extras = ["codestyle", "tests", "integtests", "dev"] - -[[package]] -name = "keyring" -version = "25.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-metadata", marker = "python_full_version < '3.12'" }, - { name = "jaraco-classes" }, - { name = "jaraco-context" }, - { name = "jaraco-functools" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, - { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, - { name = "secretstorage", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, -] - -[[package]] -name = "mccabe" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, -] - -[[package]] -name = "mcp" -version = "1.27.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "httpx" }, - { name = "httpx-sse" }, - { name = "jsonschema" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "python-multipart" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, - { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/27/3c/347cf965d313f5d41764e7d46bea6ffe7d9ef13b983cc429b0340962a082/mcp-1.27.2.tar.gz", hash = "sha256:8e02db104096d1c25b28e64bde29a5c32b31bc241710213e12fd4d84985bdfef", size = 621116, upload-time = "2026-05-29T17:16:04.039Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/11/252c6f971dc4f16af1d98a1c469d8ba523aab00d1bb76b4d3bc1ff32eacc/mcp-1.27.2-py3-none-any.whl", hash = "sha256:d6ff5160c6ca65d93013626efb3fc249de683c30b2d8570755ceddd490344de5", size = 220498, upload-time = "2026-05-29T17:16:02.442Z" }, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - -[[package]] -name = "more-itertools" -version = "11.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/1d/f4da6f02cdffe04d6362210b807146a26044c88d839208aec273bb0d9184/more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d", size = 145772, upload-time = "2026-05-22T14:14:29.909Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, -] - -[[package]] -name = "mypy-extensions" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, -] - -[[package]] -name = "openapi-pydantic" -version = "0.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/2e/58d83848dd1a79cb92ed8e63f6ba901ca282c5f09d04af9423ec26c56fd7/openapi_pydantic-0.5.1.tar.gz", hash = "sha256:ff6835af6bde7a459fb93eb93bb92b8749b754fc6e51b2f1590a19dc3005ee0d", size = 60892, upload-time = "2025-01-08T19:29:27.083Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/cf/03675d8bd8ecbf4445504d8071adab19f5f993676795708e36402ab38263/openapi_pydantic-0.5.1-py3-none-any.whl", hash = "sha256:a3a09ef4586f5bd760a8df7f43028b60cafb6d9f61de2acba9574766255ab146", size = 96381, upload-time = "2025-01-08T19:29:25.275Z" }, -] - -[[package]] -name = "opentelemetry-api" -version = "1.42.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b4/1c/125e1c936c0873796771b7f04f6c93b9f1bf5d424cea90fda94a99f61da8/opentelemetry_api-1.42.1.tar.gz", hash = "sha256:56c63bea9f77b62856be8c47600474acad853b2924b99b1687c4cb6297166716", size = 72296, upload-time = "2026-05-21T16:32:49.335Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/ca/9520cc1f3dfbbd03ac5903bbf55833e257bc64b1cf30fa8b0d6df374d821/opentelemetry_api-1.42.1-py3-none-any.whl", hash = "sha256:51a69edacadbc03a8950ace1c4c21099cacc538820ac2c9e36277e78cebba714", size = 61311, upload-time = "2026-05-21T16:32:28.822Z" }, -] - -[[package]] -name = "packaging" -version = "26.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, -] - -[[package]] -name = "pathable" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/f3/5a20387de9bcd0607871bfc2198ee0e15836da7baa4592ccd7f24c27c986/pathable-0.6.0.tar.gz", hash = "sha256:6404b8b82aef5ff0fd478934137128b99b12212ba35afdde5525ca4f8388ea58", size = 18970, upload-time = "2026-05-19T18:15:11.911Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/e8/6d75ffd9784bce2e93d1ae4415649427e39a53bb172d4672b2b59c6f0a7b/pathable-0.6.0-py3-none-any.whl", hash = "sha256:82c4ca6c98c502ad12e0d4e9779b6210afee93c38990988c8c5d1b49bdcdf566", size = 18983, upload-time = "2026-05-19T18:15:10.728Z" }, -] - -[[package]] -name = "pathspec" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, -] - -[[package]] -name = "pep8-naming" -version = "0.15.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "flake8" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8d/59/c32862134635ba231d45f1711035550dc38246396c27269a4cde4bfe18d2/pep8_naming-0.15.1.tar.gz", hash = "sha256:f6f4a499aba2deeda93c1f26ccc02f3da32b035c8b2db9696b730ef2c9639d29", size = 17640, upload-time = "2025-05-05T20:43:12.555Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/78/25281540f1121acaa78926f599a17ce102b8971bc20b096fa7fb6b5b59c1/pep8_naming-0.15.1-py3-none-any.whl", hash = "sha256:eb63925e7fd9e028c7f7ee7b1e413ec03d1ee5de0e627012102ee0222c273c86", size = 9561, upload-time = "2025-05-05T20:43:11.626Z" }, -] - -[[package]] -name = "platformdirs" -version = "4.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, -] - -[[package]] -name = "pluggy" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, -] - -[[package]] -name = "proto-plus" -version = "1.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c9/56/e647b0c675392d2da368da7b6f158f7368b18542fd6f7d7400a2f39de000/proto_plus-1.28.0.tar.gz", hash = "sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9", size = 57221, upload-time = "2026-05-07T08:04:50.811Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/20/b122d4626976acb81132036d2ad1bb35a1a8775fceb837ec30964622516a/proto_plus-1.28.0-py3-none-any.whl", hash = "sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8", size = 50410, upload-time = "2026-05-07T08:03:31.962Z" }, -] - -[[package]] -name = "protobuf" -version = "7.35.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/60/fd/5b1491d9e4b586d621c54f4c36b888714164b6875f8d6afa3f9072906a51/protobuf-7.35.0.tar.gz", hash = "sha256:a2efd84605f41e559f1881b0912b44099d0a2ac9bf46b3474823f10fb393b0e6", size = 458677, upload-time = "2026-05-19T23:02:29.197Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/ee/93d06e358a4aa32280b00e722d3ea0a1f25fc3cc5778d80581c9cca2c10e/protobuf-7.35.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:66be6c513931c794fa92c080ffee41671390da3d79da219cf9c0c0907f035dda", size = 433225, upload-time = "2026-05-19T23:02:19.884Z" }, - { url = "https://files.pythonhosted.org/packages/8b/39/1c76c2da93f3c507e958e0aecee2391cc44d4625de6c728bbc555195b5a8/protobuf-7.35.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:fcbe42a4ac09d3ec9c987ddfcd956afd0b15f1ff613bd8371bde9405ffd5c8e5", size = 328847, upload-time = "2026-05-19T23:02:22.3Z" }, - { url = "https://files.pythonhosted.org/packages/91/1a/39f7ce90a238c1a987a4d81ec26379e02ca0aff367de68e4a1fa474215b9/protobuf-7.35.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:4cbf5cc286130e06a6c9bbefac442431173906dfcc979712183d4adcc01b37ee", size = 344030, upload-time = "2026-05-19T23:02:23.591Z" }, - { url = "https://files.pythonhosted.org/packages/70/5b/6baf9008817964454055ff3fe65f1de0b5f1e26c80c82f7fb108b7cd4ea3/protobuf-7.35.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:6c0f98f10c8a05ea30f8993dfef2de093d27b490fdae78bb60c8343795d55011", size = 327130, upload-time = "2026-05-19T23:02:24.637Z" }, - { url = "https://files.pythonhosted.org/packages/8e/e5/e46adb0badc388bfb84877a5f9f026aff63f60e611016cf64dbe77e05446/protobuf-7.35.0-cp310-abi3-win32.whl", hash = "sha256:4c4617b83ade0e279d1d2bfe04025a1adb87f9ed657de038620dc0ff959357f6", size = 428946, upload-time = "2026-05-19T23:02:25.741Z" }, - { url = "https://files.pythonhosted.org/packages/a7/ab/547fbd9e16d879dd13c167478f8ae0a83a428008ca07a5e06acdc23ad473/protobuf-7.35.0-cp310-abi3-win_amd64.whl", hash = "sha256:f05bcadf9a2a6b8dda047007075135fb7d08c73d9177aabc067e1be46881a201", size = 439996, upload-time = "2026-05-19T23:02:26.808Z" }, - { url = "https://files.pythonhosted.org/packages/b8/ef/50433d346c56657a70d27f156c7b349ac59a068b01de4eb796e747eecc43/protobuf-7.35.0-py3-none-any.whl", hash = "sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0", size = 171659, upload-time = "2026-05-19T23:02:27.842Z" }, -] - -[[package]] -name = "py-key-value-aio" -version = "0.4.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "beartype" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/e2/d689d922894a7ecde73b6daeaf9b13dab5aae06fe6aaaf7514722644d382/py_key_value_aio-0.4.5.tar.gz", hash = "sha256:c6563a2c6abe5da5e20f4f9e875c2a9b425a2244a54fadbf46cf140a9eea45d7", size = 107547, upload-time = "2026-05-27T16:37:08.107Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/95/b8ba862968712caa12a19666175334fa979e1f198b896a430adb3bacfe87/py_key_value_aio-0.4.5-py3-none-any.whl", hash = "sha256:ab862adbcb8c72547d1c57821f22cbbb71ab86509039c96f36e914e0336c8dd7", size = 170005, upload-time = "2026-05-27T16:37:06.629Z" }, -] - -[package.optional-dependencies] -filetree = [ - { name = "aiofile", version = "3.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "aiofile", version = "3.11.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "anyio" }, -] -keyring = [ - { name = "keyring" }, -] -memory = [ - { name = "cachetools" }, -] - -[[package]] -name = "pyasn1" -version = "0.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, -] - -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, -] - -[[package]] -name = "pycodestyle" -version = "2.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/11/e0/abfd2a0d2efe47670df87f3e3a0e2edda42f055053c85361f19c0e2c1ca8/pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783", size = 39472, upload-time = "2025-06-20T18:49:48.75Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/27/a58ddaf8c588a3ef080db9d0b7e0b97215cee3a45df74f3a94dbbf5c893a/pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d", size = 31594, upload-time = "2025-06-20T18:49:47.491Z" }, -] - -[[package]] -name = "pycparser" -version = "3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, -] - -[[package]] -name = "pydantic" -version = "2.13.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, -] - -[package.optional-dependencies] -email = [ - { name = "email-validator" }, -] - -[[package]] -name = "pydantic-core" -version = "2.46.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, - { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, - { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, - { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, - { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, - { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, - { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, - { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, - { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, - { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, - { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, - { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, - { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, - { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, - { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, - { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, - { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, - { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, - { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, - { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, - { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, - { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, - { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, - { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, - { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, - { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, - { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, - { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, - { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, - { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, - { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, - { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, - { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, - { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, - { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, - { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, - { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, - { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, - { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, - { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, - { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, - { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, - { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, - { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, - { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, - { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, - { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, - { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, - { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, - { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, - { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, - { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, - { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, - { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, - { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, - { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, - { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, - { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, - { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, - { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, - { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, - { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, - { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, - { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, - { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, - { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, - { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, - { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, - { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, - { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, - { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, - { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, - { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, - { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, - { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, - { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, - { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, - { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, - { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, - { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, - { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, - { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, - { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, - { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, - { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, - { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, - { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, - { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, - { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, - { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, - { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, - { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, - { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, - { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, - { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, - { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, -] - -[[package]] -name = "pydantic-settings" -version = "2.14.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, -] - -[[package]] -name = "pyflakes" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/45/dc/fd034dc20b4b264b3d015808458391acbf9df40b1e54750ef175d39180b1/pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58", size = 64669, upload-time = "2025-06-20T18:45:27.834Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/2f/81d580a0fb83baeb066698975cb14a618bdbed7720678566f1b046a95fe8/pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f", size = 63551, upload-time = "2025-06-20T18:45:26.937Z" }, -] - -[[package]] -name = "pygments" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, -] - -[[package]] -name = "pyjwt" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, -] - -[package.optional-dependencies] -crypto = [ - { name = "cryptography" }, -] - -[[package]] -name = "pyperclip" -version = "1.11.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/52/d87eba7cb129b81563019d1679026e7a112ef76855d6159d24754dbd2a51/pyperclip-1.11.0.tar.gz", hash = "sha256:244035963e4428530d9e3a6101a1ef97209c6825edab1567beac148ccc1db1b6", size = 12185, upload-time = "2025-09-26T14:40:37.245Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/df/80/fc9d01d5ed37ba4c42ca2b55b4339ae6e200b456be3a1aaddf4a9fa99b8c/pyperclip-1.11.0-py3-none-any.whl", hash = "sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273", size = 11063, upload-time = "2025-09-26T14:40:36.069Z" }, -] - -[[package]] -name = "pyproject-api" -version = "1.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/62/62/0fe346fe380b1aafaf819c8cb195d3241bb4f355f908e6339814131a830b/pyproject_api-1.10.1.tar.gz", hash = "sha256:c2b2726bd7aa9217b6c50b621fef5b2ae5def4d55b779c9e0694c15e0a8517ba", size = 23477, upload-time = "2026-05-28T14:22:14.049Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/d7/29e1e5e882f79133631f7bcace42d23db493f616463c157a1ab614bf69dd/pyproject_api-1.10.1-py3-none-any.whl", hash = "sha256:fa9e6f66c35b5017e909825d8f2b5d5482ea699d7be809d21c03bd1f7317f36a", size = 12992, upload-time = "2026-05-28T14:22:12.711Z" }, -] - -[[package]] -name = "pytest" -version = "9.0.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "iniconfig" }, - { name = "packaging" }, - { name = "pluggy" }, - { name = "pygments" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, -] - -[[package]] -name = "pytest-asyncio" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, - { name = "pytest" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, -] - -[[package]] -name = "pytest-cov" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "coverage", extra = ["toml"] }, - { name = "pluggy" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, -] - -[[package]] -name = "pytest-datadir" -version = "1.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b4/46/db060b291999ca048edd06d6fa9ee95945d088edc38b1172c59eeb46ec45/pytest_datadir-1.8.0.tar.gz", hash = "sha256:7a15faed76cebe87cc91941dd1920a9a38eba56a09c11e9ddf1434d28a0f78eb", size = 11848, upload-time = "2025-07-30T13:52:12.518Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/7a/33895863aec26ac3bb5068a73583f935680d6ab6af2a9567d409430c3ee1/pytest_datadir-1.8.0-py3-none-any.whl", hash = "sha256:5c677bc097d907ac71ca418109adc3abe34cf0bddfe6cf78aecfbabd96a15cf0", size = 6512, upload-time = "2025-07-30T13:52:11.525Z" }, -] - -[[package]] -name = "pytest-mock" -version = "3.15.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, -] - -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - -[[package]] -name = "python-discovery" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "filelock" }, - { name = "platformdirs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a6/12/38c1a0b1e64806780c9563e3fc9f6e472251839662587cfbe9bfaf2ae10a/python_discovery-1.4.0.tar.gz", hash = "sha256:eb8bc7daad3c226c147e45bb4e970a1feb1bf4048ee178e6db59e197b8010ce3", size = 68455, upload-time = "2026-05-28T01:15:37.639Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/8d/3d316429f65029532bb1e28ff77b797d86b5ac3915bb44ca4e19aa283d43/python_discovery-1.4.0-py3-none-any.whl", hash = "sha256:26ed78d703e234879a66244c7d4114563fb13ec5cd30a2d1357e5fb4850782da", size = 33217, upload-time = "2026-05-28T01:15:36.573Z" }, -] - -[[package]] -name = "python-dotenv" -version = "1.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, -] - -[[package]] -name = "python-multipart" -version = "0.0.31" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/7e/9b35ad8f3d9ca680f7c87a88f19612fdd8da9796c4d3b46e560ac79dcc4a/python_multipart-0.0.31.tar.gz", hash = "sha256:fc631183bb13e56db3158a4909908dfb2e23565286744e798241e63750e5d680", size = 46689, upload-time = "2026-06-04T08:27:49.014Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/1e/7f7f299527a5a8ad90acd5f2f78dfa6c8495c6301a3205106ea68a84de96/python_multipart-0.0.31-py3-none-any.whl", hash = "sha256:8408153d68a9773291fc1da39a8b85a50044bddbabd2dd72e9229776b7b15e28", size = 29996, upload-time = "2026-06-04T08:27:47.804Z" }, -] - -[[package]] -name = "pytokens" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/24/f206113e05cb8ef51b3850e7ef88f20da6f4bf932190ceb48bd3da103e10/pytokens-0.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a44ed93ea23415c54f3face3b65ef2b844d96aeb3455b8a69b3df6beab6acc5", size = 161522, upload-time = "2026-01-30T01:02:50.393Z" }, - { url = "https://files.pythonhosted.org/packages/d4/e9/06a6bf1b90c2ed81a9c7d2544232fe5d2891d1cd480e8a1809ca354a8eb2/pytokens-0.4.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:add8bf86b71a5d9fb5b89f023a80b791e04fba57960aa790cc6125f7f1d39dfe", size = 246945, upload-time = "2026-01-30T01:02:52.399Z" }, - { url = "https://files.pythonhosted.org/packages/69/66/f6fb1007a4c3d8b682d5d65b7c1fb33257587a5f782647091e3408abe0b8/pytokens-0.4.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:670d286910b531c7b7e3c0b453fd8156f250adb140146d234a82219459b9640c", size = 259525, upload-time = "2026-01-30T01:02:53.737Z" }, - { url = "https://files.pythonhosted.org/packages/04/92/086f89b4d622a18418bac74ab5db7f68cf0c21cf7cc92de6c7b919d76c88/pytokens-0.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4e691d7f5186bd2842c14813f79f8884bb03f5995f0575272009982c5ac6c0f7", size = 262693, upload-time = "2026-01-30T01:02:54.871Z" }, - { url = "https://files.pythonhosted.org/packages/b4/7b/8b31c347cf94a3f900bdde750b2e9131575a61fdb620d3d3c75832262137/pytokens-0.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:27b83ad28825978742beef057bfe406ad6ed524b2d28c252c5de7b4a6dd48fa2", size = 103567, upload-time = "2026-01-30T01:02:56.414Z" }, - { url = "https://files.pythonhosted.org/packages/3d/92/790ebe03f07b57e53b10884c329b9a1a308648fc083a6d4a39a10a28c8fc/pytokens-0.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440", size = 160864, upload-time = "2026-01-30T01:02:57.882Z" }, - { url = "https://files.pythonhosted.org/packages/13/25/a4f555281d975bfdd1eba731450e2fe3a95870274da73fb12c40aeae7625/pytokens-0.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc", size = 248565, upload-time = "2026-01-30T01:02:59.912Z" }, - { url = "https://files.pythonhosted.org/packages/17/50/bc0394b4ad5b1601be22fa43652173d47e4c9efbf0044c62e9a59b747c56/pytokens-0.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d", size = 260824, upload-time = "2026-01-30T01:03:01.471Z" }, - { url = "https://files.pythonhosted.org/packages/4e/54/3e04f9d92a4be4fc6c80016bc396b923d2a6933ae94b5f557c939c460ee0/pytokens-0.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16", size = 264075, upload-time = "2026-01-30T01:03:04.143Z" }, - { url = "https://files.pythonhosted.org/packages/d1/1b/44b0326cb5470a4375f37988aea5d61b5cc52407143303015ebee94abfd6/pytokens-0.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6", size = 103323, upload-time = "2026-01-30T01:03:05.412Z" }, - { url = "https://files.pythonhosted.org/packages/41/5d/e44573011401fb82e9d51e97f1290ceb377800fb4eed650b96f4753b499c/pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083", size = 160663, upload-time = "2026-01-30T01:03:06.473Z" }, - { url = "https://files.pythonhosted.org/packages/f0/e6/5bbc3019f8e6f21d09c41f8b8654536117e5e211a85d89212d59cbdab381/pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1", size = 255626, upload-time = "2026-01-30T01:03:08.177Z" }, - { url = "https://files.pythonhosted.org/packages/bf/3c/2d5297d82286f6f3d92770289fd439956b201c0a4fc7e72efb9b2293758e/pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1", size = 269779, upload-time = "2026-01-30T01:03:09.756Z" }, - { url = "https://files.pythonhosted.org/packages/20/01/7436e9ad693cebda0551203e0bf28f7669976c60ad07d6402098208476de/pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9", size = 268076, upload-time = "2026-01-30T01:03:10.957Z" }, - { url = "https://files.pythonhosted.org/packages/2e/df/533c82a3c752ba13ae7ef238b7f8cdd272cf1475f03c63ac6cf3fcfb00b6/pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68", size = 103552, upload-time = "2026-01-30T01:03:12.066Z" }, - { url = "https://files.pythonhosted.org/packages/cb/dc/08b1a080372afda3cceb4f3c0a7ba2bde9d6a5241f1edb02a22a019ee147/pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b", size = 160720, upload-time = "2026-01-30T01:03:13.843Z" }, - { url = "https://files.pythonhosted.org/packages/64/0c/41ea22205da480837a700e395507e6a24425151dfb7ead73343d6e2d7ffe/pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f", size = 254204, upload-time = "2026-01-30T01:03:14.886Z" }, - { url = "https://files.pythonhosted.org/packages/e0/d2/afe5c7f8607018beb99971489dbb846508f1b8f351fcefc225fcf4b2adc0/pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1", size = 268423, upload-time = "2026-01-30T01:03:15.936Z" }, - { url = "https://files.pythonhosted.org/packages/68/d4/00ffdbd370410c04e9591da9220a68dc1693ef7499173eb3e30d06e05ed1/pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4", size = 266859, upload-time = "2026-01-30T01:03:17.458Z" }, - { url = "https://files.pythonhosted.org/packages/a7/c9/c3161313b4ca0c601eeefabd3d3b576edaa9afdefd32da97210700e47652/pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78", size = 103520, upload-time = "2026-01-30T01:03:18.652Z" }, - { url = "https://files.pythonhosted.org/packages/8f/a7/b470f672e6fc5fee0a01d9e75005a0e617e162381974213a945fcd274843/pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321", size = 160821, upload-time = "2026-01-30T01:03:19.684Z" }, - { url = "https://files.pythonhosted.org/packages/80/98/e83a36fe8d170c911f864bfded690d2542bfcfacb9c649d11a9e6eb9dc41/pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa", size = 254263, upload-time = "2026-01-30T01:03:20.834Z" }, - { url = "https://files.pythonhosted.org/packages/0f/95/70d7041273890f9f97a24234c00b746e8da86df462620194cef1d411ddeb/pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d", size = 268071, upload-time = "2026-01-30T01:03:21.888Z" }, - { url = "https://files.pythonhosted.org/packages/da/79/76e6d09ae19c99404656d7db9c35dfd20f2086f3eb6ecb496b5b31163bad/pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324", size = 271716, upload-time = "2026-01-30T01:03:23.633Z" }, - { url = "https://files.pythonhosted.org/packages/79/37/482e55fa1602e0a7ff012661d8c946bafdc05e480ea5a32f4f7e336d4aa9/pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9", size = 104539, upload-time = "2026-01-30T01:03:24.788Z" }, - { url = "https://files.pythonhosted.org/packages/30/e8/20e7db907c23f3d63b0be3b8a4fd1927f6da2395f5bcc7f72242bb963dfe/pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb", size = 168474, upload-time = "2026-01-30T01:03:26.428Z" }, - { url = "https://files.pythonhosted.org/packages/d6/81/88a95ee9fafdd8f5f3452107748fd04c24930d500b9aba9738f3ade642cc/pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3", size = 290473, upload-time = "2026-01-30T01:03:27.415Z" }, - { url = "https://files.pythonhosted.org/packages/cf/35/3aa899645e29b6375b4aed9f8d21df219e7c958c4c186b465e42ee0a06bf/pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975", size = 303485, upload-time = "2026-01-30T01:03:28.558Z" }, - { url = "https://files.pythonhosted.org/packages/52/a0/07907b6ff512674d9b201859f7d212298c44933633c946703a20c25e9d81/pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a", size = 306698, upload-time = "2026-01-30T01:03:29.653Z" }, - { url = "https://files.pythonhosted.org/packages/39/2a/cbbf9250020a4a8dd53ba83a46c097b69e5eb49dd14e708f496f548c6612/pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918", size = 116287, upload-time = "2026-01-30T01:03:30.912Z" }, - { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, -] - -[[package]] -name = "pywin32" -version = "312" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/1b/9cfdeac80ee45bebbbcb31f1b7b99a0d81a1c72de48d837be984e0e88b1d/pywin32-312-cp310-cp310-win32.whl", hash = "sha256:772235332b5d1024c696f11cea1ae4be7930f0a8b894bb43db14e3f435f1ff7e", size = 6361387, upload-time = "2026-06-04T07:49:14.329Z" }, - { url = "https://files.pythonhosted.org/packages/33/b1/7afc96d041d982c27bc2df6f853d43f01fd273e3d39d04be3647ddeb533d/pywin32-312-cp310-cp310-win_amd64.whl", hash = "sha256:5dbc35d2b5320dc07f25fa31269cfb767471002b17de5eb067d03da68c7cb2db", size = 6926780, upload-time = "2026-06-04T07:49:16.881Z" }, - { url = "https://files.pythonhosted.org/packages/ce/3a/4140da9ad54108e517f4a16b2d83da3033e08662144623e1239587cb7db6/pywin32-312-cp310-cp310-win_arm64.whl", hash = "sha256:3020656e34f1cf7faeb7bccd2b84653a607c6ff0c55ada85e6487d61716deabd", size = 4307203, upload-time = "2026-06-04T07:49:18.993Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f5/10a6e845a00fc5e7afd0a988b744f403d4d57162a28d160a093c4d9322f0/pywin32-312-cp311-cp311-win32.whl", hash = "sha256:17948aeadbdb091f0ced6ef0841620794e68327b94ee415571c1203594b7215c", size = 6362659, upload-time = "2026-06-04T07:49:21.349Z" }, - { url = "https://files.pythonhosted.org/packages/35/c4/dcd2d62b5944b6d5db53413a5899016ccd57ffcb7278f3f81655d25d2027/pywin32-312-cp311-cp311-win_amd64.whl", hash = "sha256:d11417d84412f859b722fad0841b3614459ed0047f7542d8362e77884f6b6e8a", size = 6928825, upload-time = "2026-06-04T07:49:23.934Z" }, - { url = "https://files.pythonhosted.org/packages/b7/56/3cbb433fe4501cdba2eb9040f56a4e1a8243faa4186b25295564d1a7a79d/pywin32-312-cp311-cp311-win_arm64.whl", hash = "sha256:b2200a054ca6d6625c4842fc56a4976a4b47f96b73dbe5538c3f813a80359f47", size = 6721875, upload-time = "2026-06-04T07:49:26.416Z" }, - { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" }, - { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" }, - { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" }, - { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" }, - { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" }, - { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" }, - { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, - { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, - { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, - { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, - { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, - { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, -] - -[[package]] -name = "pywin32-ctypes" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, -] - -[[package]] -name = "pyyaml" -version = "6.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, - { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, - { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, - { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, - { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, - { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, - { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, - { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, - { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, - { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, - { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, - { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, - { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, - { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, - { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, - { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, - { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, - { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, - { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, - { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, - { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, - { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, - { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, - { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, - { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, - { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, - { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, - { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, - { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, - { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, - { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, - { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, - { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, - { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, - { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, - { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, - { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, - { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, - { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, - { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, - { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, - { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, - { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, - { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, - { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, - { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, - { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, -] - -[[package]] -name = "referencing" -version = "0.37.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "rpds-py", version = "2026.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, -] - -[[package]] -name = "requests" -version = "2.34.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, -] - -[[package]] -name = "responses" -version = "0.26.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, - { name = "requests" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c2/58/1fb6de3503428196df78638f991ec8095274f1ee9723e272ee4d9ff0092b/responses-0.26.1.tar.gz", hash = "sha256:2eb3218553cc8f79b57d257bac23af5e1bf381f5b9390b1767816f0843e01dc2", size = 83088, upload-time = "2026-05-21T19:56:39.747Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/31/6a620b4427d546b9e7cca8b3b8c5f0559d9cef2bb9eedcda7f73c1473c19/responses-0.26.1-py3-none-any.whl", hash = "sha256:8aacc4586eb08fb2208ef64a9eb4258d9b0c6e6f4260845f2f018ab847495345", size = 35502, upload-time = "2026-05-21T19:56:38.046Z" }, -] - -[[package]] -name = "rich" -version = "15.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, -] - -[[package]] -name = "rich-rst" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pygments" }, - { name = "rich" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/57/56/3191bae66b08ccc637ea8120426068bcb361cc323c96404c310886937067/rich_rst-2.0.1.tar.gz", hash = "sha256:cbe236ed0901d1ec8427cc6a50bf0a34353ba28ad014dc24def68bfe7f3b9e68", size = 300570, upload-time = "2026-05-16T00:47:57.362Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/3d/55c17d3ebdf3cd81356002afe5bef9bb8af631db2819785b6eac845b925b/rich_rst-2.0.1-py3-none-any.whl", hash = "sha256:7ee15f345ce25fa02b582c272a6cdbaf0c21243e38061cea273cff659bf3ef61", size = 272922, upload-time = "2026-05-16T00:47:55.508Z" }, -] - -[[package]] -name = "rpds-py" -version = "0.30.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.11'", -] -sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/0c/0c411a0ec64ccb6d104dcabe0e713e05e153a9a2c3c2bd2b32ce412166fe/rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288", size = 370490, upload-time = "2025-11-30T20:21:33.256Z" }, - { url = "https://files.pythonhosted.org/packages/19/6a/4ba3d0fb7297ebae71171822554abe48d7cab29c28b8f9f2c04b79988c05/rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00", size = 359751, upload-time = "2025-11-30T20:21:34.591Z" }, - { url = "https://files.pythonhosted.org/packages/cd/7c/e4933565ef7f7a0818985d87c15d9d273f1a649afa6a52ea35ad011195ea/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389a2d49eded1896c3d48b0136ead37c48e221b391c052fba3f4055c367f60a6", size = 389696, upload-time = "2025-11-30T20:21:36.122Z" }, - { url = "https://files.pythonhosted.org/packages/5e/01/6271a2511ad0815f00f7ed4390cf2567bec1d4b1da39e2c27a41e6e3b4de/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:32c8528634e1bf7121f3de08fa85b138f4e0dc47657866630611b03967f041d7", size = 403136, upload-time = "2025-11-30T20:21:37.728Z" }, - { url = "https://files.pythonhosted.org/packages/55/64/c857eb7cd7541e9b4eee9d49c196e833128a55b89a9850a9c9ac33ccf897/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f207f69853edd6f6700b86efb84999651baf3789e78a466431df1331608e5324", size = 524699, upload-time = "2025-11-30T20:21:38.92Z" }, - { url = "https://files.pythonhosted.org/packages/9c/ed/94816543404078af9ab26159c44f9e98e20fe47e2126d5d32c9d9948d10a/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67b02ec25ba7a9e8fa74c63b6ca44cf5707f2fbfadae3ee8e7494297d56aa9df", size = 412022, upload-time = "2025-11-30T20:21:40.407Z" }, - { url = "https://files.pythonhosted.org/packages/61/b5/707f6cf0066a6412aacc11d17920ea2e19e5b2f04081c64526eb35b5c6e7/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0e95f6819a19965ff420f65578bacb0b00f251fefe2c8b23347c37174271f3", size = 390522, upload-time = "2025-11-30T20:21:42.17Z" }, - { url = "https://files.pythonhosted.org/packages/13/4e/57a85fda37a229ff4226f8cbcf09f2a455d1ed20e802ce5b2b4a7f5ed053/rpds_py-0.30.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:a452763cc5198f2f98898eb98f7569649fe5da666c2dc6b5ddb10fde5a574221", size = 404579, upload-time = "2025-11-30T20:21:43.769Z" }, - { url = "https://files.pythonhosted.org/packages/f9/da/c9339293513ec680a721e0e16bf2bac3db6e5d7e922488de471308349bba/rpds_py-0.30.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0b65193a413ccc930671c55153a03ee57cecb49e6227204b04fae512eb657a7", size = 421305, upload-time = "2025-11-30T20:21:44.994Z" }, - { url = "https://files.pythonhosted.org/packages/f9/be/522cb84751114f4ad9d822ff5a1aa3c98006341895d5f084779b99596e5c/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:858738e9c32147f78b3ac24dc0edb6610000e56dc0f700fd5f651d0a0f0eb9ff", size = 572503, upload-time = "2025-11-30T20:21:46.91Z" }, - { url = "https://files.pythonhosted.org/packages/a2/9b/de879f7e7ceddc973ea6e4629e9b380213a6938a249e94b0cdbcc325bb66/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:da279aa314f00acbb803da1e76fa18666778e8a8f83484fba94526da5de2cba7", size = 598322, upload-time = "2025-11-30T20:21:48.709Z" }, - { url = "https://files.pythonhosted.org/packages/48/ac/f01fc22efec3f37d8a914fc1b2fb9bcafd56a299edbe96406f3053edea5a/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7c64d38fb49b6cdeda16ab49e35fe0da2e1e9b34bc38bd78386530f218b37139", size = 560792, upload-time = "2025-11-30T20:21:50.024Z" }, - { url = "https://files.pythonhosted.org/packages/e2/da/4e2b19d0f131f35b6146425f846563d0ce036763e38913d917187307a671/rpds_py-0.30.0-cp310-cp310-win32.whl", hash = "sha256:6de2a32a1665b93233cde140ff8b3467bdb9e2af2b91079f0333a0974d12d464", size = 221901, upload-time = "2025-11-30T20:21:51.32Z" }, - { url = "https://files.pythonhosted.org/packages/96/cb/156d7a5cf4f78a7cc571465d8aec7a3c447c94f6749c5123f08438bcf7bc/rpds_py-0.30.0-cp310-cp310-win_amd64.whl", hash = "sha256:1726859cd0de969f88dc8673bdd954185b9104e05806be64bcd87badbe313169", size = 235823, upload-time = "2025-11-30T20:21:52.505Z" }, - { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157, upload-time = "2025-11-30T20:21:53.789Z" }, - { url = "https://files.pythonhosted.org/packages/94/ba/24e5ebb7c1c82e74c4e4f33b2112a5573ddc703915b13a073737b59b86e0/rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d", size = 359676, upload-time = "2025-11-30T20:21:55.475Z" }, - { url = "https://files.pythonhosted.org/packages/84/86/04dbba1b087227747d64d80c3b74df946b986c57af0a9f0c98726d4d7a3b/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4", size = 389938, upload-time = "2025-11-30T20:21:57.079Z" }, - { url = "https://files.pythonhosted.org/packages/42/bb/1463f0b1722b7f45431bdd468301991d1328b16cffe0b1c2918eba2c4eee/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f", size = 402932, upload-time = "2025-11-30T20:21:58.47Z" }, - { url = "https://files.pythonhosted.org/packages/99/ee/2520700a5c1f2d76631f948b0736cdf9b0acb25abd0ca8e889b5c62ac2e3/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4", size = 525830, upload-time = "2025-11-30T20:21:59.699Z" }, - { url = "https://files.pythonhosted.org/packages/e0/ad/bd0331f740f5705cc555a5e17fdf334671262160270962e69a2bdef3bf76/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97", size = 412033, upload-time = "2025-11-30T20:22:00.991Z" }, - { url = "https://files.pythonhosted.org/packages/f8/1e/372195d326549bb51f0ba0f2ecb9874579906b97e08880e7a65c3bef1a99/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89", size = 390828, upload-time = "2025-11-30T20:22:02.723Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2b/d88bb33294e3e0c76bc8f351a3721212713629ffca1700fa94979cb3eae8/rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d", size = 404683, upload-time = "2025-11-30T20:22:04.367Z" }, - { url = "https://files.pythonhosted.org/packages/50/32/c759a8d42bcb5289c1fac697cd92f6fe01a018dd937e62ae77e0e7f15702/rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038", size = 421583, upload-time = "2025-11-30T20:22:05.814Z" }, - { url = "https://files.pythonhosted.org/packages/2b/81/e729761dbd55ddf5d84ec4ff1f47857f4374b0f19bdabfcf929164da3e24/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7", size = 572496, upload-time = "2025-11-30T20:22:07.713Z" }, - { url = "https://files.pythonhosted.org/packages/14/f6/69066a924c3557c9c30baa6ec3a0aa07526305684c6f86c696b08860726c/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed", size = 598669, upload-time = "2025-11-30T20:22:09.312Z" }, - { url = "https://files.pythonhosted.org/packages/5f/48/905896b1eb8a05630d20333d1d8ffd162394127b74ce0b0784ae04498d32/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85", size = 561011, upload-time = "2025-11-30T20:22:11.309Z" }, - { url = "https://files.pythonhosted.org/packages/22/16/cd3027c7e279d22e5eb431dd3c0fbc677bed58797fe7581e148f3f68818b/rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c", size = 221406, upload-time = "2025-11-30T20:22:13.101Z" }, - { url = "https://files.pythonhosted.org/packages/fa/5b/e7b7aa136f28462b344e652ee010d4de26ee9fd16f1bfd5811f5153ccf89/rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825", size = 236024, upload-time = "2025-11-30T20:22:14.853Z" }, - { url = "https://files.pythonhosted.org/packages/14/a6/364bba985e4c13658edb156640608f2c9e1d3ea3c81b27aa9d889fff0e31/rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229", size = 229069, upload-time = "2025-11-30T20:22:16.577Z" }, - { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, - { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, - { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, - { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, - { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, - { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, - { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, - { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, - { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, - { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, - { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, - { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, - { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, - { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, - { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, - { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, - { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, - { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, - { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, - { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, - { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, - { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, - { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, - { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, - { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, - { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, - { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, - { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, - { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, - { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, - { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, - { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, - { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, - { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, - { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, - { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, - { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, - { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, - { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, - { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, - { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, - { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, - { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, - { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, - { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, - { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, - { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, - { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, - { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, - { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, - { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, - { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, - { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, - { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, - { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, - { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, - { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, - { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, - { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, - { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, - { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, - { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, - { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, - { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, - { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, - { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, - { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, - { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, - { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, - { url = "https://files.pythonhosted.org/packages/69/71/3f34339ee70521864411f8b6992e7ab13ac30d8e4e3309e07c7361767d91/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58", size = 372292, upload-time = "2025-11-30T20:24:16.537Z" }, - { url = "https://files.pythonhosted.org/packages/57/09/f183df9b8f2d66720d2ef71075c59f7e1b336bec7ee4c48f0a2b06857653/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a", size = 362128, upload-time = "2025-11-30T20:24:18.086Z" }, - { url = "https://files.pythonhosted.org/packages/7a/68/5c2594e937253457342e078f0cc1ded3dd7b2ad59afdbf2d354869110a02/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb", size = 391542, upload-time = "2025-11-30T20:24:20.092Z" }, - { url = "https://files.pythonhosted.org/packages/49/5c/31ef1afd70b4b4fbdb2800249f34c57c64beb687495b10aec0365f53dfc4/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c", size = 404004, upload-time = "2025-11-30T20:24:22.231Z" }, - { url = "https://files.pythonhosted.org/packages/e3/63/0cfbea38d05756f3440ce6534d51a491d26176ac045e2707adc99bb6e60a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3", size = 527063, upload-time = "2025-11-30T20:24:24.302Z" }, - { url = "https://files.pythonhosted.org/packages/42/e6/01e1f72a2456678b0f618fc9a1a13f882061690893c192fcad9f2926553a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5", size = 413099, upload-time = "2025-11-30T20:24:25.916Z" }, - { url = "https://files.pythonhosted.org/packages/b8/25/8df56677f209003dcbb180765520c544525e3ef21ea72279c98b9aa7c7fb/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738", size = 392177, upload-time = "2025-11-30T20:24:27.834Z" }, - { url = "https://files.pythonhosted.org/packages/4a/b4/0a771378c5f16f8115f796d1f437950158679bcd2a7c68cf251cfb00ed5b/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f", size = 406015, upload-time = "2025-11-30T20:24:29.457Z" }, - { url = "https://files.pythonhosted.org/packages/36/d8/456dbba0af75049dc6f63ff295a2f92766b9d521fa00de67a2bd6427d57a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877", size = 423736, upload-time = "2025-11-30T20:24:31.22Z" }, - { url = "https://files.pythonhosted.org/packages/13/64/b4d76f227d5c45a7e0b796c674fd81b0a6c4fbd48dc29271857d8219571c/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a", size = 573981, upload-time = "2025-11-30T20:24:32.934Z" }, - { url = "https://files.pythonhosted.org/packages/20/91/092bacadeda3edf92bf743cc96a7be133e13a39cdbfd7b5082e7ab638406/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4", size = 599782, upload-time = "2025-11-30T20:24:35.169Z" }, - { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191, upload-time = "2025-11-30T20:24:36.853Z" }, -] - -[[package]] -name = "rpds-py" -version = "2026.5.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.13'", - "python_full_version >= '3.11' and python_full_version < '3.13'", -] -sdist = { url = "https://files.pythonhosted.org/packages/2e/43/25a8dcd3feedd735039a8f0b5b7e3b118232b5eae288c4fd9ab200d41094/rpds_py-2026.5.1.tar.gz", hash = "sha256:07b24fea40541e28570e5b795a4a38fbdcd12550c06bd0748005ecc8116ca256", size = 64459, upload-time = "2026-05-28T12:02:13.232Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/a0/acf8b6fc20bfdcd3a45bd3f57680fb198e157b7e997b9123b10763798bd2/rpds_py-2026.5.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3397a5ed7174dc2786bb214030232fc36fe8e5584fec43a9952cc542b1a12036", size = 355609, upload-time = "2026-05-28T11:58:50.78Z" }, - { url = "https://files.pythonhosted.org/packages/b6/95/f8203fd997484b1690a6869cd0e503b6c3c6be55b0ecc36d1a491fe742f0/rpds_py-2026.5.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:99ab6ba7bfa2cb0f96a04e3652355bf04e3f51aceb1e943b8541dab7ba4828cc", size = 348460, upload-time = "2026-05-28T11:58:52.374Z" }, - { url = "https://files.pythonhosted.org/packages/33/8c/b47326ad2f0be545a5e5c1a55937a12afaea7d392ba2837bb9680f57e6c9/rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0efbe45632665e53e3db8fe1e5692db58fc5cb9bab4459d570b83efefe11164", size = 381031, upload-time = "2026-05-28T11:58:53.775Z" }, - { url = "https://files.pythonhosted.org/packages/22/0b/e83bbd97ffac6f6389b605cd4e1c8ac5761dc7e977769c9255d8c5adb7bd/rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:01d17b29c0c23d82b1f4751147ec49cf451f1fc2554eb9ef5f957e55d2656ead", size = 387121, upload-time = "2026-05-28T11:58:55.243Z" }, - { url = "https://files.pythonhosted.org/packages/fd/0e/d285d1bc8864245919c61e1ca82263e4a66d337759c3a4cef72766ff9afc/rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7559f72b94ae52659086c595dfa017cde03155f7832071d30959049052cb3ece", size = 501026, upload-time = "2026-05-28T11:58:56.788Z" }, - { url = "https://files.pythonhosted.org/packages/86/06/ccb2109a1e543437b5e43816f2b43b9554cc6783145528a4e3711e05c011/rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e25b7088f9ccbfc0dfcaa52bf969300ca229e10ecf758974ebcbb080a4b37bb", size = 391865, upload-time = "2026-05-28T11:58:58.298Z" }, - { url = "https://files.pythonhosted.org/packages/3d/33/237173db1cfef10105b3839a24de00eb8d2a523711add4632447cdf0aedd/rpds_py-2026.5.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613fc4ee9eaef26dc5840666214dd6fbcebcf32f46e76f4abc473059f4e13dda", size = 378012, upload-time = "2026-05-28T11:58:59.589Z" }, - { url = "https://files.pythonhosted.org/packages/97/64/1eae54e34d5161f9969295e80bd6b62a55f2b6ac5f2a5b60d02c2140e758/rpds_py-2026.5.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:85264a90ff4c05c1568dd65f5921c837614b67c60358fb4c17df3b7f2e90690a", size = 391111, upload-time = "2026-05-28T11:59:01.104Z" }, - { url = "https://files.pythonhosted.org/packages/d8/34/5bb334a5a0f65d77869217c4654f34c78a7d11b93938a3c076a2edeafc52/rpds_py-2026.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe71bca7d547acb17027c7fd1624ff8aae623499c498d3e7011182c4de5c25e0", size = 409225, upload-time = "2026-05-28T11:59:02.433Z" }, - { url = "https://files.pythonhosted.org/packages/16/0f/007ec21283b5b040b4ec3bd95e0402591e22bfa7d5c93dfe01c465c2d2d7/rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05fa4f41f37ec97c9c260441a940450a192f78d774d2b097eee1379f1e1246a", size = 556487, upload-time = "2026-05-28T11:59:04.012Z" }, - { url = "https://files.pythonhosted.org/packages/ff/10/5437c94508169b6b22d8418fef7a66e9ffb5f3b9e9c94460f2eedafe06ff/rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df1d2a1996755b24b9ecee92cb4d36c28f86f464a6a173349c26bab41e94b8c2", size = 620798, upload-time = "2026-05-28T11:59:05.485Z" }, - { url = "https://files.pythonhosted.org/packages/e0/d5/9937dce4d6bda74157b954e7d1460db05a22f5929dccfeeba1ed27a93df0/rpds_py-2026.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8895840ac4809e5f60c88fd07617cd71326e73d6e5a8aa783c5c0f7c24985de2", size = 584053, upload-time = "2026-05-28T11:59:06.837Z" }, - { url = "https://files.pythonhosted.org/packages/6c/31/750617dd0ae1752471bf43f9e41d263398fae7cde7849d23b8574a70e617/rpds_py-2026.5.1-cp311-cp311-win32.whl", hash = "sha256:3684a59b158a7683aaeb8e25352e9a9dd2122cec78f2d8530266e4f91b4c7b3f", size = 214390, upload-time = "2026-05-28T11:59:08.402Z" }, - { url = "https://files.pythonhosted.org/packages/3c/bb/3dcab0e1d9516303f2eb672a5d6f62eca5a69e2886301e9c8c54b520c39b/rpds_py-2026.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:7bd530e6a530bb3ea892f194fafa455f3516ac25ecf7143fd33c09be62b0470a", size = 231097, upload-time = "2026-05-28T11:59:09.786Z" }, - { url = "https://files.pythonhosted.org/packages/49/d6/c6bbf5cb1cf12b9732df8074b57f6ef8341ba884c95d40632ae8bddb44e4/rpds_py-2026.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:0a5ae4dbe43c1076983b72616496919872ae7bbe7a1e21cc48336bc3154d130b", size = 226361, upload-time = "2026-05-28T11:59:11.079Z" }, - { url = "https://files.pythonhosted.org/packages/d4/e7/a78582dc57caa592dcc7d4fb69b61390561e908eb3d2f5df5928a8e354c0/rpds_py-2026.5.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3abe24a66e57adcfa645d718063a5fa5103ecc71ddbf26d78af8f9368018ff1d", size = 353040, upload-time = "2026-05-28T11:59:12.531Z" }, - { url = "https://files.pythonhosted.org/packages/a3/43/35e3f136343aef451e545ce8c38d36c2f93c0ed88703db8b64ba2b205c68/rpds_py-2026.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58b1d94308ddf0b1982f61f2eb54bf92997c9ece8a8093ef014250f4a517906c", size = 345775, upload-time = "2026-05-28T11:59:13.827Z" }, - { url = "https://files.pythonhosted.org/packages/20/e1/0f2160c5982d3157734d5cb3ed63d8b2d583a73c9864f77b666449f32cf8/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fa92420128dadce7f54bd73ba1825a273e9268fe9e35dbf7e6362890efa4e08", size = 376329, upload-time = "2026-05-28T11:59:15.271Z" }, - { url = "https://files.pythonhosted.org/packages/d0/11/ee0ba42aff83bf4effdbc576673c6be64c5e173978c3f6d537e94482f77d/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca653c6546386227cd9800d1bef6a348099acf8db4250341da6d90f663d6dfcb", size = 383539, upload-time = "2026-05-28T11:59:16.665Z" }, - { url = "https://files.pythonhosted.org/packages/11/df/d94aa6a499d4ac40afe2d7620f2c597fd3c0f182e854ad7cf3f596a81cb6/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66c93681c4729e4e3ecba31b8179fae083ff3118841672835140338b4b9867c1", size = 494674, upload-time = "2026-05-28T11:59:17.991Z" }, - { url = "https://files.pythonhosted.org/packages/1f/75/33d30f43bb2f458de11979486a591b1bf6e5651765ed1704c6197c2dc773/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ff257542e04796880e011e15cd4dc21c2599975df2aaa8f2c8495ca574e1a5", size = 389268, upload-time = "2026-05-28T11:59:19.434Z" }, - { url = "https://files.pythonhosted.org/packages/f4/1e/2c9096fc19d5fd084b0184ca2b651e659aa0a37e6fdbecf6ece47f147fe1/rpds_py-2026.5.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6825cc329b290e93c5f6a9be2393118a763f6ccf6abd83704e0c102ca583644", size = 376280, upload-time = "2026-05-28T11:59:21Z" }, - { url = "https://files.pythonhosted.org/packages/b9/e5/61ec9f8be8211ea7f48448195549e4aaf02004083475493b0e137702ecb2/rpds_py-2026.5.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:de42116e69cb53b911cc34aee5ab98f36c597b822545045d49e938818b99e5e4", size = 387233, upload-time = "2026-05-28T11:59:22.454Z" }, - { url = "https://files.pythonhosted.org/packages/0d/ca/bcec1005c4f4a234f92a29078631fee49206c7265ccae966f18fd332e80e/rpds_py-2026.5.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0f920015df2a504bebaba6d4c31ccf3fcf942f92655c086da30b671aad19aa6", size = 405009, upload-time = "2026-05-28T11:59:23.845Z" }, - { url = "https://files.pythonhosted.org/packages/72/e6/4d5718c5cf26c522dc7c9999e238da1e77380b81d0c5d1df11e271ddfeb1/rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0408a24e44feb919423dc6d9da677cb5cddb894d2ca9e763967d156d9c60fab4", size = 553113, upload-time = "2026-05-28T11:59:25.184Z" }, - { url = "https://files.pythonhosted.org/packages/d4/25/2ee807bdb3e1f0b7eddf7782acd5665a8b5205a331a7d7244a52c4812fd9/rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cea68bcd53467561ae2f96a6bdad1544299ba97b5b0ddcd5ac3d376e5c781c24", size = 618838, upload-time = "2026-05-28T11:59:26.749Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c1/7d4c26f167f8c41501cc073d30ee22082b16ce358cf5b00ec97cbc7804ea/rpds_py-2026.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4be8b1d2a705cc37d08256004e1d07de143fa0075c8e85a3df020b776f62b732", size = 582436, upload-time = "2026-05-28T11:59:28.11Z" }, - { url = "https://files.pythonhosted.org/packages/04/1d/9d12b0a337bab46f4769f8857f4007e3b2d639e14f9a44a0efe157696e64/rpds_py-2026.5.1-cp312-cp312-win32.whl", hash = "sha256:6736718bd4fc49cbcb538ba30516fdbef161522acefb739657d48b97bd864fed", size = 212734, upload-time = "2026-05-28T11:59:29.689Z" }, - { url = "https://files.pythonhosted.org/packages/c5/93/e4116f2de7f56bc7406a76033dc501811ddeb22b7f056b92d632871ebb0c/rpds_py-2026.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:0a7d1eec967df0e9b22614a5e177622e0c89611d03727fa0cb48e45028907870", size = 229045, upload-time = "2026-05-28T11:59:31.033Z" }, - { url = "https://files.pythonhosted.org/packages/cb/53/6c3419d85eb2ec5938a37627c585b42d76a63bb731d6e42ed4b079ebf486/rpds_py-2026.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:1841d067089e117142d79b98aa0df2f08b52f2ecc1819dd2700636c0db74a473", size = 223967, upload-time = "2026-05-28T11:59:32.318Z" }, - { url = "https://files.pythonhosted.org/packages/6c/32/14c961ad295f490eb0849ada8b79683e93a59b9de3afdd983eaf55fa6867/rpds_py-2026.5.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:efef4ac29c6ff495531eb17ee705b62841ecaa291b7c7077e848ea03e237164d", size = 352787, upload-time = "2026-05-28T11:59:33.655Z" }, - { url = "https://files.pythonhosted.org/packages/ca/bb/d1b85117967c11191441a7274ae616c65d93901d082c588f89a50a8da5ae/rpds_py-2026.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c39f5b67a8a2e67179ada2a954227d670fe65fa9098457f698f56ddf248709b3", size = 345179, upload-time = "2026-05-28T11:59:35Z" }, - { url = "https://files.pythonhosted.org/packages/7c/46/d84105f062e626a1b233f863907288a4708c2d833b8b4c6fb2764bc080c0/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5c30f3f04eef4fbd362226a6f31d7c8895ca4fbb6e0b790f6890a98d8da8559", size = 376173, upload-time = "2026-05-28T11:59:36.43Z" }, - { url = "https://files.pythonhosted.org/packages/e2/ae/469d7959ce5b1201e1de135dc735b86db3b35dd0d1734f6a44246d5f061c/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:277f6c82f0580848796c7ecc8a7173aa3bfb928e4ff831261c2f60a81dc270db", size = 383162, upload-time = "2026-05-28T11:59:37.995Z" }, - { url = "https://files.pythonhosted.org/packages/dc/a2/57853d31a1116a561aa072794602ad3f6341e18d70a8523f1bd5b9fc1e5a/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63c2c4c213f1a4e3f3de28ecab029dbdee976324e729c0d7a55211be72576b02", size = 495093, upload-time = "2026-05-28T11:59:39.453Z" }, - { url = "https://files.pythonhosted.org/packages/99/63/3a8eabcad9314b7daf5c65f451d2c33d989235cd8a5762186cf2c3f5a4f8/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3350ec808fb538fe71a1f94dfaa0e29c598dfad805ce49f0caec5ae3183c652b", size = 389829, upload-time = "2026-05-28T11:59:40.896Z" }, - { url = "https://files.pythonhosted.org/packages/4b/25/05678d97fc25e2622df14dc530fb82023174ecfff6733991ed0d78f167bd/rpds_py-2026.5.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b964e3ab599e718dc46c018d104b1ebc007cbc6567d827c94a687fca56d77e", size = 374786, upload-time = "2026-05-28T11:59:42.626Z" }, - { url = "https://files.pythonhosted.org/packages/88/d1/8c90b6431e80a3b91b284a5c7c8c0c4f9c006444d90477a740d6e0f9c694/rpds_py-2026.5.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:19cb09fab7b7fc96b2a6e28f2e34b72a3705ff27b37edb77455316e5d3f3dc9b", size = 386920, upload-time = "2026-05-28T11:59:44.124Z" }, - { url = "https://files.pythonhosted.org/packages/ff/99/4638f672ab356682d633ee0da9255f5b67ce6efd0b85eb94ad3e255e65a5/rpds_py-2026.5.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:abe76bcdba31e576cb83eeb8797aa0d882b738fef6dc65d0601fc753806a5b46", size = 405059, upload-time = "2026-05-28T11:59:47.177Z" }, - { url = "https://files.pythonhosted.org/packages/66/3f/3546524b6eb4cc2e1f363a3d638fa52f6c24faae3500c25fb488b02f1740/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8bff7073db3899158fff55ebf57b113a67030af26f80a18978f9f0aa60250ddf", size = 553030, upload-time = "2026-05-28T11:59:48.603Z" }, - { url = "https://files.pythonhosted.org/packages/c6/c3/7b3388c796fcf471bd17194242d4dc1a7608567c0fa422bcc1c5e79f9c1e/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8ba264fa49be666cd9cc56bf34ec7002fb3d27a4aee5bcb4d43d0d18feb1bb6f", size = 618975, upload-time = "2026-05-28T11:59:50.314Z" }, - { url = "https://files.pythonhosted.org/packages/61/1e/a3cb07f2795075d1d88efddae2f541359fde5f08c81ee114c29c2949c90a/rpds_py-2026.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4860b603ddda0475a8885499b3729e90229d480105b42651962a5397d995fa89", size = 581178, upload-time = "2026-05-28T11:59:51.673Z" }, - { url = "https://files.pythonhosted.org/packages/a1/74/e758c03a5ef46f04c37f2651a2893db846d569ba8a7bca469d4b58939bcd/rpds_py-2026.5.1-cp313-cp313-win32.whl", hash = "sha256:7944270ae71383f6e2657dd7d5ce4eeb4ac2d0059a6738f0510583d462ab4842", size = 212481, upload-time = "2026-05-28T11:59:53.148Z" }, - { url = "https://files.pythonhosted.org/packages/70/ec/a2aca432db9c7359b40fa393eeeaa0d166c2f70175be956e75fa24197c44/rpds_py-2026.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:88647f43a73c4e01be19b04ceef0c8d3a1958153604d13c773becd8016f2a0cf", size = 228519, upload-time = "2026-05-28T11:59:54.505Z" }, - { url = "https://files.pythonhosted.org/packages/29/60/a73bfdd45b096574556acf303bbd9fa9eed36ca8a818b514e2a5d5fe2b9d/rpds_py-2026.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:453895624ecf7db7063b1004e44037522bbaef9ff6a945e59bc71662d7a03abd", size = 223446, upload-time = "2026-05-28T11:59:56.081Z" }, - { url = "https://files.pythonhosted.org/packages/18/e2/408105fd611823f00882aea810f3989a30d26b1bab8b6beb20f98c724e0e/rpds_py-2026.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:b4e4bc98639ec915f512fde3aa7a95e0041d95d9c3cc86eea841fa63cb1e8600", size = 355287, upload-time = "2026-05-28T11:59:57.448Z" }, - { url = "https://files.pythonhosted.org/packages/8d/58/5c4a43436843c90d0f6d19f82c200c80e3843ca9fa07b237623327f6d384/rpds_py-2026.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cacedb7a6e167680acba45ad5716e89067d225dc80da0d7040cae8c81d4572fa", size = 347033, upload-time = "2026-05-28T11:59:58.881Z" }, - { url = "https://files.pythonhosted.org/packages/fb/c2/1a71acdacaf4e259b10278fb87b039ded3cf80041bcd89dd8a3ea702ded6/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68700371c5d7ae1412862ddfa719090925c93ecf351c566d66f09d04b136ea00", size = 376891, upload-time = "2026-05-28T12:00:00.516Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c8/535f3d9b65addd8e28aa87b83c6e526799c3717a88273db8ea795beeef7a/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:296c799becfa849c779c8725494fe9ed94959ed886787df4364b058465bad7f0", size = 385646, upload-time = "2026-05-28T12:00:02.394Z" }, - { url = "https://files.pythonhosted.org/packages/1c/91/dc033f313345c354ade914dbe73cdb90b615a4409ea02430d5356794f3d8/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3858b908218ee108d0bbfb2095ccc237648053c9bf98affad7cb079acaf1d97", size = 498830, upload-time = "2026-05-28T12:00:04.189Z" }, - { url = "https://files.pythonhosted.org/packages/27/fc/90fcbea459dbb8ddc18a2e0fd1de9412b48bc84ffff2db771cf714bacfd6/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4fb8d2e7cb2f850b169806d61d1b991738acec96500a75c30f49caf064ce7cef", size = 392830, upload-time = "2026-05-28T12:00:05.797Z" }, - { url = "https://files.pythonhosted.org/packages/b2/1d/46cd11a228c9750684a798d98f878be6f614aa762438da7378f035e79e35/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b74c10ed6a8f190f4287f53bcfea348b92a84a9c9f70d30183d1e6172d580d", size = 379613, upload-time = "2026-05-28T12:00:07.433Z" }, - { url = "https://files.pythonhosted.org/packages/24/4a/d9b0c6af3a1de03eb93741bbe8be2bdce84d8fda8224f3005451d86df389/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:b9a6528956191c48c52294a592dbd4a8386d7048bdb25c0efcb6b966466c6d83", size = 388183, upload-time = "2026-05-28T12:00:09.227Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b4/db7aaabdda6d020afc87d981bcc2f57a434c7dec60ecfc2ab3dd50b20351/rpds_py-2026.5.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:af03e34e860047bc7a352b842856fcf78798fbb81132cc98bd2f907ab4eb9cd2", size = 408578, upload-time = "2026-05-28T12:00:10.779Z" }, - { url = "https://files.pythonhosted.org/packages/08/d6/070f6a41cbb343e2ac4171859bf3f3623e0ab002f72619d6d505313ec2de/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fea6e836d10abbe191d557d33bd58bd5987725fe63aa1eefe557d230209855bd", size = 553573, upload-time = "2026-05-28T12:00:12.443Z" }, - { url = "https://files.pythonhosted.org/packages/75/ab/1a71ea3589c4345dac0a0518f0e6a031cb42689277851b683c46d27463a5/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:fc0c0f878ea770a0a8a462456c5ad36fc9fe6358e6b76fdadc7f17575e0b8bf1", size = 620861, upload-time = "2026-05-28T12:00:14.09Z" }, - { url = "https://files.pythonhosted.org/packages/8a/22/9bf80a56069c0c443fcfefac639a86a744550a2898817a6dfd3e26654924/rpds_py-2026.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e0b360f316d966b048b085857630b3cc51f3db2f07b06f440eac8f695374d1e3", size = 585633, upload-time = "2026-05-28T12:00:15.66Z" }, - { url = "https://files.pythonhosted.org/packages/da/68/3b2c0a75c9e04125696f84ebdbbf304acf5a40b58ba4481cdb98a922c3ba/rpds_py-2026.5.1-cp313-cp313t-win32.whl", hash = "sha256:a2999883eedf72fdfb7520b92c7d4ec2572a71ff40239377aa604cc529eecafc", size = 210074, upload-time = "2026-05-28T12:00:17.291Z" }, - { url = "https://files.pythonhosted.org/packages/e7/8b/609157d5a25d37d4f29f92840ba531f416907c34ae5c5739dd21fc2bef98/rpds_py-2026.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e07be2a9d7122bd6e82dea89814ef8dc893feb1aae97fec1630f3263bbb30e55", size = 228635, upload-time = "2026-05-28T12:00:18.73Z" }, - { url = "https://files.pythonhosted.org/packages/d4/6f/19c1918a4b590d8de87e712e4abe4b3875771eff60216fb6153cf6665c68/rpds_py-2026.5.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1f2c391c3059798093b65df23aca2cac150460ae9c630d99dec83d703d9485b9", size = 349756, upload-time = "2026-05-28T12:00:20.217Z" }, - { url = "https://files.pythonhosted.org/packages/e5/60/a06fe7da34eca79dacbf958a2ba0c6eea85bc2b29de20080bf40f72f66fa/rpds_py-2026.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:413b424f7c4ee65ab5e5be91f5731be0f8b41a1ee2b12dfe810d716312e95a78", size = 343831, upload-time = "2026-05-28T12:00:21.711Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ec/b2333b97b90e2a6ef6ca8ad386ee284968e74bcfe113b3f1a8d9036429a9/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c595a1d9255dce0599e13130d1440ab2506654f2b50294226ee06402f8fef63", size = 375127, upload-time = "2026-05-28T12:00:23.326Z" }, - { url = "https://files.pythonhosted.org/packages/14/7f/e00aae54067f2b488c4637961d5f58204d470795fc791085fa3f15060d2e/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1c27c5f6102eac8c03e7595a00827a53b271ba40a53b59ff8709170e0855ea4a", size = 379034, upload-time = "2026-05-28T12:00:24.89Z" }, - { url = "https://files.pythonhosted.org/packages/be/cc/423999bbb8ae8dc93c77fc1d5e984ade5eb89d237d3bb884ccfa72ae2890/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c7fcf61d44cacecaf3aea542b0e053db77972a4573e7ceda16fb2b399161195", size = 490823, upload-time = "2026-05-28T12:00:26.676Z" }, - { url = "https://files.pythonhosted.org/packages/0f/aa/c671bf660f12e68d3c52ff86c7066ed1372df5a0f4f2ff584e419b8207e7/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c817a189d4ee14290420e5ff051e4dd6baa13f3edf84685071dee07a6d538ee", size = 388144, upload-time = "2026-05-28T12:00:28.577Z" }, - { url = "https://files.pythonhosted.org/packages/19/c8/d63bb75b68afe77b229e3021c6031bcaf01da5db5b0e69d0d10f9ba679a7/rpds_py-2026.5.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21846aac0ed2e0589f38c12dc44e77bb64e494b771eadbcf169cba00566ba7ba", size = 371959, upload-time = "2026-05-28T12:00:30.304Z" }, - { url = "https://files.pythonhosted.org/packages/82/35/c51122014d8274ff37dc606d60049c3db7d83da02b5b282511e5a906a9a6/rpds_py-2026.5.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b317c87a13f769a4e787819bd508aaa5d69aa09b0880de9af6d3a8a54571cdec", size = 383558, upload-time = "2026-05-28T12:00:31.764Z" }, - { url = "https://files.pythonhosted.org/packages/e3/f9/2790cb99c136a5363acdeacf5c27c56f3de0d4118a1f48fca83404c99c89/rpds_py-2026.5.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ce87129d9f2c14fa6c4a8601fb80eb4488c80d38a20cd13758ef11123e14995d", size = 402789, upload-time = "2026-05-28T12:00:33.247Z" }, - { url = "https://files.pythonhosted.org/packages/e5/1b/e4fb584f8c75d35c38150ff6a332cda949e6f97acba1f4fd123b14ab56fe/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9cdddb6c1207d284d94fd1530adf57fbd797fe7c4b8704ba85f49414f2557e7d", size = 551405, upload-time = "2026-05-28T12:00:34.819Z" }, - { url = "https://files.pythonhosted.org/packages/d8/f7/a6731b4216cb3793ea1af5391da240f5683dacc0d13e034fe5fc3503f240/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:4e237e139f94d3c036fd28eb9f564c99055476ff4ff05cd42be55ce349b5aa02", size = 616975, upload-time = "2026-05-28T12:00:36.268Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ea/2e051a81d95d8e63f4b35a1c463a87e8766bc3d083c067c5dfb6bf220747/rpds_py-2026.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ed0954b524873214369184a9c82b0eaa45a3fbb9a798cd95b17e0d98499e7ea0", size = 578701, upload-time = "2026-05-28T12:00:37.82Z" }, - { url = "https://files.pythonhosted.org/packages/65/56/b5f6fdb2083e32bca8a8993d89e70db114b4756c9e2c38421328126689d2/rpds_py-2026.5.1-cp314-cp314-win32.whl", hash = "sha256:2d88621d6a7d4dfa633d21abe90f280bb205274e16b1d1e61c6ad4640b2453b7", size = 209806, upload-time = "2026-05-28T12:00:39.492Z" }, - { url = "https://files.pythonhosted.org/packages/fb/80/65a5aa96c155e611d1ed844e4e1f57f3e36b021f396d9f8585d756e6b90d/rpds_py-2026.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:cef8ac28d26f4dda3533060c20fbf80a325458fa9fd23ea72a73cdfa8e978838", size = 225985, upload-time = "2026-05-28T12:00:40.94Z" }, - { url = "https://files.pythonhosted.org/packages/27/7c/ad185212e87b05f196daef92bc5f3caf07298eb47c295b5585c3dd3093ac/rpds_py-2026.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:eaaea962c68cdc68d4a533ba985ab8e9484277910bbfaa2ab3ef7732667bfed8", size = 221219, upload-time = "2026-05-28T12:00:43.15Z" }, - { url = "https://files.pythonhosted.org/packages/23/58/e14ae18759020334646b031e708ab4158d653a938822bfb7b95ef2e93aa3/rpds_py-2026.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:21942f52dbbd5f8758bf021213d28bd45c39e873e65e2407faf5f1846f5761ad", size = 352148, upload-time = "2026-05-28T12:00:44.638Z" }, - { url = "https://files.pythonhosted.org/packages/31/9b/5f4a1e2f960bca3ac5d052b139dd31eed97b259f9d909173821760d542e8/rpds_py-2026.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f414556f6e3958300ff941e40c9f97e3dc9774ddd1b3434c475d73dd354bbed3", size = 345196, upload-time = "2026-05-28T12:00:46.14Z" }, - { url = "https://files.pythonhosted.org/packages/1a/71/1d9574d6a2fa20ab60eaa55c7467f5aa20cbc770f341a05f09c0876f59e2/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef1013a8625c74043210190b246f5b1551e09757c1f356c6e4160ef96c5bc081", size = 374981, upload-time = "2026-05-28T12:00:47.531Z" }, - { url = "https://files.pythonhosted.org/packages/0c/9a/37e99f4915a80aa71670263c1267f7ae0af95f53a3f61e6c3bdc016d4515/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc68e231a77a5f0d774ae278a1f8e55c0456501820847c1e4efb3829f3441df6", size = 379961, upload-time = "2026-05-28T12:00:49.216Z" }, - { url = "https://files.pythonhosted.org/packages/a8/ff/6e73f74b89d2e0715e0fc86b7dde893f9a61ae2f9b256ff3bdfe41ac4e94/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9baffb505aff33acc69b422a19f77806680f3c8632227d79f48de8a810d1c2c5", size = 495965, upload-time = "2026-05-28T12:00:51.111Z" }, - { url = "https://files.pythonhosted.org/packages/ea/e0/425faba25f59d74d4638b267f7c7a80e8649d2ef4db10a19b0c4a71e6e6f/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8d2f912928d426e8cfa396f7f3f8d29a59e6689c86dcca3c420730c1096322b", size = 389526, upload-time = "2026-05-28T12:00:52.77Z" }, - { url = "https://files.pythonhosted.org/packages/c6/76/7a41960e3fddae47fab43a28684d5da981401dffd88253de0944148654cb/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90f628283be835db980c941767d41c9a27b5239e54ba0a9c1335247e82406964", size = 376190, upload-time = "2026-05-28T12:00:54.215Z" }, - { url = "https://files.pythonhosted.org/packages/27/60/5f38dc70824fc6951b51d35377e577a3a3a4c81a6769cc5a2de25ebe0ad1/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:1ebb2f0ab7e16132995a72de805170e0203df0c3dd22e1ef1cd1fdd90bd7a131", size = 383921, upload-time = "2026-05-28T12:00:55.673Z" }, - { url = "https://files.pythonhosted.org/packages/60/1a/d60a38caa1505f4b9483c3fbbde12c94e1079154f4f401a6da96f7e77621/rpds_py-2026.5.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f3df3d16ded76f1f8c9cdebd0e1ea55fdf4c23b812de189814da7cf229c22a81", size = 404766, upload-time = "2026-05-28T12:00:57.518Z" }, - { url = "https://files.pythonhosted.org/packages/87/ff/602fd3f174d6425f0bce05ad0dfbec0e96b38d0f7d08a79af5aa20083885/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9af8905b8f854990e40d5206aa5ac58d9b0fe0b7f351ff2bb086c20f6c8c6a47", size = 551343, upload-time = "2026-05-28T12:00:58.978Z" }, - { url = "https://files.pythonhosted.org/packages/b8/c1/1be13327acdbead3eca1fde03b6a34dbb011f1e864e217f0d32cc1779a7f/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:036a36a87fb1cd3b214d11c4b3c4f7d2ddad933625dca1c900b56a057c07740a", size = 618502, upload-time = "2026-05-28T12:01:00.656Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d7/afb49b49d7f2be8b7ba1a9f0977fa5168003437b93086726f066544e8351/rpds_py-2026.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ae3853454fe9ef283a03c96c2d835d39e84b14643a9d62c82ef0fb87d702ca", size = 581916, upload-time = "2026-05-28T12:01:02.22Z" }, - { url = "https://files.pythonhosted.org/packages/25/d1/dbef8c1f8a10f07beb62b5f054e20099fd9924b3ec001b8f0b6ac7813a85/rpds_py-2026.5.1-cp314-cp314t-win32.whl", hash = "sha256:6c3d771a46ec18b12af06ce36243a9a80b07a5d0515236332d90863ca8bb326a", size = 207855, upload-time = "2026-05-28T12:01:03.821Z" }, - { url = "https://files.pythonhosted.org/packages/2a/72/bfa4e61ab8e7dc1c8adf397e05e6cbdd4239357bd72b248d3de662f23915/rpds_py-2026.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c93c629be4636cf54337bd5f06c104d55e42ced54d681f6fe21ae510a65116f6", size = 225422, upload-time = "2026-05-28T12:01:05.194Z" }, - { url = "https://files.pythonhosted.org/packages/27/3a/7b5da92b640f67b6717ccafc83cdd06bfa7ff2395c3685c68922bb54d703/rpds_py-2026.5.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:3574b55c604b8f75dacb007136508bbc0db406e626301778096a133327e7f2fb", size = 349576, upload-time = "2026-05-28T12:01:06.722Z" }, - { url = "https://files.pythonhosted.org/packages/d7/8a/2aafd7ad355a1bd48ca76e2262b74b15e6432b5a1efe150efd4d779cd55d/rpds_py-2026.5.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:94068eb3ae6d43f5a786b7db96a406a34e6d5c24489feef32fd6e8946ea7b291", size = 343640, upload-time = "2026-05-28T12:01:08.441Z" }, - { url = "https://files.pythonhosted.org/packages/f7/7d/6c9523c1abbe840a1b7fba3c516d48e1d3487cc80fea4366c4071cf56784/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a5b10e8ce894825f380a8f1b6444cf73c294dfea62afbb2d13e3a9e630cec1", size = 375322, upload-time = "2026-05-28T12:01:09.934Z" }, - { url = "https://files.pythonhosted.org/packages/5a/5d/0b7b03fb1dc509321f01de3149784ab773e34c8573022029af8076afcb9c/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fc09f82e63d4bcd58149572f857a431bae851dc747e313c3b5bdf7abb907fda8", size = 379066, upload-time = "2026-05-28T12:01:11.48Z" }, - { url = "https://files.pythonhosted.org/packages/d7/e2/8ef6012999ebf1cb1c22f876d9ce5e63d960fd4631d2af3202d3f480aa25/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e10464d17df3b582745c25cec695cb9558bca2cb6ddb631aee1787fc72c767b2", size = 494586, upload-time = "2026-05-28T12:01:13.051Z" }, - { url = "https://files.pythonhosted.org/packages/80/af/1eeb029bec67582c226b7809172207cd005073af4ebd906e65ff494f4983/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ba05adbf15d994c38ec0b7ab32e858e5110c21e9009a00a86545fd220f84e038", size = 388415, upload-time = "2026-05-28T12:01:14.631Z" }, - { url = "https://files.pythonhosted.org/packages/18/23/ffbe10711c4d766c1cab0557d6906c074f795814863c67b351355d29354a/rpds_py-2026.5.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77c004fdc7b891967106f78ddfd7b076bfe6813c6139c6fff6aed3bcaa960b26", size = 372427, upload-time = "2026-05-28T12:01:16.153Z" }, - { url = "https://files.pythonhosted.org/packages/bd/3a/30ba4a6ad457e5b070c18d742a33fb77d8d922b565cc881f8a5313d63bfe/rpds_py-2026.5.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:83bcf894486c9d78dd290d3c0124ff6dd8875d3025e2090a8ec49fcc37c55fdd", size = 383615, upload-time = "2026-05-28T12:01:17.809Z" }, - { url = "https://files.pythonhosted.org/packages/d3/69/62e242b53ce39c0814bd24e1a6e6eba6c92be716277745f317f9540a2e7b/rpds_py-2026.5.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3df104083952a0e0c6f10de33e440eabe98fb6317d23e1a58c68f6df08d01b9", size = 402786, upload-time = "2026-05-28T12:01:19.419Z" }, - { url = "https://files.pythonhosted.org/packages/38/c1/a770b9c186928a1ed0f7e6d7ae50e7f3950ed23e3f9e366dbc8e38cb55de/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:980450826cf22e133c57e0835070bdd0dd3f73b9b708c3ce223def2cb9469e14", size = 551583, upload-time = "2026-05-28T12:01:21.013Z" }, - { url = "https://files.pythonhosted.org/packages/21/7c/68e8579b95375b70d2a963103c42e705856cdb98569258bd807f4423891c/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:205dde846f24332ab0c1188699a043b8d165b79bb84529ce272c45048ff6be01", size = 616941, upload-time = "2026-05-28T12:01:22.548Z" }, - { url = "https://files.pythonhosted.org/packages/70/a1/a6135aed5730ff03ab957182259987ac11e55fb392a28dc6f0592048a280/rpds_py-2026.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3966b82dd563176396df030f3dd52a6e54cb69b718e95e78bd555ed3d1e0185d", size = 578349, upload-time = "2026-05-28T12:01:24.118Z" }, - { url = "https://files.pythonhosted.org/packages/09/6e/f24201a76a84e6c49d0bdfdfcb735210e21701e9b21c5bfc0ba497dd62f6/rpds_py-2026.5.1-cp315-cp315-win32.whl", hash = "sha256:7818f8d0a415be74d2be3590b0a1c1f463a642f4d0217e7d10602dceef5b79aa", size = 209922, upload-time = "2026-05-28T12:01:25.522Z" }, - { url = "https://files.pythonhosted.org/packages/9e/e4/966bc240bb0485fc265278f6de44d05834bf0b3618886e0b22e33d54c49a/rpds_py-2026.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:b3cc20c0d800af78fd0fac68086e28c1856cec51ea528bb81ea851aa40d39325", size = 226003, upload-time = "2026-05-28T12:01:27.062Z" }, - { url = "https://files.pythonhosted.org/packages/5c/5c/a15a59269cd5e74472734516c73795c15eccfc841b3d4b0228c3f53f19d0/rpds_py-2026.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:3609e9939a8a76cd904cf98a3f1f13b5dc7e150adeaee89e0ea09652ea213e16", size = 221245, upload-time = "2026-05-28T12:01:28.51Z" }, - { url = "https://files.pythonhosted.org/packages/e0/22/135ce03804e179a71ceb13be095deda4a279bc88f7a6b8fa161c5ad44e12/rpds_py-2026.5.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:5d333a7127d4b307601ac37792bee01bb95c867cbfacf21b6375b804d6bbd723", size = 352015, upload-time = "2026-05-28T12:01:30.214Z" }, - { url = "https://files.pythonhosted.org/packages/3b/5f/f1f6d2652eb9d848f6eb369d8db83a2da6249bb49ad2c2a48f45d54538d3/rpds_py-2026.5.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:b5f077b44a4f7808520f66dae234988d867deb9aed9be5da057ce9ba831b2a41", size = 345016, upload-time = "2026-05-28T12:01:31.656Z" }, - { url = "https://files.pythonhosted.org/packages/88/66/b74182775691ea2290c99e52ac8d5db844e56fbec90ce421f107658c8314/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d8f9b7b78c9538fc9e04e82ec0e888ff0c3cffcfad152c77e57cd09351a98a", size = 374775, upload-time = "2026-05-28T12:01:33.136Z" }, - { url = "https://files.pythonhosted.org/packages/ff/8f/15e5a61d9f0a43902d36561d4f07cae6ae9f4716be825159fd72717f33af/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e3a8ae58895ac107ed934a6bf51e5846f95c53b9b940c2c6d310838fd5846358", size = 380270, upload-time = "2026-05-28T12:01:34.574Z" }, - { url = "https://files.pythonhosted.org/packages/02/c3/f859b12763a80540cdf2af0f15b19904cf756a71d7bdd3f82ff3e5b1bbf9/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0957cf3c2b8632ec7aaebffebea8005b353cc2a237b6e2ae3c2cac0820704cfb", size = 495285, upload-time = "2026-05-28T12:01:36.127Z" }, - { url = "https://files.pythonhosted.org/packages/1c/c7/ff27c2ac8411d30b03b1829fd88cae8dad1a4d0da48dd25e57c4038042e6/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c396c1304de421050b3681ea70f371874b54d41b0151e96109758144c231e30b", size = 389581, upload-time = "2026-05-28T12:01:37.635Z" }, - { url = "https://files.pythonhosted.org/packages/6e/67/fe92ee32a6cc05c77228a2f8b1762e7124f386ec20ff83d0757b762d58d0/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad1bff7f666b9598e573815affd666aac6a13a585dde336f843e33350c7fadc", size = 376041, upload-time = "2026-05-28T12:01:39.307Z" }, - { url = "https://files.pythonhosted.org/packages/f8/91/b4d6685c27aba55bd82f25b278be8237038117d05f9659a6213ad3408130/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:656a042550878f12d45752452d47094b7cfe5ad1e9d7b87b5a22ad3ae5ff8015", size = 383946, upload-time = "2026-05-28T12:01:41.043Z" }, - { url = "https://files.pythonhosted.org/packages/bd/79/2c1d832a53c8e0f8e98fc970ec257b950fecd4f62be2ab7182b500a0cbc8/rpds_py-2026.5.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c4bd4f70294737b5206a3e8e30ccadbf8a60301831c8ea23eec5dbeea1ecfa", size = 405526, upload-time = "2026-05-28T12:01:43.032Z" }, - { url = "https://files.pythonhosted.org/packages/78/c4/c98117b03c6a8581ab2c2dfccfe9a5ad82bd8128a3c28b46a6ad2d97c393/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:43bca78665423cabae77146f2fe7ce55272b6c8d55d82cca83effd42c7e13972", size = 551165, upload-time = "2026-05-28T12:01:44.648Z" }, - { url = "https://files.pythonhosted.org/packages/3b/c1/bc479ca069200af730881b1bd525e3114b2b391a351509fcb1b772f28086/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:42d0f20e85e549c870749d0e247f0c10d318a45b7e9676d575d2dcb04a1b2e66", size = 618778, upload-time = "2026-05-28T12:01:46.337Z" }, - { url = "https://files.pythonhosted.org/packages/77/65/38ab2f90df44c2febfb63cc10ced40763d9b4bc94d173e734528663fe7f5/rpds_py-2026.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b1be5c35683684d5331b93600c210e8367c254683d8a6df6bd21bd2da3a334fb", size = 581839, upload-time = "2026-05-28T12:01:48.109Z" }, - { url = "https://files.pythonhosted.org/packages/15/2d/ce1f605fe036aadd460e5822e578c6c7ec3a860936cca37d6e0f299daa77/rpds_py-2026.5.1-cp315-cp315t-win32.whl", hash = "sha256:75808f6c38ce7749bb68cc2770161aae5045e6c6f6781a9782e74b93304399df", size = 207866, upload-time = "2026-05-28T12:01:49.648Z" }, - { url = "https://files.pythonhosted.org/packages/79/cb/966040123eb102371559746908ef2c9471f4d43e17ec9a645a2258dab64b/rpds_py-2026.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:90bd6630002a1c7f09e7843dd79f0d24f3d2897cc25a753480917865d14f15b3", size = 225441, upload-time = "2026-05-28T12:01:51.408Z" }, - { url = "https://files.pythonhosted.org/packages/42/56/3fe0fb34820ff667be791b3a3c22b85e8bcba54e9c832f47438c191fa7be/rpds_py-2026.5.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:edf2765d84e42447f112ad877af8fe1db0089aaec5b28e88d6eab45e7fe99cea", size = 357151, upload-time = "2026-05-28T12:01:53.43Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f2/3eb9ccdb9f143b8c9b003978898cb497f942a324c077401e6b8834238e63/rpds_py-2026.5.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ad3773236e95f7f33991eb125224b7da66f206504d032a253a02da7e134519fb", size = 350195, upload-time = "2026-05-28T12:01:54.901Z" }, - { url = "https://files.pythonhosted.org/packages/a7/24/dbda232bc4f3ed732120692ab0d2c8402cb020516556d8bee622dcef2413/rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a04df86b3f0fade39ec8fd0e0aab089b1da9fbd2b48df778a57ef96f5e7d38df", size = 381850, upload-time = "2026-05-28T12:01:56.601Z" }, - { url = "https://files.pythonhosted.org/packages/40/30/32e769839a358f78810c234f160f2cc21d1e4e47e1c0e0e0d535be5a0219/rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6142dbd80c4df62a5d899f0d616d417f84e0bc8d32526c8e5589019d75d028a7", size = 387899, upload-time = "2026-05-28T12:01:58.212Z" }, - { url = "https://files.pythonhosted.org/packages/ab/86/ec84d243aadb3b34b71dd26a010d0930b2d284ff5fc9a69fec53810ee6fd/rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0b35217adefe87f2fe4db7e9766cabe84744bfe9616d9667be18988928c7f2dc", size = 501618, upload-time = "2026-05-28T12:01:59.888Z" }, - { url = "https://files.pythonhosted.org/packages/74/25/b60e52686bbff777a64f9e4f4d3dd57980dc846913777177a2c92e4937aa/rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b95d5e11fc712b752081183a55a244c03cd00570489edd7014d8899f8ceb8162", size = 394003, upload-time = "2026-05-28T12:02:01.482Z" }, - { url = "https://files.pythonhosted.org/packages/9b/c7/b3a6a588cc2219510ef3f42e207483a93950bedd1e3a0fd4015c95cff9e5/rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:141c9498daf2ace9eda35d2b0e376f9ea8b058d84f2aef4f96fccfd449a2f251", size = 379778, upload-time = "2026-05-28T12:02:03.197Z" }, - { url = "https://files.pythonhosted.org/packages/31/00/c7dba3fc8a3da8cb3f6db1eb3386be4d79c2e97c6890d20eb9ac66ae8c43/rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:6f249f8b860a200ad35193af961183ebe9132710484e6f6ce0cf89fd83c63a9a", size = 392359, upload-time = "2026-05-28T12:02:04.817Z" }, - { url = "https://files.pythonhosted.org/packages/93/dd/472ba494c70753f93745992c99855bee0636daf74e6984e5e003f150316f/rpds_py-2026.5.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e4abbf391a70be864920858bf360f4fb380577c9a0f732438a1996726e2c195b", size = 412820, upload-time = "2026-05-28T12:02:06.401Z" }, - { url = "https://files.pythonhosted.org/packages/1d/6f/93831a3bfe789542ed0c1d0d74b78b440f055d6dc3ea4640eba2d95e6e23/rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:c74005a7bb87752acf351c93897ec63ad77a07a0da7ecad9c050e32e7286ba34", size = 557243, upload-time = "2026-05-28T12:02:08.013Z" }, - { url = "https://files.pythonhosted.org/packages/1f/ff/0b3d604614ffc77522c6b288fdbce68957eb583da1002aa65ba38ac0ee40/rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:8213afbe8a3a906fb9acb2014423fe3359ee783d0bf90995f70623a3217bfa6c", size = 623541, upload-time = "2026-05-28T12:02:09.661Z" }, - { url = "https://files.pythonhosted.org/packages/ea/ea/e7b0251441da9adfeaebcf29601d10f2a1455fcf0772fae9e7e19032bd96/rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8c43a8a973270fd173bf48cdf80bbe66312421cba68d40845034f174f2389049", size = 586326, upload-time = "2026-05-28T12:02:11.47Z" }, -] - -[[package]] -name = "rsa" -version = "4.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, -] - -[[package]] -name = "s3transfer" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e0/1f/12417f7f493fc45e1f9fd5d4a9b6c125cf8d2cf3f8ddbdfab3e76406e9d6/s3transfer-0.18.0.tar.gz", hash = "sha256:3760b8b7ec1315da54048b2d626276732bee4300d054d492d4e1d43e20d4ecbd", size = 160560, upload-time = "2026-05-28T19:39:09.124Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/58/a58fc997655386daa2e25784e30c288aa3e3819e401f77029ee4899fb55a/s3transfer-0.18.0-py3-none-any.whl", hash = "sha256:239c13b09e65ad0346e1be7348b8a202dcad44ac7ea7c6eb858fc881dce739b6", size = 88572, upload-time = "2026-05-28T19:39:07.999Z" }, -] - -[[package]] -name = "secretstorage" -version = "3.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, -] - -[[package]] -name = "setuptools" -version = "82.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, -] - -[[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - -[[package]] -name = "sqlglot" -version = "30.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/64/89299aefc6ebdf4fc899f5dc14c7fcb7eb9da9290a2b4d615ae7ab884b17/sqlglot-30.8.0.tar.gz", hash = "sha256:1c5f93fb742dd9aaa75eee6bb33a637794a858b9a86375fac23a2dc0f7bc127e", size = 5869750, upload-time = "2026-05-13T09:04:38.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/4e/80705091aaf9c95e125d243f0aa871bc9f3670b4c9d963e6bad3b3dce8ff/sqlglot-30.8.0-py3-none-any.whl", hash = "sha256:af903378c331d5b72277a1b41118f07bc3e50cf4478e2d47eed12c96ee6a22a4", size = 687831, upload-time = "2026-05-13T09:04:36.336Z" }, -] - -[[package]] -name = "sse-starlette" -version = "3.4.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "starlette" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f7/2b/58abc2d1fd397e7dde08e947e05c884d8ef2f78d5e2588c17a12d42d6994/sse_starlette-3.4.4.tar.gz", hash = "sha256:07e0fa0460138baf25cdd5fb28683472c3995dc1642225191b3832d62526bcb0", size = 31819, upload-time = "2026-05-12T17:37:17.019Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/67/805710444ea8cc75fbf70b920ed431a560c4bf9c57f7d5a3117213189399/sse_starlette-3.4.4-py3-none-any.whl", hash = "sha256:3f4dd50d8aed2771a091f3a83000323fc3844541c16b4fe585ae2420cc6df973", size = 16514, upload-time = "2026-05-12T17:37:15.601Z" }, -] - -[[package]] -name = "starlette" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, -] - -[[package]] -name = "tomli" -version = "2.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, - { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, - { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, - { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, - { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, - { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, - { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, - { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, - { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, - { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, - { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, - { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, - { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, - { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, - { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, - { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, - { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, - { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, - { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, - { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, - { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, - { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, - { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, - { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, - { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, - { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, - { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, - { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, - { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, - { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, - { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, - { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, - { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, - { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, - { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, - { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, - { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, - { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, - { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, - { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, - { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, - { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, - { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, - { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, - { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, - { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, -] - -[[package]] -name = "toon-format" -version = "0.9.0b1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/15/d23d6d3e36aa4ec96dd5692bc7715fe17015b669e8f0d1c5c7fa906a3ceb/toon_format-0.9.0b1.tar.gz", hash = "sha256:8f391dd6ad9677c78366bd8eb6762d064a2183f67b9b7da1f348fdb6ee8738e7", size = 87398, upload-time = "2025-11-08T19:22:53.059Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/f3/27ab1d982bb81bf9ac5be70b4c774996eb8562b93c77e93c253c22be951f/toon_format-0.9.0b1-py3-none-any.whl", hash = "sha256:efeee919501f91137f017f7bed34789c067f76178111aa872a49bc8653dce3be", size = 36173, upload-time = "2025-11-08T19:22:51.523Z" }, -] - -[[package]] -name = "tox" -version = "4.35.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cachetools" }, - { name = "chardet" }, - { name = "colorama" }, - { name = "filelock" }, - { name = "packaging" }, - { name = "platformdirs" }, - { name = "pluggy" }, - { name = "pyproject-api" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, - { name = "virtualenv" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a9/7c/d2b9d58c7fe3a36224a71c86fc7072936af126425b7af40ca18103f6f691/tox-4.35.0.tar.gz", hash = "sha256:74d2fe33eb37233d506f854196bd7bd7e2fbb79e8d9b4bed214ab3da98740876", size = 205701, upload-time = "2026-02-12T22:47:29.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/5f/8df349c4e9ea0747cfc12a44c2e952c2f7a1a12fb54f36543dae17638a57/tox-4.35.0-py3-none-any.whl", hash = "sha256:282aa2e1f96328ad197ee09878ff241610426cd8ec01e62a04eb51c987da922d", size = 176999, upload-time = "2026-02-12T22:47:27.768Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, -] - -[[package]] -name = "typing-inspection" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, -] - -[[package]] -name = "uncalled-for" -version = "0.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/82/345cc927f7fbdae6065e7768759932fcc827fc20b29b45dfbafa2f1f7da4/uncalled_for-0.3.2.tar.gz", hash = "sha256:89f5dbcd71e2b8f47c030b1fa302e6cce2ec795d1ac565eeb6525c5fe55cb8a2", size = 50032, upload-time = "2026-05-06T13:38:25.204Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/25/2c87754f3a9e692315f7b811244090e68f362979fc8886b3fbd2985a1d8c/uncalled_for-0.3.2-py3-none-any.whl", hash = "sha256:0ff60b142c7d1f8070bde9d42afaa70aedc77dcc10998c227687e9c15713418e", size = 11444, upload-time = "2026-05-06T13:38:24.025Z" }, -] - -[[package]] -name = "urllib3" -version = "1.26.20" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/e8/6ff5e6bc22095cfc59b6ea711b687e2b7ed4bdb373f7eeec370a97d7392f/urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32", size = 307380, upload-time = "2024-08-29T15:43:11.37Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e", size = 144225, upload-time = "2024-08-29T15:43:08.921Z" }, -] - -[[package]] -name = "uvicorn" -version = "0.49.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "h11" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/1f/fa18009dea8469069cca78a4e877a008ab78f08b064bfc9ab891579077ff/uvicorn-0.49.0.tar.gz", hash = "sha256:ebf4271aa580d9de97f93192d4595176df6e91f9aae919ca73e4fc07df1e66a3", size = 91284, upload-time = "2026-06-03T22:01:30.448Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/fa/e1388bbcf24ef3274f45c0c1c7b501fd14971037c1b6ee23610553307497/uvicorn-0.49.0-py3-none-any.whl", hash = "sha256:ba3d14c3ee7e41c6c654c46c9eb489d33213cdd30aa1696eab1374337c13f68f", size = 71376, upload-time = "2026-06-03T22:01:29.037Z" }, -] - -[[package]] -name = "virtualenv" -version = "21.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "distlib" }, - { name = "filelock" }, - { name = "platformdirs" }, - { name = "python-discovery" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e1/0d/4e93c8e6d1001a75763f87d8f5ecda8ebc7f4aa2153dddfaf4ae8892821a/virtualenv-21.4.2.tar.gz", hash = "sha256:38e6ee0a555615c0ea9da2ac7e9998fe8dc3b911dd33ad8eaad2020957653b0c", size = 7613326, upload-time = "2026-05-31T17:01:22.827Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/c4/557dc082be035381b85fdb2b74e21d3d21b57750b74f2b47a32f3a639ff9/virtualenv-21.4.2-py3-none-any.whl", hash = "sha256:854210ca524a1a4d0d744734f4acbc721c3ffe163b85bbf5d56d14d5ae2f0fae", size = 7594079, upload-time = "2026-05-31T17:01:20.735Z" }, -] - -[[package]] -name = "watchfiles" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/5a/2bf22ecb24916983bf1cc0095e7dea2741d14d6553b0d6a2ac8bc96eca93/watchfiles-1.2.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:bb68bf4df85abebe5efddc53cf2075520f243a59868d9b3973278b23e76962a9", size = 400471, upload-time = "2026-05-18T04:31:08.908Z" }, - { url = "https://files.pythonhosted.org/packages/55/70/dea1f6a0e76607841a60fb51af150e70124864673f61704abb62b90cdcc7/watchfiles-1.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c16cb06dd17d43b9d185094268459eac92c9538356f050e55b54e82cf700e1d4", size = 394599, upload-time = "2026-05-18T04:30:19.845Z" }, - { url = "https://files.pythonhosted.org/packages/18/52/752dcc7dc817baef5e89518732925795ce52e36a683a9a3c9fb68b21504e/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77a0feab9af4c021c581f695258c642b3d10c5fd4c676e33a0d8606425d82631", size = 455458, upload-time = "2026-05-18T04:30:29.126Z" }, - { url = "https://files.pythonhosted.org/packages/12/48/366ebbb22fcc504c2f72b45f0b7e72f40a18795cc01752c16066d597b67a/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a16ffe19bf5cf9f5edaa1ad1dd830c5a816e8feec430c522302ab55483a4b994", size = 460513, upload-time = "2026-05-18T04:31:40.85Z" }, - { url = "https://files.pythonhosted.org/packages/ad/44/1f9e1b15e7a729062e0d0c3d0d7225ea4ab98b2267ef87287153be2495fc/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:204f299afcbd65918ab78dbc52626b0ae45e9d8cef403fdbf33ecf9e40eac66e", size = 493616, upload-time = "2026-05-18T04:30:58.47Z" }, - { url = "https://files.pythonhosted.org/packages/7e/55/8b1086dcc8a1d6a697a62767bd7ea368e74c61c6fd171683cfe24a3fe5d2/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11743adfa510bfffebe97659fb280182b5c9b238708f667e866f308c3430dc19", size = 573154, upload-time = "2026-05-18T04:30:37.903Z" }, - { url = "https://files.pythonhosted.org/packages/14/7a/242f400cc77fafa7b18d53d19d9cb64fc6a6f61f28c55913bae7c674d92a/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb72919d93e3a16fc451d3aa3d4b1698423daca1b382d3d959c9ac51297c12a8", size = 467046, upload-time = "2026-05-18T04:30:41.869Z" }, - { url = "https://files.pythonhosted.org/packages/02/c8/79eee650c62d2c186598489814468e389b5def0ebe755399ff645b35b1b2/watchfiles-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62f042afde2dde21ec1d2c1a74361e804673df86f51e418a999c9acfe671b07", size = 457100, upload-time = "2026-05-18T04:31:13.064Z" }, - { url = "https://files.pythonhosted.org/packages/81/36/519f6dbb7a95e4fe7c1513ed25b1520295ef9905a27f1f2226a73892bfb7/watchfiles-1.2.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:027ae72bfdfd254862065d8b3e2a815c6ab9b1853ce41e6648ece84afd34a551", size = 467038, upload-time = "2026-05-18T04:30:32.915Z" }, - { url = "https://files.pythonhosted.org/packages/2f/12/951af6b9f89097e02511122258402cb3578443021930b70cf968d6310dc0/watchfiles-1.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e1cfd51e97e13ff3bd047c140764d277fc9b95b7cb5da59e46a47d167adab310", size = 632563, upload-time = "2026-05-18T04:30:11.539Z" }, - { url = "https://files.pythonhosted.org/packages/28/cc/0cba1f0a6117b7ec117271bdc3cb3a5a252005959755a2c09a745e0942cc/watchfiles-1.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:24b2405c0a46738dd9e1cf7135aa5dbdb9d42d024628651b3b13d5117e99f8df", size = 660851, upload-time = "2026-05-18T04:31:53.186Z" }, - { url = "https://files.pythonhosted.org/packages/d0/f2/26347558cc8bf6877845e66b315f644d03c173906aa09e233a3f4fd23928/watchfiles-1.2.0-cp310-cp310-win32.whl", hash = "sha256:8c520725602756229f045b032a1ff33d7ef0f7404189d62f6c2438cb6d8ef6a1", size = 277023, upload-time = "2026-05-18T04:30:18.825Z" }, - { url = "https://files.pythonhosted.org/packages/6d/68/a5e67b6b68e94f4c1511d61c46c55eba0737583620b6febf194c7b9cc23f/watchfiles-1.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:03b14855c6f35539e2d95c442ae9530a75762f1e26567152b9ed05f96534a74d", size = 290107, upload-time = "2026-05-18T04:32:09.677Z" }, - { url = "https://files.pythonhosted.org/packages/fc/3d/8024c801df84d1587740d0359e7fdd80afeae3d159011f3d5376dd82f18e/watchfiles-1.2.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:704fd259e332e01f9b9c178f4bce9e49027e5587cc2600eeeaf8e76e1c846201", size = 400242, upload-time = "2026-05-18T04:31:19.014Z" }, - { url = "https://files.pythonhosted.org/packages/87/5b/f4dfd45323e949984a3a7f9dc31d1cbb049921e7d98253488dda72ccdaa9/watchfiles-1.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6543cf55d170003296d185c0af981f3e1311564907e1f4e08671fc7693a890a5", size = 394562, upload-time = "2026-05-18T04:30:08.46Z" }, - { url = "https://files.pythonhosted.org/packages/98/d8/19483ef075d601c409bce8bcbb5c0f81a10876fff870400568f08ce484a1/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89d8c2394a065ca86f5d2910ff263ae67c127e1376ccc4f9fc35c71db879f80a", size = 456611, upload-time = "2026-05-18T04:30:45.723Z" }, - { url = "https://files.pythonhosted.org/packages/b1/6a/cc81fbe7ee42f2f22e661a6e12def7807e01b14b2f39e0ff83fd373fd307/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:772b80df316480d894a0e3165fdd19cf77f5d17f9a787f94029465ad0e3529d1", size = 461379, upload-time = "2026-05-18T04:31:29.292Z" }, - { url = "https://files.pythonhosted.org/packages/b1/57/7e669002082c0a0f4fb5113bb70125f7110124b846b0a11bc5ae8e90eac1/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d158cd89df6053823533e06fb1d73c549133bff5f0396170c0e53d9559340717", size = 493556, upload-time = "2026-05-18T04:30:05.44Z" }, - { url = "https://files.pythonhosted.org/packages/45/7d/f60a2b19807b21fe8281f3a8da4f59eef0d5f96825ac4680ba2d4f2ebf91/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d516b3283a758e087841aedb8031549fb41ced08f3db10aa6d2bf32dc042525b", size = 575255, upload-time = "2026-05-18T04:30:40.568Z" }, - { url = "https://files.pythonhosted.org/packages/bd/49/77f5b5e6efbcd57482f74948ebb1b97e5c0046d6b61475042d830c84b3ff/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53b2290c92e0506d102cd448fbc610d87079553f86caa39d67440856a8b8bba5", size = 467052, upload-time = "2026-05-18T04:31:17.942Z" }, - { url = "https://files.pythonhosted.org/packages/ee/5a/73e2959af1b97fd5d556f9a8bdba017be23ceeef731869d5eaa0a753d5a3/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a711b51aec4370d0dcda5b6c09463206f133a5759341d7744b953a7b62e1100e", size = 456858, upload-time = "2026-05-18T04:30:30.182Z" }, - { url = "https://files.pythonhosted.org/packages/50/57/1bc8c27fad7e6c19bddee15d276dbb6ab72480ec01c127afff1673aee417/watchfiles-1.2.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:e2ca07fa7d89195ec0865d3d285666286740bfa83d83e5cee204043a31ecc165", size = 467579, upload-time = "2026-05-18T04:32:15.897Z" }, - { url = "https://files.pythonhosted.org/packages/09/6c/3c2e44edba3553c5e3c3b8c8a2a6dee6b9e12ae2cf4bd2378bebf9dc3038/watchfiles-1.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e0618518f282c4ebff60f5e5b1247b6d91bb8b9f4476947563a1e74acc66f3c6", size = 633253, upload-time = "2026-05-18T04:31:37.123Z" }, - { url = "https://files.pythonhosted.org/packages/30/c2/d8c84a882ab39bbefcc4915ab3e91830b7a7e990c5570b0b69075aba3faf/watchfiles-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d191c054d0715c3c95c99df9b8dbf6fd096d8c1e021e8f212e1bd8bc444ccb5", size = 660713, upload-time = "2026-05-18T04:31:24.62Z" }, - { url = "https://files.pythonhosted.org/packages/a9/07/f97736a5fc605364fe67b25e9fa4a6965dfd4840d50c406ada507e9d735f/watchfiles-1.2.0-cp311-cp311-win32.whl", hash = "sha256:9342472aff9b093c5acd4f6d8f70ae0937964ab56542502bcf5579782da69ae8", size = 277222, upload-time = "2026-05-18T04:31:21.131Z" }, - { url = "https://files.pythonhosted.org/packages/cf/99/2b04981977fc2608afd60360d928c6aecf6b950292ca221d98f4005f6694/watchfiles-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:dbd6c97045dad81227c8d040173da044c1de08de64a5ea8b555da4aee1d5fa22", size = 290274, upload-time = "2026-05-18T04:31:45.966Z" }, - { url = "https://files.pythonhosted.org/packages/3c/74/f7f58a7075ee9cf612b0cfcddb78b8cd8234f0742d6f0075cf0da2dde1c6/watchfiles-1.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:57a2d9fa4fb4c2ecae57b13dfff2c7ab53e21a2ba674fe9f05506680fcdcc0d7", size = 283460, upload-time = "2026-05-18T04:31:39.126Z" }, - { url = "https://files.pythonhosted.org/packages/b8/2f/e42c992d2afda3108ea1c02acecc991b9f31d05c14adc2a7cee9ee211fc4/watchfiles-1.2.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bc13eb17538be00c874699dc0abe4ee2bc8d50bb1166a6b9e175ef3fd7eb8f26", size = 400115, upload-time = "2026-05-18T04:32:02.06Z" }, - { url = "https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c", size = 393659, upload-time = "2026-05-18T04:30:50.951Z" }, - { url = "https://files.pythonhosted.org/packages/13/01/b32a967c56fb3e3e5be3db52c3d3b87fa4513aa367d8ed1ad96d42952e5f/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc", size = 453207, upload-time = "2026-05-18T04:31:04.231Z" }, - { url = "https://files.pythonhosted.org/packages/04/98/97557a812180338cb1abd32e1cffcc4588f59b5f23e0cb006b2ba95ba64a/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0", size = 459273, upload-time = "2026-05-18T04:31:50.377Z" }, - { url = "https://files.pythonhosted.org/packages/e8/a8/b4b08dcb7653b8087c6586f7ce649505900e866bbcfe40dc9587af02e686/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c", size = 489927, upload-time = "2026-05-18T04:31:42.485Z" }, - { url = "https://files.pythonhosted.org/packages/50/94/3dceea03545d2e5ddfd839f0ddd5e1cecbf1697b5a428d5ba11cef6af95d/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01", size = 570476, upload-time = "2026-05-18T04:31:03.071Z" }, - { url = "https://files.pythonhosted.org/packages/cc/f2/d39a5450c3532092b91f81d274360e613c2371bc874a89c7a1a3c5e8d138/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8", size = 465650, upload-time = "2026-05-18T04:30:12.701Z" }, - { url = "https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5", size = 456398, upload-time = "2026-05-18T04:30:13.784Z" }, - { url = "https://files.pythonhosted.org/packages/0d/64/982ef4a4e5bab5b6e5b6becc8cd5e732f6130a78b855f0abec6439a9a135/watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d", size = 465140, upload-time = "2026-05-18T04:31:52.111Z" }, - { url = "https://files.pythonhosted.org/packages/a0/0c/95282abf4ed680b6096010bcfc30c5fa7a041fc5aa5a2ad17a2cc6c75bba/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c", size = 630259, upload-time = "2026-05-18T04:31:25.676Z" }, - { url = "https://files.pythonhosted.org/packages/30/45/607c1de1530c4bdcf2cf1d1ecc2505ddba5d96bd43ba9f2b0e79876f850f/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906", size = 659859, upload-time = "2026-05-18T04:30:24.333Z" }, - { url = "https://files.pythonhosted.org/packages/fa/08/d9e2e0f9e8e6791d33aefc694ad7eefa7f901f63caff84a81ded38692f9c/watchfiles-1.2.0-cp312-cp312-win32.whl", hash = "sha256:7a2cffd17d27d2ecbb310c2b1d8174f222a5495b1a721894afa88ec11e25b898", size = 275480, upload-time = "2026-05-18T04:30:31.307Z" }, - { url = "https://files.pythonhosted.org/packages/1c/e6/9d42569c0102645cc8cea5d8c7d8a1e9d4ada2cb7f05f75e554b8aa2202a/watchfiles-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:f155b3a1b2a5fc89cdc70d47ee5d54e3b75e88efa34982028a35daef9ba00379", size = 288718, upload-time = "2026-05-18T04:32:10.745Z" }, - { url = "https://files.pythonhosted.org/packages/0a/26/88e0dc6ee3898169d7fa22bb6a69cabf2502d2ee25cb8c876d1262d204f8/watchfiles-1.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:8fa585ede612ee9f9e91b18bebf9ba11b9ae29a4e3a0d0cf6fca3e382133f0d5", size = 281026, upload-time = "2026-05-18T04:30:22.23Z" }, - { url = "https://files.pythonhosted.org/packages/d1/4d/70a7feced9f87e2ff26dba42667290f41694fc64646c67261fbb8cab5d5c/watchfiles-1.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:01ea8d66f0693b9b60a6541c8d10263091ca9a9060d242f3c1f3143f9aad2c98", size = 399730, upload-time = "2026-05-18T04:31:38.162Z" }, - { url = "https://files.pythonhosted.org/packages/31/3a/0da302f2307aee316922806ebd5726c542cbd787c938271cf14a074c7daf/watchfiles-1.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7ba0480b9a74af058f43b337e937a451e109295c420916d68ad24e3dc02f5e44", size = 392842, upload-time = "2026-05-18T04:30:27.051Z" }, - { url = "https://files.pythonhosted.org/packages/db/ef/d5bdb705c224dbc256aa0c1ec47bf4e61ec52558f2afb44a71a1fe4d7015/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f34e26a19f91f710c08e0183429f0d1d15df734e6bc78c31e77b9ea9c433658", size = 452989, upload-time = "2026-05-18T04:31:11.945Z" }, - { url = "https://files.pythonhosted.org/packages/71/29/5495f2c1661949ef7a35e4d71111d129cfe7606414a26887a919d0a55406/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4e77f6a55f858504069abd35d336a637555c09bca453dde1ee1e5ada8a6a1fb", size = 458978, upload-time = "2026-05-18T04:30:52.606Z" }, - { url = "https://files.pythonhosted.org/packages/d5/8c/7f9c07c433811c2fffd93e13fdfb7135de9aab5f2ae41be08960fa0047dc/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cb4d80e212f116474a545c21c912b445f16bb0cef9e6a73a498164223e14e2f", size = 490248, upload-time = "2026-05-18T04:31:36.003Z" }, - { url = "https://files.pythonhosted.org/packages/3c/11/d93632febc52fbc21be90231bb7c17fd5387f46c9076fd40a5f9c2ae6910/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b974946a10af379d425e2eef5b62f5c6ebeaccf91d45eaad6f5b27ecd4f91aa0", size = 571847, upload-time = "2026-05-18T04:31:10.862Z" }, - { url = "https://files.pythonhosted.org/packages/55/b4/383173e73aabb07ad1d9c7aa859d95437ac46a6d6a1e11005facda0c9d19/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86bc13c25a8d1fcd70b51d0ce7c9b65e90de5666fcbfd3e34957cc73ee19aeb5", size = 465974, upload-time = "2026-05-18T04:30:17.006Z" }, - { url = "https://files.pythonhosted.org/packages/a7/6c/89b1a230a78f57c52dd8893adb1f92f94411721b6ec12596c56d98c74356/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca148d73dea36c9763aaa351e4d7a51780ec1584217c45276f4fe8239c768b71", size = 454782, upload-time = "2026-05-18T04:30:35.656Z" }, - { url = "https://files.pythonhosted.org/packages/24/62/1732118367cfff0a9fce3bf62ff4bfded09ef5df21d9d446b858b3f70a96/watchfiles-1.2.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:c525543d91961c6955b2636b308569e84a1d1c5f5f2932041ab9ef46422f43e3", size = 465182, upload-time = "2026-05-18T04:30:20.846Z" }, - { url = "https://files.pythonhosted.org/packages/28/96/716f7e5f51339bf22963f3345f9f27d7f3b30e2eadc597e257c881dd3c53/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a204794696ffb8f9b10fba6f7cb5216d42f3b2b71860ccac6b6e42f5f10973b0", size = 629841, upload-time = "2026-05-18T04:31:05.397Z" }, - { url = "https://files.pythonhosted.org/packages/4c/fe/c40783950fd771ccf66ab3ec2722d188a9af1c7f96c6e811f36e40c6e03f/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:10d86db20695afe7997ac9e1717637d6714a8d0220458c33f3d2061f54cec427", size = 658028, upload-time = "2026-05-18T04:31:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/71/72/4508db1856d1d87fcbb3b63f4839bab1b5682cb0e8d224d122263c09654a/watchfiles-1.2.0-cp313-cp313-win32.whl", hash = "sha256:eb283ee99e21ad6443c8cdb06ac5b34b1308c329cbdf03fa02b445363714c799", size = 275183, upload-time = "2026-05-18T04:30:59.57Z" }, - { url = "https://files.pythonhosted.org/packages/f9/36/14b76ca57652e5cc5fd1c11f32a261292c08a0d19a00351013c2549cbfb2/watchfiles-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:a0f27f01bee51861392bb6b7c4fdb290b27d1eb194e9e28788d68102a0e898d9", size = 288059, upload-time = "2026-05-18T04:32:07.937Z" }, - { url = "https://files.pythonhosted.org/packages/1b/8d/0a85e395398d8d20fadfe5c5d32c726eee17a519e78fb356f2cf7531bffe/watchfiles-1.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:3651aa7058595e9cfb75d35dd5ada2bf9f48a5b8a0f3562821d3e210c507e077", size = 280186, upload-time = "2026-05-18T04:31:54.484Z" }, - { url = "https://files.pythonhosted.org/packages/37/68/36db056f1fdcc5f07302f56e631774d6835bcd6fa3ace402304621d5f9e5/watchfiles-1.2.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:faea288b6f0ab1902ef08f4ca6de005dccf856c4e0c4f21b8c5fce02d90a1b08", size = 399031, upload-time = "2026-05-18T04:30:44.576Z" }, - { url = "https://files.pythonhosted.org/packages/c1/64/01a9d6f66a82a5c101ce939274106cc72759d62427e153f01edd2b9f87c2/watchfiles-1.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:01859b11fd9fbca670f4d5da00fbac282cfea9bd67a2125d8b2833a3b5617ea9", size = 391205, upload-time = "2026-05-18T04:30:25.413Z" }, - { url = "https://files.pythonhosted.org/packages/84/2c/0a44fe058cb4bb7b8ede6b6670698bbb7c0400740e378d00022189b7b31d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fff610d7bb2256a317bb1e96f0d7862c7aa8076733ee5df0fd41bbe76a24a4f4", size = 451892, upload-time = "2026-05-18T04:32:14.005Z" }, - { url = "https://files.pythonhosted.org/packages/67/a1/351e0d56cd35e6488b5c8b4fb11a809a5bc923e8fe8fed9faf8920be0c89/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b141a4891c995a039cd89e9a49e62df1dc8a559a5d1a6e4c7106d16c12777a55", size = 458867, upload-time = "2026-05-18T04:31:22.279Z" }, - { url = "https://files.pythonhosted.org/packages/d5/7d/9d09605187f1b838998624049fcf8bf47b73c1a3b76901fcac1782f62277/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22943b7770483f6ea0721c6b11d022947a98eb0acae14694de034f4d0d38925", size = 490217, upload-time = "2026-05-18T04:31:43.657Z" }, - { url = "https://files.pythonhosted.org/packages/60/5d/a17a16eccb182f04188cd308ec24b1a71a9b5c4e7098269cf35d9fa56d02/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bc6195825b7dcd217968bb1f801a60fd4c16e8eeab5bedc7fe917d7d5995ab4", size = 571458, upload-time = "2026-05-18T04:32:11.875Z" }, - { url = "https://files.pythonhosted.org/packages/d3/3d/4dd457062083ab1938e5dfd45032eb425cee2ac817287ca8ff4356183e5d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4a4b147f5dca2a5d325a06a832fb43f345751adfbc63204aec30e0d9ca965a2", size = 464707, upload-time = "2026-05-18T04:30:43.492Z" }, - { url = "https://files.pythonhosted.org/packages/c6/71/ea8c57b128f5383de74d0c7d2d9c57ad7c9a65a930c451bd25d524b295b7/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4543579a9bdb0c9560039b4ffddbdb39545707659fbc430ce4c10f3f68d557f9", size = 454663, upload-time = "2026-05-18T04:30:16.061Z" }, - { url = "https://files.pythonhosted.org/packages/53/fd/2e812bf938406d7db351f0703ddd3fc6c061cf30d96153a77bc79a943a44/watchfiles-1.2.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:20aa0e708b920bde876a4aa82dc7dd6ebea228a63a67cda6632c2fc87b787efa", size = 463537, upload-time = "2026-05-18T04:31:44.9Z" }, - { url = "https://files.pythonhosted.org/packages/86/56/d17a7f1dd1bc3035f1072694a551301272f1739c2d8e319c927cb9e29b38/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:d413349d565dab74297f2a63e84a097936be69bf8f3b3801f27f380e32040f44", size = 629194, upload-time = "2026-05-18T04:31:14.141Z" }, - { url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" }, - { url = "https://files.pythonhosted.org/packages/e7/54/a9c7ea9a82a4ac65e7004c0a03920b5cdd2f9c3b678757d9cd425aa51d53/watchfiles-1.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b8c8358484d5fa12ef34f05b7f4168eaf1932f408725ff6d023c33ec17bd79d4", size = 400205, upload-time = "2026-05-18T04:32:05.153Z" }, - { url = "https://files.pythonhosted.org/packages/aa/5d/c9ab3534374a4a67450696905d6ef16a04405448b8dc52bd752ae50423d4/watchfiles-1.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9f04b092229ad2c50126dd3c922c8822e51e605993764a33058d4a791ab42281", size = 392508, upload-time = "2026-05-18T04:30:54.849Z" }, - { url = "https://files.pythonhosted.org/packages/26/ca/1ad30103535cf0cecd7b993e8d50edc5351b1820e38f2d22e3df58962feb/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7ce236284f002a156f70add88efe5c70879cccbb658be0822c54b1306fc09d", size = 452448, upload-time = "2026-05-18T04:30:53.727Z" }, - { url = "https://files.pythonhosted.org/packages/37/a1/ceee2cdf2afbd715fa07758d39c9859513eae411b23196f7fd039e5feedd/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b9909cc2b48468b575eefa944919e1fe8a36c5849d5c7c168f80a8c1db69398e", size = 459605, upload-time = "2026-05-18T04:30:23.312Z" }, - { url = "https://files.pythonhosted.org/packages/e8/f6/421e30fd1cb3907a84ed92ab3f1983e37ba2dca015e9a894a048418417a2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a37faaed405c67e28e6be45a1fa4f206ef5a2860f27c237db9fa30704c38242", size = 490757, upload-time = "2026-05-18T04:30:47.358Z" }, - { url = "https://files.pythonhosted.org/packages/41/b0/55ed1b97ed08be7bba6f9a541cac15f2a858e1d74d2b07b6da70a82aab00/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9649193aa27bd9ff2e80ff29bfaa93085496c7a3a377592823cc58b77ee88add", size = 568672, upload-time = "2026-05-18T04:30:38.915Z" }, - { url = "https://files.pythonhosted.org/packages/d1/cf/d8ae8a80dd7bafab395ea7681c10237311bbf34d37704a8c744e7cf31fc7/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e4ff8e37f99cf1da89e255e07c9c4b37c214038c4283707bdec308cb1b0ea1f", size = 464197, upload-time = "2026-05-18T04:30:09.914Z" }, - { url = "https://files.pythonhosted.org/packages/7c/8a/3076c496ca8dafe0e8cd03fcebdfc47be4b1174b4e5b24ff6e396e6b3af2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:054dc20fd2e3132b4c3883b4a00d72fd6e1f56fdaf89fccd12e8057d74cd74d7", size = 453181, upload-time = "2026-05-18T04:30:14.829Z" }, - { url = "https://files.pythonhosted.org/packages/e5/10/9745e17c98e7b8a86454df0a3c7b5686bd650383f1e9f26e4ebcbd6cc0c0/watchfiles-1.2.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e140ed30ebde76796b686e67c182cff10ea2fbab186fafd1560f74bb5a473a6e", size = 465109, upload-time = "2026-05-18T04:30:28.123Z" }, - { url = "https://files.pythonhosted.org/packages/8f/95/8ef4a95481d3e0cb52d62a06fa6e972e81424be2d9698b91a2fecca9904c/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:bb7e52ecf68ba46d22df23467b87cffeb2146908aa523ebfe803019618cfda06", size = 630653, upload-time = "2026-05-18T04:31:49.304Z" }, - { url = "https://files.pythonhosted.org/packages/fd/e4/3b3bf36b0f829b50c6ebcb8d031583863c59f923d6a6af3d485e470d0fac/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:23282a321c8baf9b3a3c4afff673f9fe65eb7fdc2338d765ccad9d3d1916a5ba", size = 657838, upload-time = "2026-05-18T04:31:06.497Z" }, - { url = "https://files.pythonhosted.org/packages/21/b1/6cbbb50c1f3002ab568777d44aa21206dfb8807a840990c4037523b51812/watchfiles-1.2.0-cp314-cp314-win32.whl", hash = "sha256:c0db965c5f79aa49fe672d297cf1febc5ad149b658594944f49a54a2b96270a7", size = 275108, upload-time = "2026-05-18T04:30:06.891Z" }, - { url = "https://files.pythonhosted.org/packages/92/45/190ce6db8dcb4536682cf75d3889ff1a27182a58cb519d343cb6d9ea63d8/watchfiles-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:71283b39fd17e5408eb123bd37aeecfd9d54c81fc184421943208aadb879d103", size = 288441, upload-time = "2026-05-18T04:32:12.901Z" }, - { url = "https://files.pythonhosted.org/packages/74/0d/3eae1c2313ab08378431d907c3f8095ecca00f3eda33111cf4f0f2591799/watchfiles-1.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:c5c19526f4e54a00f2666a6c0e9e40d582c09e865055ea7378bf0009aab857b3", size = 280684, upload-time = "2026-05-18T04:31:26.902Z" }, - { url = "https://files.pythonhosted.org/packages/b1/75/fb64e6c25d6b5ca636d03df34ffb1c6e9873303e76d27967e045f8df088f/watchfiles-1.2.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:d73a585accffa5ae39c17264c36ec3166d2fad7000c780f5ef83b2722afb9dd2", size = 398857, upload-time = "2026-05-18T04:32:17.108Z" }, - { url = "https://files.pythonhosted.org/packages/73/4e/9f7adf01754cbf81843722ccfec169d8f26c69778281a302855cecd2ee08/watchfiles-1.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ae99b14c5f21e026e0e9d96f40e07d8570ebee6cafd9d8fc318354606daa7a28", size = 392413, upload-time = "2026-05-18T04:31:07.911Z" }, - { url = "https://files.pythonhosted.org/packages/47/c8/bec626bcc2d69f44b9acb24ce7d60ed7b16b73628eea747fcbd169d8edda/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4429f3b105524a10b72c3a819b091c495d2811d419c1e1e8df773a5a5974f831", size = 452409, upload-time = "2026-05-18T04:31:20.142Z" }, - { url = "https://files.pythonhosted.org/packages/00/b7/b6362068e81e7c556d155a34c35d40ac3ef42d747b06d7f6e5bf58e359c2/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:43d818978d06062d9b22c4fab2ebe44cf5213d42dc8e62bda8c2760cfa2eeb33", size = 458827, upload-time = "2026-05-18T04:32:06.219Z" }, - { url = "https://files.pythonhosted.org/packages/67/f8/9a813fa42afb1e0b4625e75f0479826644d3ee8dc287e093799bc01f390c/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9f732dc58b2dbe69e464ccf8fff7a03b0dd0be439da4c0720d3558527d3d6b4", size = 490104, upload-time = "2026-05-18T04:31:56.034Z" }, - { url = "https://files.pythonhosted.org/packages/2f/bf/27dfb6094ca4c9aad21298b5525b6c53cb36121ee454331d05161e58d130/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f200104103feb097de4cab8fe4f5dd18a2026934c7dea98c55a2f5fd6d5a33b", size = 571360, upload-time = "2026-05-18T04:31:57.133Z" }, - { url = "https://files.pythonhosted.org/packages/fb/39/44a096d67270ea93df91d33877dbe91fbda3aa4f8ec2edf799d93eda8736/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:63ac26eefbf4af1741247d6fb68b11c49a25b2f7413fbd318a83a12aaa9cf666", size = 464644, upload-time = "2026-05-18T04:30:57.33Z" }, - { url = "https://files.pythonhosted.org/packages/0e/80/c7472203bad6268e3ef1ad260739704847898938ad7ea8b63a5131f46b50/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c4997d4e4a55f0d02b6cde327322daf3a0400e5df6c6b15948994bf72497925", size = 454771, upload-time = "2026-05-18T04:30:48.736Z" }, - { url = "https://files.pythonhosted.org/packages/51/cf/3b10b268b4b7f0fc26e9debb5eef1998b515887840f444cd3ec80c688755/watchfiles-1.2.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:4c887eba18b7945ac73067a8b4a66f21cd46c2539b2bc68588f7be6c7eb6d26b", size = 463494, upload-time = "2026-05-18T04:31:33.826Z" }, - { url = "https://files.pythonhosted.org/packages/3d/3e/a4302545cd589262a0dc7d140e86f7688eba3f9c72776c27f7e23b8864c4/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3416ff151bb6b5a8d8d11664974fbef4d9305b9b2957839ab5a270468fd8df30", size = 629383, upload-time = "2026-05-18T04:31:15.596Z" }, - { url = "https://files.pythonhosted.org/packages/db/99/d5649df0a9a410d45b7c882304d0b790903ac9b6e8f2cfd12114e0c6b9f2/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:0e831a271c035d89789cffc386b6aa1375f39f1cd25eb7ca0997e4970d152fc5", size = 656093, upload-time = "2026-05-18T04:31:58.707Z" }, - { url = "https://files.pythonhosted.org/packages/92/b9/362702539275019a54dd2e94511b31a9b89c5f9e6a21966de7eb692549fc/watchfiles-1.2.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:37a6721cdf3f65dbb13aa9503510ccb4451603ac837e44d265d7992a597e1374", size = 400109, upload-time = "2026-05-18T04:31:16.879Z" }, - { url = "https://files.pythonhosted.org/packages/8f/75/71d5ba62db781e5587bded1d944c675374bc4aa37ff33d5018d98e8b6538/watchfiles-1.2.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2b37d10b5a63bd4d87e18472d80fa525bd670586fae62e5dd580452764879b65", size = 392167, upload-time = "2026-05-18T04:31:28.058Z" }, - { url = "https://files.pythonhosted.org/packages/3c/01/c66dd95d0423fe30d31820e2d1d5bda773764131bbb6ac0cb1cf303ac328/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a105bc2283f67e8fbec74253ec2d94925de92ed72c0393f1206bf326b7b7b69", size = 452372, upload-time = "2026-05-18T04:31:00.836Z" }, - { url = "https://files.pythonhosted.org/packages/91/15/2fe99557e72f85627c6a8eed50d889e8d101623e060a22ad75b875cb932d/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5327989a465505f05cfe06f04fa9d0c2fd5432bb243e10e6f012b1bdca3c8579", size = 459596, upload-time = "2026-05-18T04:31:34.96Z" }, - { url = "https://files.pythonhosted.org/packages/ed/23/d4acfa0023367428ed48351b3b9b267893037b6cadae55620c61c24bcfd4/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecb47f183a8025b2aa18b546725c3657e542112ae9c0613a2af79b4fa8d04ad7", size = 490869, upload-time = "2026-05-18T04:31:59.923Z" }, - { url = "https://files.pythonhosted.org/packages/a4/5f/3164cbdce06c9fb95c4f7b9e2f9760b5e2797af43a9ecc317ef42a23a278/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8520a4ab0e37f770afc34459c4f8f7019e153f9124dc101c15538365875d1ab2", size = 571641, upload-time = "2026-05-18T04:32:00.948Z" }, - { url = "https://files.pythonhosted.org/packages/41/e6/85d3731c55e65cd7690f3f803d24c139588aaf863e4bf2148fe7a7fa1a19/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71cd71740ed2c15211ebb237ced4e39a1cdf6f80566e5fe95428da1626f4fde6", size = 464444, upload-time = "2026-05-18T04:30:34.298Z" }, - { url = "https://files.pythonhosted.org/packages/f4/7d/562641012b8b09872742c3b8adf9629ec479fd78f8d68ae4a0c13da8add6/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f88af53d6ddaf72179ef613ddc905e6f4785f712b49b80b3bef9f3525e6194b4", size = 453593, upload-time = "2026-05-18T04:31:23.464Z" }, - { url = "https://files.pythonhosted.org/packages/56/fe/cb8ef3d6f929d14158fdaaad9925985b7310abc9384dcd4d82dd0016fb59/watchfiles-1.2.0-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:cee9d5efd929efdac5f7e58f72b3376f676b64050a91c5b99a7094c5b2317488", size = 465096, upload-time = "2026-05-18T04:31:30.384Z" }, - { url = "https://files.pythonhosted.org/packages/25/91/80908e835e100527a9267147b08c0eee1fa6ab0ffec15edc04d1d44885f7/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:b718bf356bbc15e559bd8ef41782b573b8ae0e3f177ab244b440568d7ea02cfb", size = 630638, upload-time = "2026-05-18T04:30:49.89Z" }, - { url = "https://files.pythonhosted.org/packages/46/4b/95ab2f256bb4af3cb2eb23b9317bda984ee6e0f11733a5c004a6c95b06e3/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:922c0e019fe68b3ae392965a766b02a71ba1168c932cebc3733cd52c5fe5b377", size = 657684, upload-time = "2026-05-18T04:31:32.027Z" }, - { url = "https://files.pythonhosted.org/packages/23/f4/7513ef1e85fc4c6331b59479d6d72661fc391fbe543678052ac72c8b6c19/watchfiles-1.2.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4674d49eb94706dfe666c069fc0a1b646ffcf920473492e209f6d5f60d3f0cc2", size = 403050, upload-time = "2026-05-18T04:30:36.753Z" }, - { url = "https://files.pythonhosted.org/packages/27/0b/a54103cfd732bb703c7a749222011a0483ef3705948dae3b203158601119/watchfiles-1.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:094b9b70103d4e963499bdea001ee3c2697b144cd9ae6218a62c0f89ec9e31db", size = 396629, upload-time = "2026-05-18T04:32:03.268Z" }, - { url = "https://files.pythonhosted.org/packages/5e/2c/73f31a3b893886206c3f54d73e8ad8dee58cdb2f69ad2622e0a8a9e07f4e/watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0ef001f8c25ad0fa9529f914c1600647ecd0f542d11c19b7894768c67b6acb7", size = 457318, upload-time = "2026-05-18T04:31:01.932Z" }, - { url = "https://files.pythonhosted.org/packages/e9/f9/45d021e4a5cc7b9dd567f7cbb06d3b75f751a690063fb6cc7ec60f4e46b7/watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a88fc94e647bc4eec523f1caa540258eb71d14278b9daf72fa1e2658a98df0f0", size = 457771, upload-time = "2026-05-18T04:30:56.331Z" }, -] - -[[package]] -name = "websockets" -version = "16.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/04/24/4b2031d72e840ce4c1ccb255f693b15c334757fc50023e4db9537080b8c4/websockets-16.0.tar.gz", hash = "sha256:5f6261a5e56e8d5c42a4497b364ea24d94d9563e8fbd44e78ac40879c60179b5", size = 179346, upload-time = "2026-01-10T09:23:47.181Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/74/221f58decd852f4b59cc3354cccaf87e8ef695fede361d03dc9a7396573b/websockets-16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a", size = 177343, upload-time = "2026-01-10T09:22:21.28Z" }, - { url = "https://files.pythonhosted.org/packages/19/0f/22ef6107ee52ab7f0b710d55d36f5a5d3ef19e8a205541a6d7ffa7994e5a/websockets-16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0", size = 175021, upload-time = "2026-01-10T09:22:22.696Z" }, - { url = "https://files.pythonhosted.org/packages/10/40/904a4cb30d9b61c0e278899bf36342e9b0208eb3c470324a9ecbaac2a30f/websockets-16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:583b7c42688636f930688d712885cf1531326ee05effd982028212ccc13e5957", size = 175320, upload-time = "2026-01-10T09:22:23.94Z" }, - { url = "https://files.pythonhosted.org/packages/9d/2f/4b3ca7e106bc608744b1cdae041e005e446124bebb037b18799c2d356864/websockets-16.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7d837379b647c0c4c2355c2499723f82f1635fd2c26510e1f587d89bc2199e72", size = 183815, upload-time = "2026-01-10T09:22:25.469Z" }, - { url = "https://files.pythonhosted.org/packages/86/26/d40eaa2a46d4302becec8d15b0fc5e45bdde05191e7628405a19cf491ccd/websockets-16.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df57afc692e517a85e65b72e165356ed1df12386ecb879ad5693be08fac65dde", size = 185054, upload-time = "2026-01-10T09:22:27.101Z" }, - { url = "https://files.pythonhosted.org/packages/b0/ba/6500a0efc94f7373ee8fefa8c271acdfd4dca8bd49a90d4be7ccabfc397e/websockets-16.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2b9f1e0d69bc60a4a87349d50c09a037a2607918746f07de04df9e43252c77a3", size = 184565, upload-time = "2026-01-10T09:22:28.293Z" }, - { url = "https://files.pythonhosted.org/packages/04/b4/96bf2cee7c8d8102389374a2616200574f5f01128d1082f44102140344cc/websockets-16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:335c23addf3d5e6a8633f9f8eda77efad001671e80b95c491dd0924587ece0b3", size = 183848, upload-time = "2026-01-10T09:22:30.394Z" }, - { url = "https://files.pythonhosted.org/packages/02/8e/81f40fb00fd125357814e8c3025738fc4ffc3da4b6b4a4472a82ba304b41/websockets-16.0-cp310-cp310-win32.whl", hash = "sha256:37b31c1623c6605e4c00d466c9d633f9b812ea430c11c8a278774a1fde1acfa9", size = 178249, upload-time = "2026-01-10T09:22:32.083Z" }, - { url = "https://files.pythonhosted.org/packages/b4/5f/7e40efe8df57db9b91c88a43690ac66f7b7aa73a11aa6a66b927e44f26fa/websockets-16.0-cp310-cp310-win_amd64.whl", hash = "sha256:8e1dab317b6e77424356e11e99a432b7cb2f3ec8c5ab4dabbcee6add48f72b35", size = 178685, upload-time = "2026-01-10T09:22:33.345Z" }, - { url = "https://files.pythonhosted.org/packages/f2/db/de907251b4ff46ae804ad0409809504153b3f30984daf82a1d84a9875830/websockets-16.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:31a52addea25187bde0797a97d6fc3d2f92b6f72a9370792d65a6e84615ac8a8", size = 177340, upload-time = "2026-01-10T09:22:34.539Z" }, - { url = "https://files.pythonhosted.org/packages/f3/fa/abe89019d8d8815c8781e90d697dec52523fb8ebe308bf11664e8de1877e/websockets-16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:417b28978cdccab24f46400586d128366313e8a96312e4b9362a4af504f3bbad", size = 175022, upload-time = "2026-01-10T09:22:36.332Z" }, - { url = "https://files.pythonhosted.org/packages/58/5d/88ea17ed1ded2079358b40d31d48abe90a73c9e5819dbcde1606e991e2ad/websockets-16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af80d74d4edfa3cb9ed973a0a5ba2b2a549371f8a741e0800cb07becdd20f23d", size = 175319, upload-time = "2026-01-10T09:22:37.602Z" }, - { url = "https://files.pythonhosted.org/packages/d2/ae/0ee92b33087a33632f37a635e11e1d99d429d3d323329675a6022312aac2/websockets-16.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:08d7af67b64d29823fed316505a89b86705f2b7981c07848fb5e3ea3020c1abe", size = 184631, upload-time = "2026-01-10T09:22:38.789Z" }, - { url = "https://files.pythonhosted.org/packages/c8/c5/27178df583b6c5b31b29f526ba2da5e2f864ecc79c99dae630a85d68c304/websockets-16.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7be95cfb0a4dae143eaed2bcba8ac23f4892d8971311f1b06f3c6b78952ee70b", size = 185870, upload-time = "2026-01-10T09:22:39.893Z" }, - { url = "https://files.pythonhosted.org/packages/87/05/536652aa84ddc1c018dbb7e2c4cbcd0db884580bf8e95aece7593fde526f/websockets-16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d6297ce39ce5c2e6feb13c1a996a2ded3b6832155fcfc920265c76f24c7cceb5", size = 185361, upload-time = "2026-01-10T09:22:41.016Z" }, - { url = "https://files.pythonhosted.org/packages/6d/e2/d5332c90da12b1e01f06fb1b85c50cfc489783076547415bf9f0a659ec19/websockets-16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1c1b30e4f497b0b354057f3467f56244c603a79c0d1dafce1d16c283c25f6e64", size = 184615, upload-time = "2026-01-10T09:22:42.442Z" }, - { url = "https://files.pythonhosted.org/packages/77/fb/d3f9576691cae9253b51555f841bc6600bf0a983a461c79500ace5a5b364/websockets-16.0-cp311-cp311-win32.whl", hash = "sha256:5f451484aeb5cafee1ccf789b1b66f535409d038c56966d6101740c1614b86c6", size = 178246, upload-time = "2026-01-10T09:22:43.654Z" }, - { url = "https://files.pythonhosted.org/packages/54/67/eaff76b3dbaf18dcddabc3b8c1dba50b483761cccff67793897945b37408/websockets-16.0-cp311-cp311-win_amd64.whl", hash = "sha256:8d7f0659570eefb578dacde98e24fb60af35350193e4f56e11190787bee77dac", size = 178684, upload-time = "2026-01-10T09:22:44.941Z" }, - { url = "https://files.pythonhosted.org/packages/84/7b/bac442e6b96c9d25092695578dda82403c77936104b5682307bd4deb1ad4/websockets-16.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:71c989cbf3254fbd5e84d3bff31e4da39c43f884e64f2551d14bb3c186230f00", size = 177365, upload-time = "2026-01-10T09:22:46.787Z" }, - { url = "https://files.pythonhosted.org/packages/b0/fe/136ccece61bd690d9c1f715baaeefd953bb2360134de73519d5df19d29ca/websockets-16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8b6e209ffee39ff1b6d0fa7bfef6de950c60dfb91b8fcead17da4ee539121a79", size = 175038, upload-time = "2026-01-10T09:22:47.999Z" }, - { url = "https://files.pythonhosted.org/packages/40/1e/9771421ac2286eaab95b8575b0cb701ae3663abf8b5e1f64f1fd90d0a673/websockets-16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:86890e837d61574c92a97496d590968b23c2ef0aeb8a9bc9421d174cd378ae39", size = 175328, upload-time = "2026-01-10T09:22:49.809Z" }, - { url = "https://files.pythonhosted.org/packages/18/29/71729b4671f21e1eaa5d6573031ab810ad2936c8175f03f97f3ff164c802/websockets-16.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9b5aca38b67492ef518a8ab76851862488a478602229112c4b0d58d63a7a4d5c", size = 184915, upload-time = "2026-01-10T09:22:51.071Z" }, - { url = "https://files.pythonhosted.org/packages/97/bb/21c36b7dbbafc85d2d480cd65df02a1dc93bf76d97147605a8e27ff9409d/websockets-16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0334872c0a37b606418ac52f6ab9cfd17317ac26365f7f65e203e2d0d0d359f", size = 186152, upload-time = "2026-01-10T09:22:52.224Z" }, - { url = "https://files.pythonhosted.org/packages/4a/34/9bf8df0c0cf88fa7bfe36678dc7b02970c9a7d5e065a3099292db87b1be2/websockets-16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a0b31e0b424cc6b5a04b8838bbaec1688834b2383256688cf47eb97412531da1", size = 185583, upload-time = "2026-01-10T09:22:53.443Z" }, - { url = "https://files.pythonhosted.org/packages/47/88/4dd516068e1a3d6ab3c7c183288404cd424a9a02d585efbac226cb61ff2d/websockets-16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:485c49116d0af10ac698623c513c1cc01c9446c058a4e61e3bf6c19dff7335a2", size = 184880, upload-time = "2026-01-10T09:22:55.033Z" }, - { url = "https://files.pythonhosted.org/packages/91/d6/7d4553ad4bf1c0421e1ebd4b18de5d9098383b5caa1d937b63df8d04b565/websockets-16.0-cp312-cp312-win32.whl", hash = "sha256:eaded469f5e5b7294e2bdca0ab06becb6756ea86894a47806456089298813c89", size = 178261, upload-time = "2026-01-10T09:22:56.251Z" }, - { url = "https://files.pythonhosted.org/packages/c3/f0/f3a17365441ed1c27f850a80b2bc680a0fa9505d733fe152fdf5e98c1c0b/websockets-16.0-cp312-cp312-win_amd64.whl", hash = "sha256:5569417dc80977fc8c2d43a86f78e0a5a22fee17565d78621b6bb264a115d4ea", size = 178693, upload-time = "2026-01-10T09:22:57.478Z" }, - { url = "https://files.pythonhosted.org/packages/cc/9c/baa8456050d1c1b08dd0ec7346026668cbc6f145ab4e314d707bb845bf0d/websockets-16.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:878b336ac47938b474c8f982ac2f7266a540adc3fa4ad74ae96fea9823a02cc9", size = 177364, upload-time = "2026-01-10T09:22:59.333Z" }, - { url = "https://files.pythonhosted.org/packages/7e/0c/8811fc53e9bcff68fe7de2bcbe75116a8d959ac699a3200f4847a8925210/websockets-16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:52a0fec0e6c8d9a784c2c78276a48a2bdf099e4ccc2a4cad53b27718dbfd0230", size = 175039, upload-time = "2026-01-10T09:23:01.171Z" }, - { url = "https://files.pythonhosted.org/packages/aa/82/39a5f910cb99ec0b59e482971238c845af9220d3ab9fa76dd9162cda9d62/websockets-16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6578ed5b6981005df1860a56e3617f14a6c307e6a71b4fff8c48fdc50f3ed2c", size = 175323, upload-time = "2026-01-10T09:23:02.341Z" }, - { url = "https://files.pythonhosted.org/packages/bd/28/0a25ee5342eb5d5f297d992a77e56892ecb65e7854c7898fb7d35e9b33bd/websockets-16.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95724e638f0f9c350bb1c2b0a7ad0e83d9cc0c9259f3ea94e40d7b02a2179ae5", size = 184975, upload-time = "2026-01-10T09:23:03.756Z" }, - { url = "https://files.pythonhosted.org/packages/f9/66/27ea52741752f5107c2e41fda05e8395a682a1e11c4e592a809a90c6a506/websockets-16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0204dc62a89dc9d50d682412c10b3542d748260d743500a85c13cd1ee4bde82", size = 186203, upload-time = "2026-01-10T09:23:05.01Z" }, - { url = "https://files.pythonhosted.org/packages/37/e5/8e32857371406a757816a2b471939d51c463509be73fa538216ea52b792a/websockets-16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:52ac480f44d32970d66763115edea932f1c5b1312de36df06d6b219f6741eed8", size = 185653, upload-time = "2026-01-10T09:23:06.301Z" }, - { url = "https://files.pythonhosted.org/packages/9b/67/f926bac29882894669368dc73f4da900fcdf47955d0a0185d60103df5737/websockets-16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6e5a82b677f8f6f59e8dfc34ec06ca6b5b48bc4fcda346acd093694cc2c24d8f", size = 184920, upload-time = "2026-01-10T09:23:07.492Z" }, - { url = "https://files.pythonhosted.org/packages/3c/a1/3d6ccdcd125b0a42a311bcd15a7f705d688f73b2a22d8cf1c0875d35d34a/websockets-16.0-cp313-cp313-win32.whl", hash = "sha256:abf050a199613f64c886ea10f38b47770a65154dc37181bfaff70c160f45315a", size = 178255, upload-time = "2026-01-10T09:23:09.245Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ae/90366304d7c2ce80f9b826096a9e9048b4bb760e44d3b873bb272cba696b/websockets-16.0-cp313-cp313-win_amd64.whl", hash = "sha256:3425ac5cf448801335d6fdc7ae1eb22072055417a96cc6b31b3861f455fbc156", size = 178689, upload-time = "2026-01-10T09:23:10.483Z" }, - { url = "https://files.pythonhosted.org/packages/f3/1d/e88022630271f5bd349ed82417136281931e558d628dd52c4d8621b4a0b2/websockets-16.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8cc451a50f2aee53042ac52d2d053d08bf89bcb31ae799cb4487587661c038a0", size = 177406, upload-time = "2026-01-10T09:23:12.178Z" }, - { url = "https://files.pythonhosted.org/packages/f2/78/e63be1bf0724eeb4616efb1ae1c9044f7c3953b7957799abb5915bffd38e/websockets-16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:daa3b6ff70a9241cf6c7fc9e949d41232d9d7d26fd3522b1ad2b4d62487e9904", size = 175085, upload-time = "2026-01-10T09:23:13.511Z" }, - { url = "https://files.pythonhosted.org/packages/bb/f4/d3c9220d818ee955ae390cf319a7c7a467beceb24f05ee7aaaa2414345ba/websockets-16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd3cb4adb94a2a6e2b7c0d8d05cb94e6f1c81a0cf9dc2694fb65c7e8d94c42e4", size = 175328, upload-time = "2026-01-10T09:23:14.727Z" }, - { url = "https://files.pythonhosted.org/packages/63/bc/d3e208028de777087e6fb2b122051a6ff7bbcca0d6df9d9c2bf1dd869ae9/websockets-16.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:781caf5e8eee67f663126490c2f96f40906594cb86b408a703630f95550a8c3e", size = 185044, upload-time = "2026-01-10T09:23:15.939Z" }, - { url = "https://files.pythonhosted.org/packages/ad/6e/9a0927ac24bd33a0a9af834d89e0abc7cfd8e13bed17a86407a66773cc0e/websockets-16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caab51a72c51973ca21fa8a18bd8165e1a0183f1ac7066a182ff27107b71e1a4", size = 186279, upload-time = "2026-01-10T09:23:17.148Z" }, - { url = "https://files.pythonhosted.org/packages/b9/ca/bf1c68440d7a868180e11be653c85959502efd3a709323230314fda6e0b3/websockets-16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19c4dc84098e523fd63711e563077d39e90ec6702aff4b5d9e344a60cb3c0cb1", size = 185711, upload-time = "2026-01-10T09:23:18.372Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f8/fdc34643a989561f217bb477cbc47a3a07212cbda91c0e4389c43c296ebf/websockets-16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5e18a238a2b2249c9a9235466b90e96ae4795672598a58772dd806edc7ac6d3", size = 184982, upload-time = "2026-01-10T09:23:19.652Z" }, - { url = "https://files.pythonhosted.org/packages/dd/d1/574fa27e233764dbac9c52730d63fcf2823b16f0856b3329fc6268d6ae4f/websockets-16.0-cp314-cp314-win32.whl", hash = "sha256:a069d734c4a043182729edd3e9f247c3b2a4035415a9172fd0f1b71658a320a8", size = 177915, upload-time = "2026-01-10T09:23:21.458Z" }, - { url = "https://files.pythonhosted.org/packages/8a/f1/ae6b937bf3126b5134ce1f482365fde31a357c784ac51852978768b5eff4/websockets-16.0-cp314-cp314-win_amd64.whl", hash = "sha256:c0ee0e63f23914732c6d7e0cce24915c48f3f1512ec1d079ed01fc629dab269d", size = 178381, upload-time = "2026-01-10T09:23:22.715Z" }, - { url = "https://files.pythonhosted.org/packages/06/9b/f791d1db48403e1f0a27577a6beb37afae94254a8c6f08be4a23e4930bc0/websockets-16.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:a35539cacc3febb22b8f4d4a99cc79b104226a756aa7400adc722e83b0d03244", size = 177737, upload-time = "2026-01-10T09:23:24.523Z" }, - { url = "https://files.pythonhosted.org/packages/bd/40/53ad02341fa33b3ce489023f635367a4ac98b73570102ad2cdd770dacc9a/websockets-16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b784ca5de850f4ce93ec85d3269d24d4c82f22b7212023c974c401d4980ebc5e", size = 175268, upload-time = "2026-01-10T09:23:25.781Z" }, - { url = "https://files.pythonhosted.org/packages/74/9b/6158d4e459b984f949dcbbb0c5d270154c7618e11c01029b9bbd1bb4c4f9/websockets-16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:569d01a4e7fba956c5ae4fc988f0d4e187900f5497ce46339c996dbf24f17641", size = 175486, upload-time = "2026-01-10T09:23:27.033Z" }, - { url = "https://files.pythonhosted.org/packages/e5/2d/7583b30208b639c8090206f95073646c2c9ffd66f44df967981a64f849ad/websockets-16.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:50f23cdd8343b984957e4077839841146f67a3d31ab0d00e6b824e74c5b2f6e8", size = 185331, upload-time = "2026-01-10T09:23:28.259Z" }, - { url = "https://files.pythonhosted.org/packages/45/b0/cce3784eb519b7b5ad680d14b9673a31ab8dcb7aad8b64d81709d2430aa8/websockets-16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:152284a83a00c59b759697b7f9e9cddf4e3c7861dd0d964b472b70f78f89e80e", size = 186501, upload-time = "2026-01-10T09:23:29.449Z" }, - { url = "https://files.pythonhosted.org/packages/19/60/b8ebe4c7e89fb5f6cdf080623c9d92789a53636950f7abacfc33fe2b3135/websockets-16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bc59589ab64b0022385f429b94697348a6a234e8ce22544e3681b2e9331b5944", size = 186062, upload-time = "2026-01-10T09:23:31.368Z" }, - { url = "https://files.pythonhosted.org/packages/88/a8/a080593f89b0138b6cba1b28f8df5673b5506f72879322288b031337c0b8/websockets-16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32da954ffa2814258030e5a57bc73a3635463238e797c7375dc8091327434206", size = 185356, upload-time = "2026-01-10T09:23:32.627Z" }, - { url = "https://files.pythonhosted.org/packages/c2/b6/b9afed2afadddaf5ebb2afa801abf4b0868f42f8539bfe4b071b5266c9fe/websockets-16.0-cp314-cp314t-win32.whl", hash = "sha256:5a4b4cc550cb665dd8a47f868c8d04c8230f857363ad3c9caf7a0c3bf8c61ca6", size = 178085, upload-time = "2026-01-10T09:23:33.816Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3e/28135a24e384493fa804216b79a6a6759a38cc4ff59118787b9fb693df93/websockets-16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b14dc141ed6d2dde437cddb216004bcac6a1df0935d79656387bd41632ba0bbd", size = 178531, upload-time = "2026-01-10T09:23:35.016Z" }, - { url = "https://files.pythonhosted.org/packages/72/07/c98a68571dcf256e74f1f816b8cc5eae6eb2d3d5cfa44d37f801619d9166/websockets-16.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:349f83cd6c9a415428ee1005cadb5c2c56f4389bc06a9af16103c3bc3dcc8b7d", size = 174947, upload-time = "2026-01-10T09:23:36.166Z" }, - { url = "https://files.pythonhosted.org/packages/7e/52/93e166a81e0305b33fe416338be92ae863563fe7bce446b0f687b9df5aea/websockets-16.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:4a1aba3340a8dca8db6eb5a7986157f52eb9e436b74813764241981ca4888f03", size = 175260, upload-time = "2026-01-10T09:23:37.409Z" }, - { url = "https://files.pythonhosted.org/packages/56/0c/2dbf513bafd24889d33de2ff0368190a0e69f37bcfa19009ef819fe4d507/websockets-16.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f4a32d1bd841d4bcbffdcb3d2ce50c09c3909fbead375ab28d0181af89fd04da", size = 176071, upload-time = "2026-01-10T09:23:39.158Z" }, - { url = "https://files.pythonhosted.org/packages/a5/8f/aea9c71cc92bf9b6cc0f7f70df8f0b420636b6c96ef4feee1e16f80f75dd/websockets-16.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0298d07ee155e2e9fda5be8a9042200dd2e3bb0b8a38482156576f863a9d457c", size = 176968, upload-time = "2026-01-10T09:23:41.031Z" }, - { url = "https://files.pythonhosted.org/packages/9a/3f/f70e03f40ffc9a30d817eef7da1be72ee4956ba8d7255c399a01b135902a/websockets-16.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a653aea902e0324b52f1613332ddf50b00c06fdaf7e92624fbf8c77c78fa5767", size = 178735, upload-time = "2026-01-10T09:23:42.259Z" }, - { url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" }, -] - -[[package]] -name = "zipp" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, -] diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 000000000..80c0bcc24 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,19 @@ +import path from 'node:path'; +import { defineConfig } from 'vitest/config'; + +const r = (p: string) => path.resolve(import.meta.dirname, p); + +export default defineConfig({ + test: { + globals: true, + include: ['__tests__/**/*.test.ts'], + testTimeout: 15000, + // Inline-process @keboola/api-client so vitest's transform pipeline resolves the + // extensionless `dayjs/plugin/utc` imports in its chunks (Node's strict ESM resolver + // otherwise can't; esbuild/tsup resolve them fine at build time). + server: { deps: { inline: [/@keboola\/api-client/] } }, + }, + resolve: { + alias: [{ find: /^@\/(.*)/, replacement: `${r('src')}/$1` }], + }, +}); diff --git a/vitest.integ.config.ts b/vitest.integ.config.ts new file mode 100644 index 000000000..6d33ab795 --- /dev/null +++ b/vitest.integ.config.ts @@ -0,0 +1,26 @@ +import path from 'node:path'; +import { defineConfig } from 'vitest/config'; + +const r = (p: string) => path.resolve(import.meta.dirname, p); + +// Integration tests run against real Keboola projects leased from the redis-backed pool +// (see integtests/testproject + feature_spec/integration-tests/RFC.md). They are slow and +// require TEST_KBC_PROJECTS_FILE (+ redis lock env on CI), so they live behind a separate +// config and the `test:integ` script — never part of the default `vitest` unit run. +export default defineConfig({ + test: { + globals: true, + include: ['integtests/**/*.test.ts'], + // A leased project can wait for the pool + run real API calls; allow generous time. + testTimeout: 120_000, + hookTimeout: 120_000, + // Files run in parallel workers; each worker leases its own project. + fileParallelism: true, + // See vitest.config.ts: inline @keboola/api-client so its extensionless dayjs imports + // resolve under vitest. + server: { deps: { inline: [/@keboola\/api-client/] } }, + }, + resolve: { + alias: [{ find: /^@\/(.*)/, replacement: `${r('src')}/$1` }], + }, +});