diff --git a/BUNDLE_API.md b/BUNDLE_API.md index 20233a709372..3a6accd617da 100644 --- a/BUNDLE_API.md +++ b/BUNDLE_API.md @@ -86,7 +86,8 @@ that does not list `str(BUNDLE_API_VERSION)` is rejected at install time with | Manifest schema (`extension.json` / `[tool.langflow.extension]`) | `lfx.extension.manifest.ExtensionManifest` | | `BundleRef` (one entry in optional `bundles[]`; bundle names must be unique) | `lfx.extension.manifest.BundleRef` | | `ProviderManifestEntry` (one entry in the optional `providers[]`) | `lfx.extension.manifest.ProviderManifestEntry` | -| `IntegrationProvider` entries in optional `integrations[]` | `lfx.integrations.IntegrationProvider` | +| `IntegrationManifestRef` entries in optional `integrations[]` | `lfx.extension.manifest.IntegrationManifestRef` | +| `IntegrationCapabilityManifest` (bundle-owned versioned provider catalog) | `lfx.integrations.IntegrationCapabilityManifest` | | `LfxCompat` (declared as `manifest.lfx`) | `lfx.extension.manifest.LfxCompat` | | `BUNDLE_API_VERSION` (the integer this lfx ships) | `lfx.extension.manifest` | | `EXTENSION_SCHEMA_URL` / `SCHEMA_VERSION` | `lfx.extension.manifest` | @@ -105,6 +106,7 @@ Component IDs at runtime are `ext::@`. | `discover_inline_bundles()` | `lfx.extension.loader` | | `discover_installed_extensions()` / `discover_seed_extensions()` / `discover_all_extensions()` | `lfx.extension.discovery` | | `LoadedComponent` | `lfx.extension.loader` (frozen dataclass; what the registry stores) | +| `LoadedIntegration` | `lfx.extension.loader` (validated capability metadata for discovery and policy) | | `LoadResult` | `lfx.extension.loader` | | `SLOT_OFFICIAL` / `SLOT_EXTRA` | `lfx.extension.loader` | @@ -113,8 +115,8 @@ Component IDs at runtime are `ext::@`. | Symbol | Source | | --- | --- | | `reload_bundle(registry, bundle_name)` | `lfx.extension.reload` | -| `BundleRegistry` | `lfx.extension.bundle_registry` | -| `BundleRecord` | `lfx.extension.bundle_registry` | +| `BundleRegistry` (`list_components()`, `list_integrations()`) | `lfx.extension.bundle_registry` | +| `BundleRecord` (components plus validated integration metadata) | `lfx.extension.bundle_registry` | | `ReloadInProgressError` | `lfx.extension.bundle_registry` | | `POST /api/v1/extensions/{id}/bundles/{name}/reload` | `langflow.api.v1.extensions` | @@ -205,6 +207,19 @@ the deserialize half is covered by ### v0 (this release) +- **Bundle-owned integration capability manifests (additive).** + `ExtensionManifest.integrations[]` now carries `IntegrationManifestRef` + values (`provider_id`, owning `bundle`, relative JSON `path`). The referenced + `IntegrationCapabilityManifest` is versioned with `schema_version=1` and + declares authentication profiles plus executable actions, required and + conditional scopes, policy keys, substrate, maturity, deployment contexts, + risk, and execution targets. The loader validates bundle ownership and + provider identity and exposes the parsed catalog as `LoadedIntegration` in + `LoadResult.integrations` and retains it in `BundleRecord.integrations` for + process-wide discovery and policy reads through + `BundleRegistry.list_integrations()`. Manifests that omit `integrations` + still load with an empty list; `BUNDLE_API_VERSION` remains `1`. + - Initial surface enumerated above. Frozen as `BUNDLE_API_VERSION = 1`. - Added the provider-neutral connection-reference, resolver, capability, integration-error, and telemetry contracts used by dedicated integration diff --git a/design/dedicated-integrations/connection-contract.md b/design/dedicated-integrations/connection-contract.md index 173c68e95e2b..6258b0b88dbc 100644 --- a/design/dedicated-integrations/connection-contract.md +++ b/design/dedicated-integrations/connection-contract.md @@ -273,8 +273,8 @@ dataclass (not an `Exception`; loader-specific code namespace). ## 8. Capability metadata types shared with INT-3 -**Decision: frozen Pydantic models in `lfx/integrations/capabilities.py`; reserve `ExtensionManifest.integrations` -now.** +**Decision: frozen Pydantic models in `lfx/integrations/capabilities.py`; `ExtensionManifest.integrations` +references bundle-owned, versioned capability manifests.** - `IntegrationProvider{provider_id, display_name, icon, auth_profiles: tuple[OAuthProfile, ...], capabilities, docs_url}`. A provider may expose more than one named profile; Slack has `slack-user-oauth` and @@ -287,7 +287,9 @@ now.** `oauth_client_type_by_context`; profiles may omit an unsupported context. This represents both Tauri public-client loopback and Microsoft's `{tenant}` authority without pretending one provider-wide auth object fits every action. - `IntegrationCapability{id, display_name, auth_profile_id, identity, required_scopes, conditional_scopes, - risk: read | write | destructive, component_ref, mcp_tool}`. `ConditionalScopeRequirement{scope, role, + policy_keys, substrate: sdk | rest | mcp, maturity: ga | preview | developer_preview | beta | deprecated, + deployment_contexts, risk: read | write | destructive, component_ref, mcp_tool}`. + `ConditionalScopeRequirement{scope, role, condition}` preserves `optional` versus `alternative`; `ScopeCondition{kind: input_present | input_truthy, input}` is evaluated against the action's declared input schema. The matrix checker rejects a condition that names a missing input. `ScopeSet.covers(capability, inputs, granted) -> missing` first activates conditional @@ -296,8 +298,12 @@ now.** - Capability ids are the matrices' `action_id` values. `required` rows become `required_scopes`; `optional` and `alternative` rows become `conditional_scopes` without losing their role or predicate. The capability's `auth_profile_id` and `identity` must match the selected connection before scope coverage is evaluated. -- `ExtensionManifest.integrations: tuple[IntegrationProvider, ...] = ()` is added as an optional field (additive; - `manifest.py` is in the changelog gate); loader wiring is INT-3. +- `ExtensionManifest.integrations: tuple[IntegrationManifestRef, ...] = ()`, where each reference declares + `{provider_id, bundle, path}` and `path` is relative to the named bundle. The referenced + `IntegrationCapabilityManifest` extends `IntegrationProvider` with `schema_version: Literal[1]`. Validation + rejects missing, malformed, provider-mismatched, or bundle-escaping files; the loader exposes successful records + through `LoadResult.integrations` as `LoadedIntegration` values and retains them in the process-wide + `BundleRegistry` for discovery and policy consumers. Rejected: reusing `ProviderManifestEntry` (model-provider registry semantics would route integrations into model-provider policy); JSON-schema only (the resolver and the picker need the same scope math in Python). diff --git a/design/dedicated-integrations/schema/capability_matrix.schema.json b/design/dedicated-integrations/schema/capability_matrix.schema.json index 62d8fb324aa5..04993bea9f53 100644 --- a/design/dedicated-integrations/schema/capability_matrix.schema.json +++ b/design/dedicated-integrations/schema/capability_matrix.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://schemas.langflow.org/design/dedicated-integrations/capability-matrix-v1.json", "title": "Dedicated Integrations wave-1 capability matrix", - "description": "One provider's wave-1 capability matrix for the INT-1 discovery gate. Hand-written for the gate; INT-3 lifts the enums into Pydantic under src/lfx/src/lfx/extension/ and regenerates this file. The enum values here are asserted equal to VALID_VALUES in scripts/ci/check_capability_matrices.py.", + "description": "One provider's wave-1 capability matrix for the discovery gate. This remains the historical planning record; runtime capability manifests use the matching Pydantic enums in src/lfx/src/lfx/integrations/capabilities.py. Tests and scripts/ci/check_capability_matrices.py guard enum parity.", "type": "object", "additionalProperties": false, "required": [ diff --git a/docs/docs/Lfx/extensions-manifest.mdx b/docs/docs/Lfx/extensions-manifest.mdx index eaaea46f6728..733198e6c3f3 100644 --- a/docs/docs/Lfx/extensions-manifest.mdx +++ b/docs/docs/Lfx/extensions-manifest.mdx @@ -40,6 +40,7 @@ Use the `$schema` reference in your manifest so editors can autocomplete and val | `bundles` | array | no | [Bundle list](#bundles). v0 accepts at most one bundle; omit it for a provider-only extension. | | `providers` | array | no | [Model providers](#providers) contributed to the unified model-provider registry. | | `capabilities` | object | no | [Optional capability flags](#capabilities). Defaults to all-false. | +| `integrations` | array | no | [Bundle-owned integration capability manifests](#integrations). Defaults to an empty list. | | `$schema` | string | no | Optional pointer to this JSON Schema; editors use it for autocomplete. | `additionalProperties: false` — any field not listed here is rejected with a typed error. Reserved names (`services`, `routes`, `hooks`, `starterProjects`, `userConfig`) are documented under [Deferred fields](#deferred-fields) and surface a more specific error code. @@ -142,6 +143,72 @@ Optional. Defaults to `{ "requiresCredentials": false }`. Additional capability keys are rejected with `extra="forbid"` so a misspelled key surfaces immediately rather than silently turning a feature off. +## `integrations` + +Integration bundles publish provider capabilities in a separate, versioned JSON file owned by the bundle. The extension manifest points to that file: + +```json +"integrations": [ + { + "provider_id": "google", + "bundle": "google", + "path": "capabilities.v1.json" + } +] +``` + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `provider_id` | string | yes | Stable lowercase provider key used by discovery, policy, and connection resolution. | +| `bundle` | string | yes | Name of the bundle that owns the capability manifest. It must match an entry in `bundles[]`. | +| `path` | string | yes | JSON file path relative to that bundle's directory. Absolute paths, `..`, and symlink escapes are rejected. | + +The referenced file uses this shape: + +```json +{ + "schema_version": 1, + "provider_id": "google", + "display_name": "Google Workspace", + "auth_profiles": [ + { + "id": "user", + "kind": "oauth2_authorization_code", + "identity": "user_delegated", + "supports_pkce": true, + "supports_refresh": true + } + ], + "capabilities": [ + { + "id": "google.drive.files.search", + "display_name": "Drive: Search Files", + "auth_profile_id": "user", + "identity": "user_delegated", + "required_scopes": ["drive.file"], + "policy_keys": ["integrations.google.drive.search"], + "substrate": "sdk", + "maturity": "ga", + "deployment_contexts": ["hosted", "self_managed", "desktop", "headless"], + "risk": "read", + "component_ref": "GoogleDriveSearchComponent" + } + ] +} +``` + +`schema_version` is currently `1`. Authentication modes are declared by `auth_profiles[].kind`; actions reference a profile through `auth_profile_id`. Each action declares scopes, policy keys, execution substrate (`sdk`, `rest`, or `mcp`), maturity, supported deployment contexts, risk, and at least one execution target (`component_ref` or `mcp_tool`). Unknown fields are rejected throughout. + +During loading, each valid file is exposed as a `LoadedIntegration` in `LoadResult.integrations` and retained in `BundleRecord.integrations`. Discovery and policy code can consume the parsed `capability_manifest` from the load result or call `BundleRegistry.list_integrations()` for the process-wide snapshot. Extensions that omit `integrations` continue to load with an empty list. + +After adding an integration manifest on a release branch, synchronize bundle dependency floors with the repository script instead of editing `pyproject.toml` by hand: + +```bash +uv run python scripts/ci/sync_bundle_lfx_pin.py 1.13.0 +``` + +The command is idempotent and applies the canonical `lfx>=1.13.0.dev0,<2.0.0` floor to bundles that need an update. + ## Deferred fields The schema strips these names from the published `properties` map but reserves them via [`x-deferred-fields`](https://schemas.langflow.org/extension/v1.json), so a manifest that sets one gets a specific error code instead of the generic "additional property" message. @@ -182,7 +249,7 @@ The loader and validator both emit typed errors keyed by the manifest field that | Code | Cause | | --- | --- | -| `manifest-invalid` | Schema validation failed; the message names the field. | +| `manifest-invalid` | The extension manifest or a referenced integration capability manifest fails schema validation. | | `manifest-not-found` | No `extension.json` and no `[tool.langflow.extension]` section at the extension root. | | `version-constraint-unsatisfied` | `lfx.compat` does not include this Langflow's `BUNDLE_API_VERSION`. | | `field-deferred-in-this-milestone` | A reserved field was set to a non-null value. | diff --git a/src/lfx/src/lfx/extension/__init__.py b/src/lfx/src/lfx/extension/__init__.py index 1f6768eae813..918418924202 100644 --- a/src/lfx/src/lfx/extension/__init__.py +++ b/src/lfx/src/lfx/extension/__init__.py @@ -72,6 +72,7 @@ SLOT_EXTRA, SLOT_OFFICIAL, LoadedComponent, + LoadedIntegration, LoadResult, discover_inline_bundles, filter_component_entry_points, @@ -90,6 +91,7 @@ SCHEMA_VERSION, BundleRef, ExtensionManifest, + IntegrationManifestRef, LfxCompat, ManifestSource, load_manifest, @@ -135,6 +137,7 @@ "SCHEMA_VERSION": "manifest", "BundleRef": "manifest", "ExtensionManifest": "manifest", + "IntegrationManifestRef": "manifest", "LfxCompat": "manifest", "ManifestSource": "manifest", "load_manifest": "manifest", @@ -155,6 +158,7 @@ "SLOT_OFFICIAL": "loader", "LoadResult": "loader", "LoadedComponent": "loader", + "LoadedIntegration": "loader", "discover_inline_bundles": "loader", "filter_component_entry_points": "loader", "filter_plugin_entry_points": "loader", @@ -240,10 +244,12 @@ def __dir__() -> list[str]: "ExtensionManifest", "ExtensionRegistry", "InitOptions", + "IntegrationManifestRef", "LfxCompat", "LoadResult", "LoadStatus", "LoadedComponent", + "LoadedIntegration", "ManifestSource", "MigrationEntry", "MigrationReport", diff --git a/src/lfx/src/lfx/extension/bundle_registry.py b/src/lfx/src/lfx/extension/bundle_registry.py index d4b7923c5b81..e0c7348aceab 100644 --- a/src/lfx/src/lfx/extension/bundle_registry.py +++ b/src/lfx/src/lfx/extension/bundle_registry.py @@ -55,7 +55,7 @@ class set even after Stage 3 has flipped the registry's internal pointer. from pathlib import Path from typing import Literal - from lfx.extension.loader import LoadedComponent + from lfx.extension.loader import LoadedComponent, LoadedIntegration # --------------------------------------------------------------------------- @@ -96,6 +96,7 @@ class BundleRecord: extension_version: str slot: Literal["official", "extra"] components: tuple[LoadedComponent, ...] = () + integrations: tuple[LoadedIntegration, ...] = () distribution: str | None = None source_path: Path | None = None # Provenance: True for manifest-less lfx.bundles metapackage providers. @@ -168,6 +169,14 @@ def list_components(self) -> list[LoadedComponent]: out.extend(snap[name].components) return out + def list_integrations(self) -> list[LoadedIntegration]: + """Flatten validated integration metadata for discovery and policy consumers.""" + snap = self.snapshot() + out: list[LoadedIntegration] = [] + for name in sorted(snap): + out.extend(snap[name].integrations) + return out + # -- write paths --------------------------------------------------------- @contextmanager diff --git a/src/lfx/src/lfx/extension/integration_manifest.py b/src/lfx/src/lfx/extension/integration_manifest.py new file mode 100644 index 000000000000..e5ed6eb0993b --- /dev/null +++ b/src/lfx/src/lfx/extension/integration_manifest.py @@ -0,0 +1,93 @@ +"""Resolve and validate bundle-owned integration capability manifests.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from pydantic import ValidationError + +from lfx.extension._paths import is_within +from lfx.extension.errors import ExtensionError +from lfx.integrations.capabilities import IntegrationCapabilityManifest + +if TYPE_CHECKING: + from pathlib import Path + + from lfx.extension.manifest import IntegrationManifestRef + + +@dataclass(frozen=True) +class ResolvedIntegrationManifest: + """A validated capability catalog paired with its canonical source path.""" + + path: Path + manifest: IntegrationCapabilityManifest + + +def resolve_integration_manifest( + bundle_root: Path, + reference: IntegrationManifestRef, +) -> tuple[ResolvedIntegrationManifest | None, ExtensionError | None]: + """Load one reference while enforcing bundle ownership and provider identity.""" + try: + resolved_root = bundle_root.resolve(strict=True) + candidate = (resolved_root / reference.path).resolve(strict=False) + except (OSError, RuntimeError, ValueError) as exc: + return None, ExtensionError( + code="manifest-unreadable", + message=f"Could not resolve integration capability manifest: {exc}", + location=reference.path, + content=reference.path, + hint="Check the capability-manifest path and bundle permissions.", + ) + + if not is_within(candidate, resolved_root): + return None, ExtensionError( + code="path-escape", + message=( + f"Integration capability-manifest path {reference.path!r} resolves outside " + f"its owning bundle {reference.bundle!r}." + ), + location=f"integrations[{reference.provider_id}].path", + content=reference.path, + hint="Move the capability manifest inside the bundle directory.", + ) + + try: + raw = candidate.read_text(encoding="utf-8") + except (OSError, UnicodeError, ValueError) as exc: + return None, ExtensionError( + code="manifest-unreadable", + message=f"Could not read integration capability manifest: {exc}", + location=str(candidate), + content=reference.path, + hint="Create the referenced JSON file inside the bundle and check its permissions.", + ) + + try: + payload = json.loads(raw) + manifest = IntegrationCapabilityManifest.model_validate(payload) + except (json.JSONDecodeError, TypeError, ValidationError) as exc: + return None, ExtensionError( + code="manifest-invalid", + message=f"Integration capability manifest is invalid: {exc}", + location=str(candidate), + content=reference.path, + hint="Fix the referenced capability manifest so it matches the versioned integration schema.", + ) + + if manifest.provider_id != reference.provider_id: + return None, ExtensionError( + code="manifest-invalid", + message=( + f"Integration provider {reference.provider_id!r} references a capability manifest for " + f"{manifest.provider_id!r}." + ), + location=str(candidate), + content=manifest.provider_id, + hint="Make provider_id identical in extension.json and the capability manifest.", + ) + + return ResolvedIntegrationManifest(path=candidate, manifest=manifest), None diff --git a/src/lfx/src/lfx/extension/loader/__init__.py b/src/lfx/src/lfx/extension/loader/__init__.py index 353a3160c0b9..b6663038baf8 100644 --- a/src/lfx/src/lfx/extension/loader/__init__.py +++ b/src/lfx/src/lfx/extension/loader/__init__.py @@ -78,6 +78,7 @@ SLOT_OFFICIAL, SLOT_VALUES, LoadedComponent, + LoadedIntegration, LoadResult, ) @@ -89,6 +90,7 @@ "SLOT_VALUES", "LoadResult", "LoadedComponent", + "LoadedIntegration", "discover_inline_bundles", "filter_component_entry_points", "filter_plugin_entry_points", diff --git a/src/lfx/src/lfx/extension/loader/_orchestrator.py b/src/lfx/src/lfx/extension/loader/_orchestrator.py index c53456d115e9..160f0ae8019b 100644 --- a/src/lfx/src/lfx/extension/loader/_orchestrator.py +++ b/src/lfx/src/lfx/extension/loader/_orchestrator.py @@ -26,6 +26,7 @@ from lfx.extension._paths import SKIP_DIR_NAMES, is_within from lfx.extension.errors import ExtensionError +from lfx.extension.integration_manifest import resolve_integration_manifest from lfx.extension.loader._detection import collect_component_classes from lfx.extension.loader._discovery import ( DEFAULT_MODULE_NAMESPACE, @@ -38,6 +39,7 @@ SLOT_OFFICIAL, SLOT_VALUES, LoadedComponent, + LoadedIntegration, LoadResult, ) from lfx.extension.manifest import ( @@ -128,6 +130,36 @@ def _resolve_bundle_path(root: Path, bundle: BundleRef) -> tuple[Path | None, Ex return resolved, None +def _load_bundle_integrations( + *, + bundle_root: Path, + bundle: BundleRef, + manifest: ExtensionManifest, + result: LoadResult, +) -> bool: + """Attach this bundle's validated capability catalogs to ``result``.""" + for reference in manifest.integrations: + if reference.bundle != bundle.name: + continue + resolved, error = resolve_integration_manifest(bundle_root, reference) + if error is not None or resolved is None: + if error is not None: + result.errors.append(error) + continue + result.integrations.append( + LoadedIntegration( + extension_id=manifest.id, + extension_version=manifest.version, + bundle=bundle.name, + provider_id=reference.provider_id, + manifest_path=resolved.path, + capability_manifest=resolved.manifest, + distribution=result.distribution, + ) + ) + return not result.errors + + # --------------------------------------------------------------------------- # Core load: a directory + identity tuple -> LoadResult # --------------------------------------------------------------------------- @@ -466,12 +498,6 @@ def load_extension( ) return result - # Register any model providers this extension declares before touching the - # component bundle: providers are independent of components, and a - # provider-only extension ships providers with no bundle. - if _register_providers: - _register_manifest_providers(manifest, source, result) - if bundle_name is None and len(manifest.bundles) > 1: result.errors.append( ExtensionError( @@ -486,9 +512,10 @@ def load_extension( ) return result - # Provider-only extension: no component bundle to load. Providers (if any) - # were registered above; nothing else to do. + # Provider-only extension: no component or integration bundle to load. if not manifest.bundles: + if _register_providers: + _register_manifest_providers(manifest, source, result) return result if bundle_name is None: @@ -514,6 +541,14 @@ def load_extension( result.errors.append(path_error) return result + if not _load_bundle_integrations(bundle_root=bundle_root, bundle=bundle, manifest=manifest, result=result): + return result + + # Avoid a partial registry side effect when bundle-owned integration + # metadata is missing or malformed. + if _register_providers: + _register_manifest_providers(manifest, source, result) + _load_bundle_directory( bundle_root=bundle_root, bundle_name=bundle.name, diff --git a/src/lfx/src/lfx/extension/loader/_startup.py b/src/lfx/src/lfx/extension/loader/_startup.py index 9dd31b3a1d90..9e801a09687d 100644 --- a/src/lfx/src/lfx/extension/loader/_startup.py +++ b/src/lfx/src/lfx/extension/loader/_startup.py @@ -104,6 +104,7 @@ def load_installed_extensions( ) ) result.components = [] + result.integrations = [] else: seen_bundles[result.bundle] = result results.append(result) diff --git a/src/lfx/src/lfx/extension/loader/_types.py b/src/lfx/src/lfx/extension/loader/_types.py index cd45f9e42ae8..fc4a5abe7ca6 100644 --- a/src/lfx/src/lfx/extension/loader/_types.py +++ b/src/lfx/src/lfx/extension/loader/_types.py @@ -15,6 +15,7 @@ from pathlib import Path from lfx.extension.errors import ExtensionError + from lfx.integrations import IntegrationCapabilityManifest # --------------------------------------------------------------------------- # Slot names @@ -94,6 +95,19 @@ def namespaced_id(self) -> str: return f"ext:{self.bundle}:{self.class_name}@{self.slot}" +@dataclass(frozen=True) +class LoadedIntegration: + """Validated provider capability metadata exposed by the extension loader.""" + + extension_id: str + extension_version: str + bundle: str + provider_id: str + manifest_path: Path + capability_manifest: IntegrationCapabilityManifest + distribution: str | None = None + + # --------------------------------------------------------------------------- # LoadResult # --------------------------------------------------------------------------- @@ -129,6 +143,7 @@ class LoadResult: """ components: list[LoadedComponent] = field(default_factory=list) + integrations: list[LoadedIntegration] = field(default_factory=list) errors: list[ExtensionError] = field(default_factory=list) warnings: list[ExtensionError] = field(default_factory=list) extension_id: str | None = None diff --git a/src/lfx/src/lfx/extension/manifest.py b/src/lfx/src/lfx/extension/manifest.py index 27397511583b..3a9b274a7c3a 100644 --- a/src/lfx/src/lfx/extension/manifest.py +++ b/src/lfx/src/lfx/extension/manifest.py @@ -8,7 +8,9 @@ - what component-base-class API surface the Bundle was built against (``lfx.compat``), - what optional capabilities the Bundle declares - (``capabilities.requiresCredentials`` is the only v0 slot). + (``capabilities.requiresCredentials`` is the only v0 flag), + - which bundle-owned integration capability manifests to load + (``integrations``). Manifest source forms (both supported): @@ -49,7 +51,6 @@ model_validator, ) -from lfx.integrations.capabilities import IntegrationProvider from lfx.integrations.models import provider_env_segment # --------------------------------------------------------------------------- @@ -472,6 +473,54 @@ def _live_mutually_exclusive(self) -> ProviderManifestEntry: return self +# --------------------------------------------------------------------------- +# Integration capability manifests +# --------------------------------------------------------------------------- + + +class IntegrationManifestRef(BaseModel): + """Reference to a versioned capability manifest owned by one bundle.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + provider_id: StrictStr = Field( + ..., + pattern=_PROVIDER_ID_RE.pattern, + description="Stable provider key used by discovery, policy, and connection resolution.", + ) + bundle: StrictStr = Field( + ..., + pattern=BUNDLE_NAME_RE.pattern, + description="Name of the bundle that owns the capability manifest.", + ) + path: StrictStr = Field( + ..., + min_length=1, + description="JSON capability-manifest path, relative to the owning bundle directory.", + json_schema_extra={ + "pattern": r"^(?!.*\u0000)(?![\\/])(?!.*(?:^|[\\/])\.\.(?:[\\/]|$)).+\.json$", + }, + ) + + @field_validator("path") + @classmethod + def _validate_path_shape(cls, value: str) -> str: + if "\x00" in value: + msg = "Integration capability-manifest path must not contain a null byte" + raise ValueError(msg) + path = Path(value) + if path.is_absolute(): + msg = f"Integration capability-manifest path {value!r} must be relative to the owning bundle" + raise ValueError(msg) + if any(part == ".." for part in path.parts): + msg = f"Integration capability-manifest path {value!r} must not contain '..'" + raise ValueError(msg) + if path.suffix.casefold() != ".json": + msg = f"Integration capability-manifest path {value!r} must name a JSON file" + raise ValueError(msg) + return value + + # --------------------------------------------------------------------------- # ExtensionManifest # --------------------------------------------------------------------------- @@ -556,9 +605,9 @@ class ExtensionManifest(BaseModel): description="Optional declared capabilities (v0: requiresCredentials only).", ) - integrations: tuple[IntegrationProvider, ...] = Field( + integrations: tuple[IntegrationManifestRef, ...] = Field( default=(), - description="Provider authentication profiles and executable integration capabilities.", + description="Bundle-owned, versioned provider capability-manifest references.", ) # ------------------------------------------------------------------ @@ -624,7 +673,7 @@ def _validate_provider_name_uniqueness(self) -> ExtensionManifest: @model_validator(mode="after") def _validate_integration_provider_uniqueness(self) -> ExtensionManifest: - provider_ids = [provider.provider_id for provider in self.integrations] + provider_ids = [integration.provider_id for integration in self.integrations] if len(set(provider_ids)) != len(provider_ids): msg = "Integration provider ids must be unique within an extension" raise ValueError(msg) @@ -632,6 +681,11 @@ def _validate_integration_provider_uniqueness(self) -> ExtensionManifest: if len(set(env_segments)) != len(env_segments): msg = "Integration provider ids must map to unique environment-key segments" raise ValueError(msg) + bundle_names = {bundle.name for bundle in self.bundles} + unknown_bundles = sorted({integration.bundle for integration in self.integrations} - bundle_names) + if unknown_bundles: + msg = f"Integration references unknown bundles: {', '.join(unknown_bundles)}" + raise ValueError(msg) return self @model_validator(mode="after") diff --git a/src/lfx/src/lfx/extension/reload.py b/src/lfx/src/lfx/extension/reload.py index 6906dce2bd93..6e3551ce6e81 100644 --- a/src/lfx/src/lfx/extension/reload.py +++ b/src/lfx/src/lfx/extension/reload.py @@ -505,6 +505,7 @@ def _run_pipeline_body( extension_version=staging.extension_version or (previous.extension_version if previous else "0.0.0"), slot=effective_slot, components=new_components, + integrations=tuple(staging.integrations), distribution=staging.distribution, source_path=effective_source, ) diff --git a/src/lfx/src/lfx/extension/validate.py b/src/lfx/src/lfx/extension/validate.py index b7f8ff11c449..baedaccae7b2 100644 --- a/src/lfx/src/lfx/extension/validate.py +++ b/src/lfx/src/lfx/extension/validate.py @@ -43,6 +43,7 @@ ExtensionError, ExtensionErrorCollection, ) +from lfx.extension.integration_manifest import resolve_integration_manifest from lfx.extension.manifest import ( DEFERRED_FIELDS, ExtensionManifest, @@ -870,6 +871,12 @@ def validate_extension( if path_error is not None: report.errors.add_error(path_error) continue + for reference in manifest.integrations: + if reference.bundle != bundle.name: + continue + _, integration_error = resolve_integration_manifest(resolved, reference) + if integration_error is not None: + report.errors.add_error(integration_error) summary = _scan_bundle(bundle.name, resolved, report.errors) # Count files we actually scanned for ``bundle_files_scanned`` stat. report.bundle_files_scanned += sum(1 for _ in summary.bundle_root.rglob("*.py") if _.is_file()) diff --git a/src/lfx/src/lfx/integrations/__init__.py b/src/lfx/src/lfx/integrations/__init__.py index 1d240a95d0a4..a8c2a4825b21 100644 --- a/src/lfx/src/lfx/integrations/__init__.py +++ b/src/lfx/src/lfx/integrations/__init__.py @@ -3,6 +3,7 @@ from lfx.integrations.capabilities import ( ConditionalScopeRequirement, IntegrationCapability, + IntegrationCapabilityManifest, IntegrationProvider, OAuthProfile, ScopeCondition, @@ -44,6 +45,7 @@ "ConnectionUnresolvedError", "CredentialLease", "IntegrationCapability", + "IntegrationCapabilityManifest", "IntegrationError", "IntegrationProvider", "OAuthProfile", diff --git a/src/lfx/src/lfx/integrations/capabilities.py b/src/lfx/src/lfx/integrations/capabilities.py index 0f4da56daf1d..1a9d54323b85 100644 --- a/src/lfx/src/lfx/integrations/capabilities.py +++ b/src/lfx/src/lfx/integrations/capabilities.py @@ -4,7 +4,7 @@ from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, StrictStr, model_validator +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator, model_validator from lfx.integrations.models import PROVIDER_ID_PATTERN @@ -19,6 +19,8 @@ ] IntegrationIdentity = Literal["user_delegated", "bot", "service"] DeploymentContext = Literal["hosted", "self_managed", "desktop", "headless"] +ExecutionSubstrate = Literal["sdk", "rest", "mcp"] +CapabilityMaturity = Literal["ga", "preview", "developer_preview", "beta", "deprecated"] class ScopeCondition(BaseModel): @@ -77,9 +79,24 @@ class IntegrationCapability(BaseModel): identity: IntegrationIdentity required_scopes: tuple[StrictStr, ...] = () conditional_scopes: tuple[ConditionalScopeRequirement, ...] = () + policy_keys: tuple[StrictStr, ...] = Field(min_length=1) + substrate: ExecutionSubstrate + maturity: CapabilityMaturity + deployment_contexts: tuple[DeploymentContext, ...] = Field(min_length=1) risk: Literal["read", "write", "destructive"] - component_ref: StrictStr | None = None - mcp_tool: StrictStr | None = None + component_ref: StrictStr | None = Field(default=None, min_length=1) + mcp_tool: StrictStr | None = Field(default=None, min_length=1) + + @field_validator("required_scopes", "policy_keys", "deployment_contexts") + @classmethod + def _values_are_unique(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if len(set(value)) != len(value): + msg = "Integration capability lists must not contain duplicate values" + raise ValueError(msg) + if any(not item.strip() for item in value): + msg = "Integration capability values must not be blank" + raise ValueError(msg) + return value @model_validator(mode="after") def _has_an_execution_target(self) -> IntegrationCapability: @@ -111,6 +128,15 @@ def _references_known_profiles(self) -> IntegrationProvider: if len(set(capability_ids)) != len(capability_ids): msg = f"Integration provider {self.provider_id!r} has duplicate capability ids" raise ValueError(msg) + wrong_provider = sorted( + capability.id for capability in self.capabilities if not capability.id.startswith(f"{self.provider_id}.") + ) + if wrong_provider: + msg = ( + f"Integration provider {self.provider_id!r} has capability ids outside its provider namespace: " + f"{', '.join(wrong_provider)}" + ) + raise ValueError(msg) unknown = sorted({cap.auth_profile_id for cap in self.capabilities} - set(profile_ids)) if unknown: msg = f"Integration provider {self.provider_id!r} references unknown auth profiles: {', '.join(unknown)}" @@ -130,6 +156,22 @@ def _references_known_profiles(self) -> IntegrationProvider: return self +class IntegrationCapabilityManifest(IntegrationProvider): + """Versioned provider capability catalog stored inside an extension bundle.""" + + schema_version: Literal[1] + + @model_validator(mode="after") + def _has_profiles_and_capabilities(self) -> IntegrationCapabilityManifest: + if not self.auth_profiles: + msg = "An integration capability manifest must declare at least one auth profile" + raise ValueError(msg) + if not self.capabilities: + msg = "An integration capability manifest must declare at least one capability" + raise ValueError(msg) + return self + + class ScopeSet: """Provider-aware scope coverage shared by pickers and resolvers.""" diff --git a/src/lfx/src/lfx/interface/components.py b/src/lfx/src/lfx/interface/components.py index 26bb2bdf7c9f..5d5173e9196a 100644 --- a/src/lfx/src/lfx/interface/components.py +++ b/src/lfx/src/lfx/interface/components.py @@ -1098,6 +1098,7 @@ def _resolve_bundle_shadowing( # Drop components so the registry-population and palette-construction # loops naturally skip this result; the typed warning still emits. result.components = [] + result.integrations = [] return extension_results, seed_results, lfx_bundles_results, dev_results, inline_results @@ -1230,6 +1231,7 @@ async def import_extension_components( extension_version=result.extension_version or "0.0.0", slot=result.slot, components=tuple(result.components), + integrations=tuple(result.integrations), distribution=result.distribution, source_path=result.source_path, manifestless=result.manifestless, diff --git a/src/lfx/tests/unit/extension/loader/test_types.py b/src/lfx/tests/unit/extension/loader/test_types.py index 2e5cc61f2691..9a31f6ddcfe8 100644 --- a/src/lfx/tests/unit/extension/loader/test_types.py +++ b/src/lfx/tests/unit/extension/loader/test_types.py @@ -45,6 +45,7 @@ def test_load_result_default_is_ok() -> None: assert result.ok assert bool(result) is True assert result.components == [] + assert result.integrations == [] # --------------------------------------------------------------------------- diff --git a/src/lfx/tests/unit/extension/test_integration_manifest.py b/src/lfx/tests/unit/extension/test_integration_manifest.py new file mode 100644 index 000000000000..04f317b38462 --- /dev/null +++ b/src/lfx/tests/unit/extension/test_integration_manifest.py @@ -0,0 +1,199 @@ +"""Integration capability-manifest validation and loader exposure.""" + +from __future__ import annotations + +import json +import os +from typing import TYPE_CHECKING + +import pytest +from lfx.extension import load_extension, validate_extension +from lfx.extension.bundle_registry import BundleRecord, BundleRegistry + +if TYPE_CHECKING: + from pathlib import Path + + +def _capability_manifest(*, provider_id: str = "google") -> dict: + return { + "schema_version": 1, + "provider_id": provider_id, + "display_name": "Google Workspace", + "icon": "Google", + "docs_url": "https://developers.google.com/workspace", + "auth_profiles": [ + { + "id": "user", + "kind": "oauth2_authorization_code", + "identity": "user_delegated", + "supports_pkce": True, + "supports_refresh": True, + "default_scopes": ["drive.file"], + "client_type_by_context": {"hosted": "confidential", "desktop": "public"}, + "owner_by_context": {"hosted": "langflow", "desktop": "langflow"}, + } + ], + "capabilities": [ + { + "id": "google.drive.files.search", + "display_name": "Drive: Search Files", + "auth_profile_id": "user", + "identity": "user_delegated", + "required_scopes": ["drive.file"], + "conditional_scopes": [], + "policy_keys": ["integrations.google.drive.search"], + "substrate": "sdk", + "maturity": "ga", + "deployment_contexts": ["hosted", "self_managed", "desktop", "headless"], + "risk": "read", + "component_ref": "GoogleDriveSearchComponent", + } + ], + } + + +def _write_extension(tmp_path: Path, *, capability_manifest: dict | None = None) -> None: + manifest = { + "id": "lfx-google", + "version": "1.13.0", + "name": "Google", + "lfx": {"compat": ["1"]}, + "bundles": [{"name": "google", "path": "google"}], + } + if capability_manifest is not None: + manifest["integrations"] = [{"provider_id": "google", "bundle": "google", "path": "capabilities.v1.json"}] + (tmp_path / "extension.json").write_text(json.dumps(manifest), encoding="utf-8") + bundle = tmp_path / "google" + bundle.mkdir() + (bundle / "component.py").write_text( + "class Component:\n pass\n\n" + "class GoogleDriveSearchComponent(Component):\n" + " display_name = 'Drive: Search Files'\n" + " def build(self):\n return None\n", + encoding="utf-8", + ) + if capability_manifest is not None: + (bundle / "capabilities.v1.json").write_text(json.dumps(capability_manifest), encoding="utf-8") + + +def test_loader_exposes_validated_integration_metadata(tmp_path: Path) -> None: + _write_extension(tmp_path, capability_manifest=_capability_manifest()) + + result = load_extension(tmp_path, distribution="lfx-google") + + assert result.ok, result.errors + assert len(result.integrations) == 1 + loaded = result.integrations[0] + assert loaded.provider_id == "google" + assert loaded.bundle == "google" + assert loaded.distribution == "lfx-google" + assert loaded.capability_manifest.schema_version == 1 + capability = loaded.capability_manifest.capabilities[0] + assert capability.id == "google.drive.files.search" + assert capability.policy_keys == ("integrations.google.drive.search",) + assert capability.substrate == "sdk" + assert capability.maturity == "ga" + assert capability.deployment_contexts == ("hosted", "self_managed", "desktop", "headless") + + +def test_bundle_registry_exposes_integration_snapshot(tmp_path: Path) -> None: + _write_extension(tmp_path, capability_manifest=_capability_manifest()) + result = load_extension(tmp_path, distribution="lfx-google") + assert result.ok, result.errors + registry = BundleRegistry() + registry.install_bundle( + BundleRecord( + bundle="google", + extension_id="lfx-google", + extension_version="1.13.0", + slot="official", + components=tuple(result.components), + integrations=tuple(result.integrations), + distribution="lfx-google", + source_path=tmp_path, + ) + ) + + assert registry.list_integrations() == result.integrations + + +def test_manifest_without_integrations_loads_unchanged(tmp_path: Path) -> None: + _write_extension(tmp_path) + + result = load_extension(tmp_path) + + assert result.ok, result.errors + assert result.integrations == [] + assert len(result.components) == 1 + + +@pytest.mark.parametrize( + "mutation", + [ + lambda payload: payload.update(schema_version=2), + lambda payload: payload["capabilities"][0].update(substrate="webhook"), + lambda payload: payload["capabilities"][0].update(policy_keys=[]), + lambda payload: payload["capabilities"][0].update(deployment_contexts=["mobile"]), + lambda payload: payload["capabilities"][0].update(component_ref=""), + lambda payload: payload.update(unknown=True), + ], +) +def test_validate_rejects_malformed_capability_manifest(tmp_path: Path, mutation) -> None: + capability_manifest = _capability_manifest() + mutation(capability_manifest) + _write_extension(tmp_path, capability_manifest=capability_manifest) + + report = validate_extension(tmp_path) + + assert not report.ok + assert "manifest-invalid" in [error.code for error in report.errors.errors] + + +def test_loader_rejects_capability_manifest_for_another_provider(tmp_path: Path) -> None: + _write_extension(tmp_path, capability_manifest=_capability_manifest(provider_id="microsoft")) + + result = load_extension(tmp_path) + + assert not result.ok + assert result.integrations == [] + assert [error.code for error in result.errors] == ["manifest-invalid"] + assert "microsoft" in result.errors[0].message + + +def test_validate_rejects_missing_capability_manifest(tmp_path: Path) -> None: + _write_extension(tmp_path, capability_manifest=_capability_manifest()) + (tmp_path / "google" / "capabilities.v1.json").unlink() + + report = validate_extension(tmp_path) + + assert not report.ok + assert [error.code for error in report.errors.errors] == ["manifest-unreadable"] + + +def test_validate_rejects_action_outside_provider_namespace(tmp_path: Path) -> None: + capability_manifest = _capability_manifest() + capability_manifest["capabilities"][0]["id"] = "microsoft.drive.files.search" + _write_extension(tmp_path, capability_manifest=capability_manifest) + + report = validate_extension(tmp_path) + + assert not report.ok + assert "provider namespace" in report.errors.errors[0].message + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks are unreliable on Windows CI") +def test_validate_rejects_capability_manifest_symlink_escape(tmp_path: Path) -> None: + _write_extension(tmp_path, capability_manifest=_capability_manifest()) + outside = tmp_path.parent / f"{tmp_path.name}-outside-capabilities.json" + outside.write_text(json.dumps(_capability_manifest()), encoding="utf-8") + capability_path = tmp_path / "google" / "capabilities.v1.json" + capability_path.unlink() + try: + capability_path.symlink_to(outside) + except (OSError, NotImplementedError): + pytest.skip("symlinks unsupported in this environment") + + report = validate_extension(tmp_path) + + assert not report.ok + assert "path-escape" in [error.code for error in report.errors.errors] diff --git a/src/lfx/tests/unit/extension/test_schema.py b/src/lfx/tests/unit/extension/test_schema.py index e6fba2fb0ffd..826f3c39c578 100644 --- a/src/lfx/tests/unit/extension/test_schema.py +++ b/src/lfx/tests/unit/extension/test_schema.py @@ -119,6 +119,30 @@ def test_schema_validates_v0_example() -> None: _validator().validate(_VALID) +def test_schema_validates_integrations_reference() -> None: + _validator().validate( + { + **_VALID, + "integrations": [{"provider_id": "google", "bundle": "openai", "path": "capabilities.v1.json"}], + } + ) + + +@pytest.mark.parametrize( + "integration", + [ + {"provider_id": "google", "bundle": "openai"}, + {"provider_id": "Google", "bundle": "openai", "path": "capabilities.v1.json"}, + {"provider_id": "google", "bundle": "openai", "path": "../capabilities.json"}, + {"provider_id": "google", "bundle": "openai", "path": "capabilities\u0000.json"}, + {"provider_id": "google", "bundle": "openai", "path": "capabilities.yaml"}, + {"provider_id": "google", "bundle": "openai", "path": "capabilities.json", "extra": True}, + ], +) +def test_schema_rejects_malformed_integrations_reference(integration: dict[str, Any]) -> None: + assert list(_validator().iter_errors({**_VALID, "integrations": [integration]})) + + @pytest.mark.parametrize( "version", ["1.2.3", "1.2.3-alpha.1+build.5", "1.2.3.dev0", "1.2.3a1", "1.2.3b2", "1.2.3rc3"], diff --git a/src/lfx/tests/unit/integrations/test_contracts.py b/src/lfx/tests/unit/integrations/test_contracts.py index 6adff13b397b..22cc3f327ada 100644 --- a/src/lfx/tests/unit/integrations/test_contracts.py +++ b/src/lfx/tests/unit/integrations/test_contracts.py @@ -20,7 +20,7 @@ ScopeCondition, ScopeSet, ) -from lfx.integrations.capabilities import OAuthKind +from lfx.integrations.capabilities import CapabilityMaturity, DeploymentContext, ExecutionSubstrate, OAuthKind from lfx.io.schema import create_input_schema, create_input_schema_from_dict from lfx.schema.dotdict import dotdict from pydantic import SecretStr, ValidationError @@ -89,6 +89,10 @@ def _provider(provider_id: str = "google") -> IntegrationProvider: condition=ScopeCondition(kind="input_truthy", input="write"), ), ), + policy_keys=("integrations.google.drive.read",), + substrate="sdk", + maturity="ga", + deployment_contexts=("hosted", "self_managed", "desktop", "headless"), risk="read", component_ref="GoogleDriveComponent", ) @@ -114,14 +118,24 @@ def test_oauth_profile_kinds_match_discovery_schema() -> None: assert set(get_args(OAuthKind)) == set(schema["$defs"]["auth_mode"]["enum"]) -def test_extension_manifest_accepts_unique_integration_providers() -> None: +def test_capability_enums_match_discovery_schema() -> None: + schema_path = Path(__file__).parents[5] / "design/dedicated-integrations/schema/capability_matrix.schema.json" + schema = json.loads(schema_path.read_text(encoding="utf-8")) + + assert set(get_args(ExecutionSubstrate)) == set(schema["$defs"]["substrate"]["enum"]) + assert set(get_args(CapabilityMaturity)) == set(schema["$defs"]["substrate_ga_status"]["enum"]) + deployment_contexts = schema["$defs"]["action"]["properties"]["deployment_contexts"]["properties"] + assert set(get_args(DeploymentContext)) == set(deployment_contexts) + + +def test_extension_manifest_accepts_unique_integration_references() -> None: manifest_data = { "id": "lfx-google", "version": "1.0.0", "name": "Google", "lfx": {"compat": ["1"]}, "bundles": [{"name": "google", "path": "google"}], - "integrations": [_provider().model_dump(mode="json")], + "integrations": [{"provider_id": "google", "bundle": "google", "path": "capabilities.json"}], } manifest = ExtensionManifest.model_validate(manifest_data) @@ -129,7 +143,7 @@ def test_extension_manifest_accepts_unique_integration_providers() -> None: def test_extension_manifest_rejects_duplicate_integration_provider_ids() -> None: - provider = _provider().model_dump(mode="json") + reference = {"provider_id": "google", "bundle": "google", "path": "capabilities.json"} with pytest.raises(ValidationError, match="must be unique"): ExtensionManifest.model_validate( { @@ -138,7 +152,21 @@ def test_extension_manifest_rejects_duplicate_integration_provider_ids() -> None "name": "Google", "lfx": {"compat": ["1"]}, "bundles": [{"name": "google", "path": "google"}], - "integrations": [provider, provider], + "integrations": [reference, reference], + } + ) + + +def test_extension_manifest_rejects_integration_reference_to_unknown_bundle() -> None: + with pytest.raises(ValidationError, match="unknown bundles"): + ExtensionManifest.model_validate( + { + "id": "lfx-google", + "version": "1.0.0", + "name": "Google", + "lfx": {"compat": ["1"]}, + "bundles": [{"name": "google", "path": "google"}], + "integrations": [{"provider_id": "google", "bundle": "microsoft", "path": "capabilities.json"}], } )