diff --git a/.gitignore b/.gitignore index 30c10388..08076534 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,5 @@ Thumbs.db # Python __pycache__/ *.pyc +backend/autodeploy-detect +tasks/autodeploy-results.jsonl diff --git a/backend/cmd/autodeploy-detect/main.go b/backend/cmd/autodeploy-detect/main.go new file mode 100644 index 00000000..e961d187 --- /dev/null +++ b/backend/cmd/autodeploy-detect/main.go @@ -0,0 +1,49 @@ +// Command autodeploy-detect runs the production source detector over local +// archives and prints one JSON object per archive. +// +// It exists so the autodeploy benchmark (tasks/autodeploy-benchmark-50-oss.md) +// measures the detector that actually ships in the upload path, instead of a +// Python re-implementation of it that could agree with the corpus while the +// real code disagrees. +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/dada-tuda/console/backend/internal/sourcedetect" +) + +type output struct { + Archive string `json:"archive"` + Format string `json:"format"` + Framework string `json:"framework"` + Port int `json:"port"` + Error string `json:"error,omitempty"` +} + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "usage: autodeploy-detect [archive...]") + os.Exit(2) + } + enc := json.NewEncoder(os.Stdout) + for _, path := range os.Args[1:] { + out := output{Archive: path} + data, err := os.ReadFile(path) + if err != nil { + out.Error = err.Error() + } else if res, derr := sourcedetect.Detect(data); derr != nil { + out.Error = derr.Error() + } else { + out.Format = string(res.Format) + out.Framework = res.Framework + out.Port = res.Port + } + if err := enc.Encode(out); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } +} diff --git a/backend/internal/api/databases.go b/backend/internal/api/databases.go index 8d4aeb9c..3fd270c2 100644 --- a/backend/internal/api/databases.go +++ b/backend/internal/api/databases.go @@ -236,6 +236,169 @@ func seedOptimisticSnapshot(ctx context.Context, tx pgx.Tx, projectID, envID uui return err } +// opFault is a rejection a shared handler core hands back to whichever endpoint +// called it: the HTTP status to answer with, the machine-readable reason the +// audit trail records, and the sentence the customer reads. It exists so a core +// can be reused by a second endpoint without either endpoint inventing its own +// status codes or audit vocabulary for the same failure. +type opFault struct { + Status int + Reason string + Message string +} + +// Error makes opFault usable where an error is expected. +func (f *opFault) Error() string { return f.Message } + +// managedDatabaseResult is what provisioning a managed database produced: the +// queued operation, plus the runtime and engine the caller needs for its audit +// record — both are decided inside the core from the environment, so a caller +// that guessed them would be recording a guess. +type managedDatabaseResult struct { + Operation models.Operation + Runtime string + Engine string + Shard string +} + +// createManagedDatabase validates a database request and queues its operation. +// +// This is the whole body of CreateServiceDatabase below except for +// authentication, membership, quota and audit, which stay in the endpoint. It +// is separated so ordering a database as part of installing a ready-made +// project goes through exactly this code: the VM track's credential generation +// and DSN injection are the kind of rules that quietly diverge when a second +// caller reimplements them, and a diverged copy hands the customer an app that +// cannot reach its own database. +// +// VM (compose) environments render the managed database as a platform-owned +// Application in the environment's aggregate stack (postgres image plus an +// external volume). The backend generates the credential and seeds the env vars +// here because it holds the encryption key; the gitops worker only materialises +// the App and re-assembles the stack. k8s keeps the Crossplane path, where the +// chart binds the database to the app through app_ref, so engine stays empty +// and no DSN is seeded. +// +// Quota tier and shard placement belong to the Crossplane path only: a VM +// compose database is a container of its own and is bounded by its own limits. +func (h *Handler) createManagedDatabase(ctx context.Context, actorID, projectID, envID uuid.UUID, req createServiceDatabaseRequest) (*managedDatabaseResult, *opFault) { + if req.Name == "" { + return nil, &opFault{http.StatusBadRequest, "name_required", "name is required"} + } + if req.Database == "" { + return nil, &opFault{http.StatusBadRequest, "database_required", "database is required"} + } + if err := validateKubeName(req.Name); err != nil { + return nil, &opFault{http.StatusBadRequest, "invalid_name", err.Error()} + } + if err := validatePgName(req.Database); err != nil { + return nil, &opFault{http.StatusBadRequest, "invalid_database_name", err.Error()} + } + + var existing int + if err := h.pool.QueryRow(ctx, + `SELECT COUNT(*) FROM resource_snapshots + WHERE project_id = $1 AND environment_id = $2 AND kind = 'ServiceDatabaseV2' AND name = $3`, + projectID, envID, req.Name, + ).Scan(&existing); err != nil { + return nil, &opFault{http.StatusInternalServerError, "uniqueness_check_failed", "failed to check name uniqueness"} + } + if existing > 0 { + return nil, &opFault{http.StatusConflict, "name_taken", "a database with that name already exists in this environment"} + } + + var runtime string + _ = h.pool.QueryRow(ctx, `SELECT runtime FROM environments WHERE id = $1`, envID).Scan(&runtime) + + engine := "" + if runtime == "vm" { + engine = "postgres" + password, perr := randomPassword() + if perr != nil { + return nil, &opFault{http.StatusInternalServerError, "credential_generation_failed", "failed to generate database credential"} + } + const dbUser = "dada" + for _, kv := range [][2]string{ + {"POSTGRES_PASSWORD", password}, + {"POSTGRES_DB", req.Database}, + {"POSTGRES_USER", dbUser}, + } { + if err := h.seedEnvVar(ctx, envID, req.Name, kv[0], kv[1], actorID); err != nil { + return nil, &opFault{http.StatusInternalServerError, "seed_credentials_failed", "failed to seed database credentials"} + } + } + if req.AppRef != "" { + dsn := fmt.Sprintf("postgres://%s:%s@%s:5432/%s", dbUser, password, req.Name, req.Database) + if err := h.seedEnvVar(ctx, envID, req.AppRef, "DATABASE_URL", dsn, actorID); err != nil { + return nil, &opFault{http.StatusInternalServerError, "seed_dsn_failed", "failed to inject database connection string"} + } + } + } + + tier := "" + shard := "" + if engine == "" { + if orgID, orgErr := h.projectOrg(ctx, projectID); orgErr == nil { + tier = h.databaseTierFor(ctx, orgID) + } + shard = h.placeTenantDatabaseShard(ctx) + } + + payload := models.CreateServiceDatabasePayload{ + Name: req.Name, + Database: req.Database, + AppRef: req.AppRef, + Engine: engine, + Tier: tier, + Shard: shard, + BackupEnabled: req.BackupEnabled, + BackupSchedule: req.BackupSchedule, + BackupRetention: req.BackupRetention, + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + return nil, &opFault{http.StatusInternalServerError, "payload_marshal_failed", "failed to marshal payload"} + } + + tx, err := h.pool.Begin(ctx) + if err != nil { + return nil, &opFault{http.StatusInternalServerError, "tx_begin_failed", "failed to create operation"} + } + defer func() { _ = tx.Rollback(ctx) }() + + var op models.Operation + row := tx.QueryRow(ctx, + `INSERT INTO operations (actor_id, project_id, environment_id, action, resource_kind, resource_name, status, payload) + VALUES ($1, $2, $3, 'CreateServiceDatabase', 'ServiceDatabaseV2', $4, 'Created', $5) + RETURNING id, actor_id, project_id, environment_id, action, resource_kind, resource_name, + status, payload, validation_result, git_commit, git_path, argo_application, + error_code, error_message, created_at, updated_at`, + actorID, projectID, envID, req.Name, payloadBytes, + ) + if err = scanOperation(row, &op); err != nil { + return nil, &opFault{http.StatusInternalServerError, "operation_insert_failed", "failed to create operation"} + } + + if err = seedOptimisticSnapshot(ctx, tx, projectID, envID, "ServiceDatabaseV2", req.Name, map[string]any{ + "name": req.Name, + "kind": "ServiceDatabaseV2", + "app_ref": req.AppRef, + "database": req.Database, + "spec": map[string]any{ + "appRef": req.AppRef, + "database": req.Database, + }, + }); err != nil { + return nil, &opFault{http.StatusInternalServerError, "snapshot_seed_failed", "failed to create operation"} + } + + if err = tx.Commit(ctx); err != nil { + return nil, &opFault{http.StatusInternalServerError, "tx_commit_failed", "failed to create operation"} + } + + return &managedDatabaseResult{Operation: op, Runtime: runtime, Engine: engine, Shard: shard}, nil +} + // CreateServiceDatabase enqueues an operation to provision a new ServiceDatabase CRD. // // @ID createDatabase @@ -324,154 +487,20 @@ func (h *Handler) CreateServiceDatabase(c *gin.Context) { return } - // Validate fields - if req.Name == "" { - rejectErr(http.StatusBadRequest, "name_required", "name is required") - return - } - if req.Database == "" { - rejectErr(http.StatusBadRequest, "database_required", "database is required") - return - } // app_ref is optional: empty = standalone, environment-level database that // owns its own chart. When set, the database is bound to that app's chart. - if err := validateKubeName(req.Name); err != nil { - rejectErr(http.StatusBadRequest, "invalid_name", err.Error()) - return - } - if err := validatePgName(req.Database); err != nil { - rejectErr(http.StatusBadRequest, "invalid_database_name", err.Error()) - return - } - - // Check name uniqueness in resource_snapshots for this project/env - var existing int - err = h.pool.QueryRow(c.Request.Context(), - `SELECT COUNT(*) FROM resource_snapshots - WHERE project_id = $1 AND environment_id = $2 AND kind = 'ServiceDatabaseV2' AND name = $3`, - projectID, envID, req.Name, - ).Scan(&existing) - if err != nil { - rejectErr(http.StatusInternalServerError, "uniqueness_check_failed", "failed to check name uniqueness") - return - } - if existing > 0 { - rejectErr(http.StatusConflict, "name_taken", "a database with that name already exists in this environment") - return - } - - // VM (compose) environments render the managed database as a platform-owned - // Application in the environment's aggregate stack (postgres image + external - // volume). The backend generates the credential and seeds env vars now (it - // holds the encryption key); the gitops worker just materialises the App and - // re-assembles the stack. k8s keeps the Crossplane path (engine stays empty). - var runtime string - _ = h.pool.QueryRow(c.Request.Context(), - `SELECT runtime FROM environments WHERE id = $1`, envID).Scan(&runtime) - - engine := "" - if runtime == "vm" { - engine = "postgres" - password, perr := randomPassword() - if perr != nil { - rejectErr(http.StatusInternalServerError, "credential_generation_failed", "failed to generate database credential") - return - } - const dbUser = "dada" - for _, kv := range [][2]string{ - {"POSTGRES_PASSWORD", password}, - {"POSTGRES_DB", req.Database}, - {"POSTGRES_USER", dbUser}, - } { - if err := h.seedEnvVar(c.Request.Context(), envID, req.Name, kv[0], kv[1], claims.UserID); err != nil { - rejectErr(http.StatusInternalServerError, "seed_credentials_failed", "failed to seed database credentials") - return - } - } - if req.AppRef != "" { - dsn := fmt.Sprintf("postgres://%s:%s@%s:5432/%s", dbUser, password, req.Name, req.Database) - if err := h.seedEnvVar(c.Request.Context(), envID, req.AppRef, "DATABASE_URL", dsn, claims.UserID); err != nil { - rejectErr(http.StatusInternalServerError, "seed_dsn_failed", "failed to inject database connection string") - return - } - } - } - - // Quota tier applies to the Crossplane (shared PostgreSQL) path only; a VM - // compose database is a container of its own and is bounded by its own limits. - tier := "" - shard := "" - if engine == "" { - if orgID, orgErr := h.projectOrg(c.Request.Context(), projectID); orgErr == nil { - tier = h.databaseTierFor(c.Request.Context(), orgID) - } - shard = h.placeTenantDatabaseShard(c.Request.Context()) - } - - // Marshal payload - payload := models.CreateServiceDatabasePayload{ - Name: req.Name, - Database: req.Database, - AppRef: req.AppRef, - Engine: engine, - Tier: tier, - Shard: shard, - BackupEnabled: req.BackupEnabled, - BackupSchedule: req.BackupSchedule, - BackupRetention: req.BackupRetention, - } - payloadBytes, err := json.Marshal(payload) - if err != nil { - rejectErr(http.StatusInternalServerError, "payload_marshal_failed", "failed to marshal payload") - return - } - - tx, err := h.pool.Begin(c.Request.Context()) - if err != nil { - rejectErr(http.StatusInternalServerError, "tx_begin_failed", "failed to create operation") + res, fault := h.createManagedDatabase(c.Request.Context(), claims.UserID, projectID, envID, req) + if fault != nil { + rejectErr(fault.Status, fault.Reason, fault.Message) return } - defer func() { _ = tx.Rollback(c.Request.Context()) }() - var op models.Operation - row := tx.QueryRow(c.Request.Context(), - `INSERT INTO operations (actor_id, project_id, environment_id, action, resource_kind, resource_name, status, payload) - VALUES ($1, $2, $3, 'CreateServiceDatabase', 'ServiceDatabaseV2', $4, 'Created', $5) - RETURNING id, actor_id, project_id, environment_id, action, resource_kind, resource_name, - status, payload, validation_result, git_commit, git_path, argo_application, - error_code, error_message, created_at, updated_at`, - claims.UserID, projectID, envID, req.Name, payloadBytes, - ) - if err = scanOperation(row, &op); err != nil { - rejectErr(http.StatusInternalServerError, "operation_insert_failed", "failed to create operation") - return - } - - if err = seedOptimisticSnapshot(c.Request.Context(), tx, projectID, envID, "ServiceDatabaseV2", req.Name, map[string]any{ - "name": req.Name, - "kind": "ServiceDatabaseV2", - "app_ref": req.AppRef, - "database": req.Database, - "spec": map[string]any{ - "appRef": req.AppRef, - "database": req.Database, - }, - }); err != nil { - rejectErr(http.StatusInternalServerError, "snapshot_seed_failed", "failed to create operation") - return - } - - if err = tx.Commit(c.Request.Context()); err != nil { - rejectErr(http.StatusInternalServerError, "tx_commit_failed", "failed to create operation") - return - } - - audit(op.ID, auditOutcomeSuccess, map[string]any{ + audit(res.Operation.ID, auditOutcomeSuccess, map[string]any{ "database": req.Database, "app_ref": req.AppRef, - "engine": engine, - "runtime": runtime, - "shard": shard, + "engine": res.Engine, + "runtime": res.Runtime, + "shard": res.Shard, "backup_enabled": req.BackupEnabled, "backup_schedule": req.BackupSchedule, "backup_retention": req.BackupRetention, @@ -479,7 +508,7 @@ func (h *Handler) CreateServiceDatabase(c *gin.Context) { h.notifyAuditEvent(claims, projectID, "CreateServiceDatabase", req.Name) c.JSON(http.StatusAccepted, gin.H{ - "operation": op, + "operation": res.Operation, "message": "ServiceDatabase creation queued", }) } diff --git a/backend/internal/api/demo_apps.go b/backend/internal/api/demo_apps.go index 8a9db78e..97b28045 100644 --- a/backend/internal/api/demo_apps.go +++ b/backend/internal/api/demo_apps.go @@ -9,32 +9,45 @@ import ( "github.com/dada-tuda/console/backend/internal/auth" "github.com/dada-tuda/console/backend/internal/models" + "github.com/dada-tuda/console/backend/internal/solutions" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/rs/zerolog/log" ) -// demoTemplateRepos is the set of platform-owned starter repositories the -// console offers as a one-click showroom deploy. An app linked to one of these -// is not the user's work: nobody pushes to them, and the only thing they prove -// is that a deploy happens. Membership is an exact match on the full name so a +// legacyDemoTemplateRepos is the set of platform-owned starter repositories the +// console used to offer as its one-click showroom. They are retired in favour of +// the ready-made project catalog (internal/solutions), but they stay listed here +// because apps deployed from them are still out there with a deadline stamped on +// them, and dropping the entry would strand those apps in the projects of people +// who never claimed them. Membership is an exact match on the full name so a // user repository that merely ends in "-starter" is never treated as disposable. -var demoTemplateRepos = map[string]struct{}{ +var legacyDemoTemplateRepos = map[string]struct{}{ "DadaDevelopment/dada-nextjs-starter": {}, "DadaDevelopment/dada-fastapi-starter": {}, "DadaDevelopment/dada-static-starter": {}, } -// isDemoTemplateRepo reports whether repoFullName is one of the platform's own -// starter templates. Case-insensitive because GitHub owner/repo names are. +// isDemoTemplateRepo reports whether an app linked to repoFullName is a showroom +// deploy rather than the customer's own work: a retired starter, or one of the +// catalog's open-source projects. +// +// The catalog half is what keeps the reaper honest after the starters are gone. +// A catalog project is something the platform offered on an empty screen, and +// the one that nobody claims is exactly the app that used to sit Ready for +// eighteen days in a project whose owner never deployed anything of their own. A +// repository the customer pasted themselves is never a demo — they chose it, and +// putting their choice on a timer would be a different product. +// +// Case-insensitive because GitHub owner/repo names are. func isDemoTemplateRepo(repoFullName string) bool { - for known := range demoTemplateRepos { + for known := range legacyDemoTemplateRepos { if strings.EqualFold(known, repoFullName) { return true } } - return false + return solutions.IsCatalogRepo(repoFullName) } // demoAppTTL is the deadline stamped on a starter-template app at link time. diff --git a/backend/internal/api/docs/docs.go b/backend/internal/api/docs/docs.go index c1633c01..f55c3d8e 100644 --- a/backend/internal/api/docs/docs.go +++ b/backend/internal/api/docs/docs.go @@ -2053,6 +2053,62 @@ const docTemplate = `{ } } }, + "/git/parse-repo-url": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Accepts a GitHub browser URL, clone URL, SSH remote or a bare owner/name and returns the canonical owner/name. Rejects anything that is not a public GitHub repository rather than guessing. Pure string handling: it does not check that the repository exists.", + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "Parse a pasted repository link", + "operationId": "parseRepoURL", + "parameters": [ + { + "type": "string", + "description": "Pasted repository link", + "name": "url", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "object with repo_full_name", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/mlflow/registered-models": { "get": { "security": [ @@ -16900,6 +16956,106 @@ const docTemplate = `{ } } }, + "/projects/{projectId}/environments/{envId}/solutions/install": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Links the repository, orders any managed database the project declares it needs (bound to the app, with the connection string injected), and queues the first build — the sequence the console used to run as three calls. Accepts a catalog slug or any public repository. Requires write access.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "Install a ready-made project", + "operationId": "installSolution", + "parameters": [ + { + "type": "string", + "description": "Project UUID", + "name": "projectId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Environment UUID", + "name": "envId", + "in": "path", + "required": true + }, + { + "description": "What to install", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/api.installSolutionRequest" + } + } + ], + "responses": { + "202": { + "description": "object with the app name, the queued build and any database operation", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "403": { + "description": "Forbidden", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/projects/{projectId}/git/detect": { "get": { "security": [ @@ -18104,6 +18260,85 @@ const docTemplate = `{ } } }, + "/projects/{projectId}/solutions/resolve": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Turns a single typed string into a ranked list of things the console can deploy: catalog entries, managed resources, a pasted repository, and GitHub search results below them. Requires write access to the project, because searching spends a rate-limit budget shared by the whole platform.", + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "Resolve what to deploy from one input string", + "operationId": "resolveSolution", + "parameters": [ + { + "type": "string", + "description": "Project UUID", + "name": "projectId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "What the customer typed", + "name": "q", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "object with a candidates array", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "403": { + "description": "Forbidden", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/promo/click": { "post": { "description": "Marks a promo token as clicked. Public, idempotent, and answers identically for unknown tokens.", @@ -18228,6 +18463,96 @@ const docTemplate = `{ } } }, + "/solutions": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Returns the catalog of open-source projects the console can deploy in one click. Each entry is a public repository plus the build spec verified for it (branch, root directory, framework, port, profile); deploying one uses the ordinary connect-repo and build path. Read-only; the catalog is the same for every project.", + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "List ready-made projects", + "operationId": "listSolutions", + "responses": { + "200": { + "description": "object with a solutions array", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/solutions/{slug}": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Returns a single catalog entry with its build spec and any parameters it asks for. Read-only.", + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "Get one ready-made project", + "operationId": "getSolution", + "parameters": [ + { + "type": "string", + "description": "Solution slug", + "name": "slug", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/telemetry/events": { "post": { "description": "Ingests a batch of browser UX events (session_start, pageview, click, input_commit, nav_leave, visibility, error_shown, view) into ux_events, the client-side half of the end-to-end path that audit_events cannot see. Unauthenticated so the pre-login part of the journey is captured; the user is resolved server-side from the dada_uid cookie, never from the payload. Rate-limited per client IP and globally, body and batch size capped, event names checked against a closed set. Carries control names and paths only -- never field values.", @@ -18814,6 +19139,10 @@ const docTemplate = `{ "token": { "description": "GitLab only: a personal/project access token to store encrypted. Ignored for GitHub.", "type": "string" + }, + "worker": { + "description": "Worker marks an app with no HTTP entrypoint: port stays 0, so nothing\ndownstream renders a Service or a default hostname for it.", + "type": "boolean" } } }, @@ -19563,6 +19892,38 @@ const docTemplate = `{ } } }, + "api.installSolutionRequest": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "branch": { + "type": "string" + }, + "framework": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "profile": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "root_dir": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "with_database": { + "type": "boolean" + } + } + }, "api.loginRequest": { "type": "object", "required": [ diff --git a/backend/internal/api/docs/swagger.json b/backend/internal/api/docs/swagger.json index 6a0a2111..a59f460b 100644 --- a/backend/internal/api/docs/swagger.json +++ b/backend/internal/api/docs/swagger.json @@ -2046,6 +2046,62 @@ } } }, + "/git/parse-repo-url": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Accepts a GitHub browser URL, clone URL, SSH remote or a bare owner/name and returns the canonical owner/name. Rejects anything that is not a public GitHub repository rather than guessing. Pure string handling: it does not check that the repository exists.", + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "Parse a pasted repository link", + "operationId": "parseRepoURL", + "parameters": [ + { + "type": "string", + "description": "Pasted repository link", + "name": "url", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "object with repo_full_name", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/mlflow/registered-models": { "get": { "security": [ @@ -16893,6 +16949,106 @@ } } }, + "/projects/{projectId}/environments/{envId}/solutions/install": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Links the repository, orders any managed database the project declares it needs (bound to the app, with the connection string injected), and queues the first build — the sequence the console used to run as three calls. Accepts a catalog slug or any public repository. Requires write access.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "Install a ready-made project", + "operationId": "installSolution", + "parameters": [ + { + "type": "string", + "description": "Project UUID", + "name": "projectId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Environment UUID", + "name": "envId", + "in": "path", + "required": true + }, + { + "description": "What to install", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/api.installSolutionRequest" + } + } + ], + "responses": { + "202": { + "description": "object with the app name, the queued build and any database operation", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "403": { + "description": "Forbidden", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/projects/{projectId}/git/detect": { "get": { "security": [ @@ -18097,6 +18253,85 @@ } } }, + "/projects/{projectId}/solutions/resolve": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Turns a single typed string into a ranked list of things the console can deploy: catalog entries, managed resources, a pasted repository, and GitHub search results below them. Requires write access to the project, because searching spends a rate-limit budget shared by the whole platform.", + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "Resolve what to deploy from one input string", + "operationId": "resolveSolution", + "parameters": [ + { + "type": "string", + "description": "Project UUID", + "name": "projectId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "What the customer typed", + "name": "q", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "object with a candidates array", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "403": { + "description": "Forbidden", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/promo/click": { "post": { "description": "Marks a promo token as clicked. Public, idempotent, and answers identically for unknown tokens.", @@ -18221,6 +18456,96 @@ } } }, + "/solutions": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Returns the catalog of open-source projects the console can deploy in one click. Each entry is a public repository plus the build spec verified for it (branch, root directory, framework, port, profile); deploying one uses the ordinary connect-repo and build path. Read-only; the catalog is the same for every project.", + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "List ready-made projects", + "operationId": "listSolutions", + "responses": { + "200": { + "description": "object with a solutions array", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/solutions/{slug}": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Returns a single catalog entry with its build spec and any parameters it asks for. Read-only.", + "produces": [ + "application/json" + ], + "tags": [ + "solutions" + ], + "summary": "Get one ready-made project", + "operationId": "getSolution", + "parameters": [ + { + "type": "string", + "description": "Solution slug", + "name": "slug", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/telemetry/events": { "post": { "description": "Ingests a batch of browser UX events (session_start, pageview, click, input_commit, nav_leave, visibility, error_shown, view) into ux_events, the client-side half of the end-to-end path that audit_events cannot see. Unauthenticated so the pre-login part of the journey is captured; the user is resolved server-side from the dada_uid cookie, never from the payload. Rate-limited per client IP and globally, body and batch size capped, event names checked against a closed set. Carries control names and paths only -- never field values.", @@ -18807,6 +19132,10 @@ "token": { "description": "GitLab only: a personal/project access token to store encrypted. Ignored for GitHub.", "type": "string" + }, + "worker": { + "description": "Worker marks an app with no HTTP entrypoint: port stays 0, so nothing\ndownstream renders a Service or a default hostname for it.", + "type": "boolean" } } }, @@ -19556,6 +19885,38 @@ } } }, + "api.installSolutionRequest": { + "type": "object", + "properties": { + "app_name": { + "type": "string" + }, + "branch": { + "type": "string" + }, + "framework": { + "type": "string" + }, + "port": { + "type": "integer" + }, + "profile": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "root_dir": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "with_database": { + "type": "boolean" + } + } + }, "api.loginRequest": { "type": "object", "required": [ diff --git a/backend/internal/api/docs/swagger.yaml b/backend/internal/api/docs/swagger.yaml index aab28da7..19b20184 100644 --- a/backend/internal/api/docs/swagger.yaml +++ b/backend/internal/api/docs/swagger.yaml @@ -330,6 +330,11 @@ definitions: description: 'GitLab only: a personal/project access token to store encrypted. Ignored for GitHub.' type: string + worker: + description: |- + Worker marks an app with no HTTP entrypoint: port stays 0, so nothing + downstream renders a Service or a default hostname for it. + type: boolean type: object api.createAIKeyRequest: properties: @@ -838,6 +843,27 @@ definitions: port: type: integer type: object + api.installSolutionRequest: + properties: + app_name: + type: string + branch: + type: string + framework: + type: string + port: + type: integer + profile: + type: string + repo: + type: string + root_dir: + type: string + slug: + type: string + with_database: + type: boolean + type: object api.loginRequest: properties: email: @@ -2616,6 +2642,45 @@ paths: summary: GitHub App install callback (Setup URL) tags: - git + /git/parse-repo-url: + get: + description: 'Accepts a GitHub browser URL, clone URL, SSH remote or a bare + owner/name and returns the canonical owner/name. Rejects anything that is + not a public GitHub repository rather than guessing. Pure string handling: + it does not check that the repository exists.' + operationId: parseRepoURL + parameters: + - description: Pasted repository link + in: query + name: url + required: true + type: string + produces: + - application/json + responses: + "200": + description: object with repo_full_name + schema: + additionalProperties: + type: string + type: object + "400": + description: Bad Request + schema: + additionalProperties: + type: string + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + type: string + type: object + security: + - BearerAuth: [] + summary: Parse a pasted repository link + tags: + - solutions /mlflow/registered-models: get: description: Returns the MLflow registered models visible to a project, filtered @@ -12843,6 +12908,76 @@ paths: summary: Reveal an S3 bucket's access credentials tags: - storage + /projects/{projectId}/environments/{envId}/solutions/install: + post: + consumes: + - application/json + description: Links the repository, orders any managed database the project declares + it needs (bound to the app, with the connection string injected), and queues + the first build — the sequence the console used to run as three calls. Accepts + a catalog slug or any public repository. Requires write access. + operationId: installSolution + parameters: + - description: Project UUID + in: path + name: projectId + required: true + type: string + - description: Environment UUID + in: path + name: envId + required: true + type: string + - description: What to install + in: body + name: body + required: true + schema: + $ref: '#/definitions/api.installSolutionRequest' + produces: + - application/json + responses: + "202": + description: object with the app name, the queued build and any database + operation + schema: + additionalProperties: true + type: object + "400": + description: Bad Request + schema: + additionalProperties: + type: string + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + type: string + type: object + "403": + description: Forbidden + schema: + additionalProperties: + type: string + type: object + "404": + description: Not Found + schema: + additionalProperties: + type: string + type: object + "409": + description: Conflict + schema: + additionalProperties: + type: string + type: object + security: + - BearerAuth: [] + summary: Install a ready-made project + tags: + - solutions /projects/{projectId}/git/detect: get: description: Best-effort framework detection for a public GitHub repository @@ -13666,6 +13801,61 @@ paths: summary: Get a project's quotas and usage tags: - quota + /projects/{projectId}/solutions/resolve: + get: + description: 'Turns a single typed string into a ranked list of things the console + can deploy: catalog entries, managed resources, a pasted repository, and GitHub + search results below them. Requires write access to the project, because searching + spends a rate-limit budget shared by the whole platform.' + operationId: resolveSolution + parameters: + - description: Project UUID + in: path + name: projectId + required: true + type: string + - description: What the customer typed + in: query + name: q + required: true + type: string + produces: + - application/json + responses: + "200": + description: object with a candidates array + schema: + additionalProperties: true + type: object + "400": + description: Bad Request + schema: + additionalProperties: + type: string + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + type: string + type: object + "403": + description: Forbidden + schema: + additionalProperties: + type: string + type: object + "404": + description: Not Found + schema: + additionalProperties: + type: string + type: object + security: + - BearerAuth: [] + summary: Resolve what to deploy from one input string + tags: + - solutions /projects/default: post: description: Returns the caller's default project, provisioning one (in your @@ -13780,6 +13970,69 @@ paths: summary: Redeem a campaign promo token tags: - growth + /solutions: + get: + description: Returns the catalog of open-source projects the console can deploy + in one click. Each entry is a public repository plus the build spec verified + for it (branch, root directory, framework, port, profile); deploying one uses + the ordinary connect-repo and build path. Read-only; the catalog is the same + for every project. + operationId: listSolutions + produces: + - application/json + responses: + "200": + description: object with a solutions array + schema: + additionalProperties: true + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + type: string + type: object + security: + - BearerAuth: [] + summary: List ready-made projects + tags: + - solutions + /solutions/{slug}: + get: + description: Returns a single catalog entry with its build spec and any parameters + it asks for. Read-only. + operationId: getSolution + parameters: + - description: Solution slug + in: path + name: slug + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + additionalProperties: true + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + type: string + type: object + "404": + description: Not Found + schema: + additionalProperties: + type: string + type: object + security: + - BearerAuth: [] + summary: Get one ready-made project + tags: + - solutions /telemetry/events: post: consumes: diff --git a/backend/internal/api/gitrepos.go b/backend/internal/api/gitrepos.go index 4151c430..65340972 100644 --- a/backend/internal/api/gitrepos.go +++ b/backend/internal/api/gitrepos.go @@ -1063,25 +1063,60 @@ func (h *Handler) ConnectGitRepo(c *gin.Context) { } appAudit = req.AppName - if req.RepoFullName == "" { - rejectErr(http.StatusBadRequest, "missing_repo_full_name", "repo_full_name is required") + r, fault := h.linkGitRepo(c.Request.Context(), claims.UserID, projectID, envID, &req) + if fault != nil { + if fault.Status == http.StatusNotFound { + reject(fault.Status, fault.Reason, func() { respondNotFound(c) }) + return + } + rejectErr(fault.Status, fault.Reason, fault.Message) return } + provider := r.Provider + + h.recordAudit(c.Request.Context(), claims.UserID, auditEntry{ + ProjectID: projectID, + EnvironmentID: envID, + Action: "ConnectGitRepo", + ResourceKind: "GitRepo", + ResourceName: req.AppName, + Outcome: auditOutcomeSuccess, + Metadata: map[string]any{ + "provider": provider, + "repo": req.RepoFullName, + "branch": req.ProductionBranch, + "auto_deploy": req.AutoDeploy, + }, + }) + h.notifyAuditEvent(claims, projectID, "ConnectGitRepo", req.AppName) + + c.JSON(http.StatusCreated, gin.H{"repos": []gitRepo{*r}}) +} + +// linkGitRepo validates a link request, applies the server-side defaults and +// writes the git_repos row. +// +// Split out of ConnectGitRepo so installing a ready-made project links its +// repository through the same code rather than through a second copy of these +// rules. The request is taken by pointer because the defaults it fills in +// (branch, root dir, port, replicas, profile) are what the caller must record +// afterwards — an audit line saying "branch: " when the row says "main" is a +// log that lies. +func (h *Handler) linkGitRepo(ctx context.Context, actorID, projectID, envID uuid.UUID, req *connectGitRepoRequest) (*gitRepo, *opFault) { + if req.RepoFullName == "" { + return nil, &opFault{http.StatusBadRequest, "missing_repo_full_name", "repo_full_name is required"} + } if req.AppName == "" { - rejectErr(http.StatusBadRequest, "missing_app_name", "app_name is required") - return + return nil, &opFault{http.StatusBadRequest, "missing_app_name", "app_name is required"} } if err := validateKubeName(req.AppName); err != nil { - rejectErr(http.StatusBadRequest, "invalid_app_name", err.Error()) - return + return nil, &opFault{http.StatusBadRequest, "invalid_app_name", err.Error()} } - provider := req.Provider - if provider == "" { - provider = "github" + if req.Provider == "" { + req.Provider = "github" } - if provider != "github" && provider != "gitlab" { - rejectErr(http.StatusBadRequest, "invalid_provider", "provider must be github or gitlab") - return + if req.Provider != "github" && req.Provider != "gitlab" { + return nil, &opFault{http.StatusBadRequest, "invalid_provider", "provider must be github or gitlab"} } if req.ProductionBranch == "" { req.ProductionBranch = "main" @@ -1089,7 +1124,6 @@ func (h *Handler) ConnectGitRepo(c *gin.Context) { if req.RootDir == "" { req.RootDir = "." } - // Intended app spec (applied when the first build creates the app). if req.Port == 0 && !req.Worker { req.Port = 8080 } @@ -1100,16 +1134,13 @@ func (h *Handler) ConnectGitRepo(c *gin.Context) { req.Profile = "small" } if !req.Worker && (req.Port < 1 || req.Port > 65535) { - rejectErr(http.StatusBadRequest, "invalid_port", "port must be between 1 and 65535") - return + return nil, &opFault{http.StatusBadRequest, "invalid_port", "port must be between 1 and 65535"} } if req.Replicas < 1 || req.Replicas > 10 { - rejectErr(http.StatusBadRequest, "invalid_replicas", "replicas must be between 1 and 10") - return + return nil, &opFault{http.StatusBadRequest, "invalid_replicas", "replicas must be between 1 and 10"} } if req.Profile != "small" && req.Profile != "medium" && req.Profile != "large" { - rejectErr(http.StatusBadRequest, "invalid_profile", "profile must be one of: small, medium, large") - return + return nil, &opFault{http.StatusBadRequest, "invalid_profile", "profile must be one of: small, medium, large"} } cloneURL := req.CloneURL if cloneURL == "" { @@ -1126,36 +1157,33 @@ func (h *Handler) ConnectGitRepo(c *gin.Context) { var resolved uuid.UUID var qerr error if instUUID, perr := uuid.Parse(req.InstallationID); perr == nil { - qerr = h.pool.QueryRow(c.Request.Context(), + qerr = h.pool.QueryRow(ctx, `SELECT gai.id FROM git_app_installations gai JOIN projects p ON p.org_id = gai.org_id WHERE gai.id = $1 AND p.id = $2`, instUUID, projectID, ).Scan(&resolved) } else if numeric, nerr := strconv.ParseInt(req.InstallationID, 10, 64); nerr == nil { - qerr = h.pool.QueryRow(c.Request.Context(), + qerr = h.pool.QueryRow(ctx, `SELECT gai.id FROM git_app_installations gai JOIN projects p ON p.org_id = gai.org_id WHERE gai.installation_id = $1 AND p.id = $2`, numeric, projectID, ).Scan(&resolved) } else { - rejectErr(http.StatusBadRequest, "invalid_installation_id", "installation_id must be the installation id (UUID) or its numeric GitHub installation id") - return + return nil, &opFault{http.StatusBadRequest, "invalid_installation_id", "installation_id must be the installation id (UUID) or its numeric GitHub installation id"} } if qerr == pgx.ErrNoRows { - reject(http.StatusNotFound, "installation_not_found", func() { respondNotFound(c) }) - return + return nil, &opFault{http.StatusNotFound, "installation_not_found", "installation not found"} } if qerr != nil { - rejectErr(http.StatusInternalServerError, "installation_check_failed", "failed to verify installation") - return + return nil, &opFault{http.StatusInternalServerError, "installation_check_failed", "failed to verify installation"} } installationID = &resolved } - if installationID == nil && provider == "github" { - if resolved, ok := h.resolveInstallationByOwner(c.Request.Context(), projectID, req.RepoFullName); ok { + if installationID == nil && req.Provider == "github" { + if resolved, ok := h.resolveInstallationByOwner(ctx, projectID, req.RepoFullName); ok { installationID = &resolved } } @@ -1163,11 +1191,11 @@ func (h *Handler) ConnectGitRepo(c *gin.Context) { // GitLab token (optional) — store encrypted. var tokenEncrypted []byte if req.Token != "" { - tokenEncrypted, err = crypto.EncryptToken(h.cfg.GitopsEncryptionKey, []byte(req.Token)) + enc, err := crypto.EncryptToken(h.cfg.GitopsEncryptionKey, []byte(req.Token)) if err != nil { - rejectErr(http.StatusInternalServerError, "token_encrypt_failed", "failed to encrypt token") - return + return nil, &opFault{http.StatusInternalServerError, "token_encrypt_failed", "failed to encrypt token"} } + tokenEncrypted = enc } var frameworkOverride *string @@ -1176,11 +1204,10 @@ func (h *Handler) ConnectGitRepo(c *gin.Context) { } webhookSecret := randomHex(32) - demoExpiresAt := h.demoExpiryFor(req.RepoFullName) var r gitRepo - row := h.pool.QueryRow(c.Request.Context(), + row := h.pool.QueryRow(ctx, `INSERT INTO git_repos (project_id, environment_id, app_name, installation_id, provider, repo_full_name, clone_url, token_encrypted, webhook_secret, @@ -1190,41 +1217,22 @@ func (h *Handler) ConnectGitRepo(c *gin.Context) { RETURNING id, project_id, environment_id, app_name, installation_id, provider, repo_full_name, production_branch, root_dir, framework_override, auto_deploy, port, replicas, profile, worker, created_at, updated_at`, - projectID, envID, req.AppName, installationID, provider, + projectID, envID, req.AppName, installationID, req.Provider, req.RepoFullName, cloneURL, tokenEncrypted, webhookSecret, req.ProductionBranch, req.RootDir, frameworkOverride, req.AutoDeploy, - req.Port, req.Replicas, req.Profile, req.Worker, claims.UserID, demoExpiresAt, + req.Port, req.Replicas, req.Profile, req.Worker, actorID, demoExpiresAt, ) if err := row.Scan(&r.ID, &r.ProjectID, &r.EnvironmentID, &r.AppName, &r.InstallationID, &r.Provider, &r.RepoFullName, &r.ProductionBranch, &r.RootDir, &r.FrameworkOverride, &r.AutoDeploy, &r.Port, &r.Replicas, &r.Profile, &r.Worker, &r.CreatedAt, &r.UpdatedAt); err != nil { if isUniqueViolation(err) { - rejectErr(http.StatusConflict, "repo_already_linked", "this app already has a linked repository in this environment") - return + return nil, &opFault{http.StatusConflict, "repo_already_linked", "this app already has a linked repository in this environment"} } - rejectErr(http.StatusInternalServerError, "link_insert_failed", "failed to link repository") - return + return nil, &opFault{http.StatusInternalServerError, "link_insert_failed", "failed to link repository"} } r.PlatformAccess = classifyPlatformAccess(r.Provider, r.InstallationID) - - h.recordAudit(c.Request.Context(), claims.UserID, auditEntry{ - ProjectID: projectID, - EnvironmentID: envID, - Action: "ConnectGitRepo", - ResourceKind: "GitRepo", - ResourceName: req.AppName, - Outcome: auditOutcomeSuccess, - Metadata: map[string]any{ - "provider": provider, - "repo": req.RepoFullName, - "branch": req.ProductionBranch, - "auto_deploy": req.AutoDeploy, - }, - }) - h.notifyAuditEvent(claims, projectID, "ConnectGitRepo", req.AppName) - - c.JSON(http.StatusCreated, gin.H{"repos": []gitRepo{r}}) + return &r, nil } // DisconnectGitRepo unlinks a repository from an app. diff --git a/backend/internal/api/router.go b/backend/internal/api/router.go index cbd9010e..c6da1768 100644 --- a/backend/internal/api/router.go +++ b/backend/internal/api/router.go @@ -377,6 +377,15 @@ func SetupRouter(pool *pgxpool.Pool, cfg *config.Config) *gin.Engine { api.POST("/projects/:projectId/app-servers/:serverName/discover", h.DiscoverWorkload) api.POST("/projects/:projectId/app-servers/:serverName/import", h.ImportComposeStack) + // Ready-made projects. The catalog is global and read-only; installing + // one runs the ordinary connect-repo + build path server-side, plus the + // managed database the entry declares it needs. + api.GET("/solutions", h.ListSolutions) + api.GET("/solutions/:slug", h.GetSolution) + api.GET("/git/parse-repo-url", h.ParseRepoURL) + api.GET("/projects/:projectId/solutions/resolve", h.ResolveSolution) + api.POST("/projects/:projectId/environments/:envId/solutions/install", h.InstallSolution) + // Boxes (ephemeral root sandboxes). A box owns exactly one environment // with runtime='box'; crystallization later promotes that same row to // runtime='vm', which is how its attachments and hostnames survive. diff --git a/backend/internal/api/solutions.go b/backend/internal/api/solutions.go new file mode 100644 index 00000000..824036b2 --- /dev/null +++ b/backend/internal/api/solutions.go @@ -0,0 +1,592 @@ +package api + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/dada-tuda/console/backend/internal/auth" + "github.com/dada-tuda/console/backend/internal/buildagent" + "github.com/dada-tuda/console/backend/internal/cache" + "github.com/dada-tuda/console/backend/internal/solutions" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// Ready-made projects: the catalog that replaces the starter templates on the +// empty-project screen. +// +// These endpoints are deliberately thin. A catalog entry is a public repository +// plus the build spec we verified for it, and deploying one runs the EXISTING +// customer path — connect the repo, build it, deploy the image — rather than a +// parallel install mechanism. So the backend's whole job here is to hand out +// the catalog and to turn whatever the customer pasted into a repository name; +// everything after that is the same code a customer's own first deploy goes +// through, which is the point (see internal/solutions). + +// solutionPayload is the wire shape of one catalog entry. Rendered by hand +// rather than by marshalling solutions.Solution, so the API surface stays a +// deliberate decision rather than a side effect of adding a field. +func solutionPayload(s solutions.Solution) gin.H { + params := make([]gin.H, 0, len(s.Params)) + for _, p := range s.Params { + params = append(params, gin.H{ + "key": p.Key, + "label": p.Label, + "help": p.Help, + "kind": string(p.Kind), + "required": p.Required, + "default": p.Default, + "options": p.Options, + "placeholder": p.Placeholder, + }) + } + return gin.H{ + "slug": s.Slug, + "name": s.Name, + "tagline": s.Tagline, + "about": s.About, + "bullets": s.Bullets, + "category": string(s.Category), + "homepage": s.Homepage, + "license": s.License, + "repo": s.Repo, + "branch": s.Branch, + "root_dir": s.RootDir, + "framework": s.Framework, + "port": s.Port, + "profile": s.Profile, + "warning": s.Warning, + "first_run": s.FirstRun, + "build_note": s.BuildNote, + "params": params, + } +} + +// ListSolutions returns the ready-made project catalog. +// +// @ID listSolutions +// @Summary List ready-made projects +// @Description Returns the catalog of open-source projects the console can deploy in one click. Each entry is a public repository plus the build spec verified for it (branch, root directory, framework, port, profile); deploying one uses the ordinary connect-repo and build path. Read-only; the catalog is the same for every project. +// @Tags solutions +// @Produce json +// @Security BearerAuth +// @Success 200 {object} map[string]interface{} "object with a solutions array" +// @Failure 401 {object} map[string]string +// @Router /solutions [get] +func (h *Handler) ListSolutions(c *gin.Context) { + if _, ok := auth.GetClaims(c); !ok { + respondUnauthorized(c) + return + } + out := make([]gin.H, 0, len(solutions.V1)) + for _, s := range solutions.V1 { + out = append(out, solutionPayload(s)) + } + c.JSON(http.StatusOK, gin.H{"solutions": out}) +} + +// GetSolution returns one catalog entry by slug. +// +// @ID getSolution +// @Summary Get one ready-made project +// @Description Returns a single catalog entry with its build spec and any parameters it asks for. Read-only. +// @Tags solutions +// @Produce json +// @Security BearerAuth +// @Param slug path string true "Solution slug" +// @Success 200 {object} map[string]interface{} +// @Failure 401 {object} map[string]string +// @Failure 404 {object} map[string]string +// @Router /solutions/{slug} [get] +func (h *Handler) GetSolution(c *gin.Context) { + if _, ok := auth.GetClaims(c); !ok { + respondUnauthorized(c) + return + } + s, ok := solutions.Lookup(c.Param("slug")) + if !ok { + respondNotFound(c) + return + } + c.JSON(http.StatusOK, solutionPayload(s)) +} + +// ParseRepoURL turns a pasted repository link into an "owner/name" pair. +// +// It exists so the "deploy any public repository" field has ONE definition of +// what a repository link is. The browser URL, the clone URL, the SSH remote and +// a bare owner/name all end up on a clipboard, and a rule that lives in both the +// console and the backend drifts until the two disagree about what the customer +// pasted — at which point one of them deploys the wrong repository. +// +// @ID parseRepoURL +// @Summary Parse a pasted repository link +// @Description Accepts a GitHub browser URL, clone URL, SSH remote or a bare owner/name and returns the canonical owner/name. Rejects anything that is not a public GitHub repository rather than guessing. Pure string handling: it does not check that the repository exists. +// @Tags solutions +// @Produce json +// @Security BearerAuth +// @Param url query string true "Pasted repository link" +// @Success 200 {object} map[string]string "object with repo_full_name" +// @Failure 400 {object} map[string]string +// @Failure 401 {object} map[string]string +// @Router /git/parse-repo-url [get] +func (h *Handler) ParseRepoURL(c *gin.Context) { + if _, ok := auth.GetClaims(c); !ok { + respondUnauthorized(c) + return + } + full, err := solutions.ParseRepoURL(c.Query("url")) + if err != nil { + respondError(c, http.StatusBadRequest, err.Error()) + return + } + c.JSON(http.StatusOK, gin.H{"repo_full_name": full}) +} + +// searchCacheTTL caches one GitHub search answer. +// +// The search endpoint allows 30 requests a minute per source IP for the entire +// cluster, and an interactive input is the fastest way ever invented to spend +// such a budget. Popular queries repeat across customers — "n8n", "postgres", +// "wordpress" — so a shared cache with a short life gives the second person to +// type a word an instant answer and costs the first one nothing. Short rather +// than long because the point of search is that it reaches things the catalog +// does not, including repositories published this morning. +const searchCacheTTL = 30 * time.Minute + +// searchResultLimit is how many search rows the console shows under the input. +const searchResultLimit = 6 + +// candidatePayload is the wire shape of one resolver row. +func candidatePayload(c solutions.Candidate) gin.H { + return gin.H{ + "kind": string(c.Kind), + "slug": c.Slug, + "name": c.Name, + "tagline": c.Tagline, + "icon": c.Icon, + "repo": c.Repo, + "branch": c.Branch, + "root_dir": c.RootDir, + "framework": c.Framework, + "port": c.Port, + "profile": c.Profile, + "engine": c.Engine, + } +} + +// ResolveSolution answers the console's single "what do you want to run?" field. +// +// One field, one ranked list, three audiences: the catalog entry for the person +// who types a product name, the managed database for the person who types +// "post", and a repository for the person who pastes a link. Anything the local +// catalog cannot answer falls through to a GitHub search, appended BELOW the +// local rows — a curated entry carries a build spec we verified, a search hit +// carries a name and a star count. +// +// Search failure is not request failure. GitHub rate-limits, GitHub has +// outages, and an App installation can be removed; in every one of those cases +// the customer still gets the catalog rows and a `search_failed` flag, because +// a suggestion list that goes blank reads as "this platform has nothing for +// you" rather than as a temporary upstream problem. A build-agent that is not +// configured at all counts as the same kind of failure: the search was owed and +// did not happen, and reporting searched=true with no rows and no flag would +// blame the customer's query for our missing dependency. +// +// @ID resolveSolution +// @Summary Resolve what to deploy from one input string +// @Description Turns a single typed string into a ranked list of things the console can deploy: catalog entries, managed resources, a pasted repository, and GitHub search results below them. Requires write access to the project, because searching spends a rate-limit budget shared by the whole platform. +// @Tags solutions +// @Produce json +// @Security BearerAuth +// @Param projectId path string true "Project UUID" +// @Param q query string true "What the customer typed" +// @Success 200 {object} map[string]interface{} "object with a candidates array" +// @Failure 400 {object} map[string]string +// @Failure 401 {object} map[string]string +// @Failure 403 {object} map[string]string +// @Failure 404 {object} map[string]string +// @Router /projects/{projectId}/solutions/resolve [get] +func (h *Handler) ResolveSolution(c *gin.Context) { + claims, ok := auth.GetClaims(c) + if !ok { + respondUnauthorized(c) + return + } + projectID, err := uuid.Parse(c.Param("projectId")) + if err != nil { + respondNotFound(c) + return + } + role, err := h.effectiveRole(c.Request.Context(), claims, projectID) + if err == pgx.ErrNoRows { + respondNotFound(c) + return + } + if err != nil { + respondError(c, http.StatusInternalServerError, "failed to check project membership") + return + } + if !canWrite(role) { + respondForbidden(c) + return + } + + query := strings.TrimSpace(c.Query("q")) + if query == "" { + respondError(c, http.StatusBadRequest, "q is required") + return + } + if len(query) > 200 { + respondError(c, http.StatusBadRequest, "q is too long") + return + } + + res := solutions.Resolve(query) + out := make([]gin.H, 0, len(res.Candidates)+searchResultLimit) + for _, cand := range res.Candidates { + out = append(out, candidatePayload(cand)) + } + + searchFailed := res.SearchQuery != "" && h.buildagent == nil + if res.SearchQuery != "" && h.buildagent != nil { + hits, err := cache.Fetch(c.Request.Context(), h.cache, + fmt.Sprintf("git:search:public:%s", strings.ToLower(res.SearchQuery)), searchCacheTTL, + func() (*[]buildagent.SearchHit, error) { + found, err := h.buildagent.SearchRepos(c.Request.Context(), res.SearchQuery, searchResultLimit) + if err != nil { + return nil, err + } + return &found, nil + }) + if err != nil { + searchFailed = true + } else if hits != nil { + for _, hit := range *hits { + if solutions.IsCatalogRepo(hit.FullName) { + continue + } + out = append(out, gin.H{ + "kind": string(solutions.CandidateRepo), + "slug": hit.FullName, + "name": repoShortName(hit.FullName), + "tagline": hit.Description, + "icon": hit.AvatarURL, + "repo": hit.FullName, + "branch": hit.DefaultBranch, + "root_dir": ".", + "stars": hit.Stars, + "license": hit.License, + "archived": hit.Archived, + "from": "search", + "homepage": hit.HTMLURL, + "framework": "", + "port": 0, + "profile": "", + "engine": "", + }) + } + } + } + + c.JSON(http.StatusOK, gin.H{ + "query": query, + "candidates": out, + "searched": res.SearchQuery != "", + "search_failed": searchFailed, + }) +} + +// installSolutionRequest is one "install this" click. +// +// Everything except what the customer picked is optional: a catalog slug +// carries the verified build spec, and a bare repository falls back to the same +// server-side defaults the connect-repo endpoint applies. WithDatabase is a +// pointer so "the customer explicitly said no" is distinguishable from "the +// customer said nothing", which is the only way a catalog entry that declares +// Needs can default to yes and still be refusable. +type installSolutionRequest struct { + Slug string `json:"slug"` + Repo string `json:"repo"` + AppName string `json:"app_name"` + Branch string `json:"branch"` + RootDir string `json:"root_dir"` + Framework string `json:"framework"` + Port int `json:"port"` + Profile string `json:"profile"` + WithDatabase *bool `json:"with_database"` +} + +// managedDatabaseNameFor derives the database resource name and PostgreSQL +// database name for an app that asked for one. +// +// Both are derived rather than asked for because the customer installing a +// ready-made project has no opinion about either, and a name they never chose +// is a name they cannot get wrong. The resource keeps the "-db" suffix so it +// reads as the app's database in every list; the database itself reuses the app +// name, prefixed when the app name starts with a digit because validatePgName +// requires a leading letter. +func managedDatabaseNameFor(appName string) (resource, database string) { + resource = appName + "-db" + if len(resource) > 63 { + resource = resource[:63] + } + database = appName + if database == "" || database[0] < 'a' || database[0] > 'z' { + database = "db-" + database + } + if len(database) > 63 { + database = database[:63] + } + return resource, database +} + +// appNameForInstall picks the app name an install lands on. +func appNameForInstall(req installSolutionRequest, repoFullName string) string { + if req.AppName != "" { + return req.AppName + } + if req.Slug != "" { + return req.Slug + } + name := strings.ToLower(repoShortName(repoFullName)) + var b strings.Builder + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + return strings.Trim(b.String(), "-") +} + +// InstallSolution turns one click into a running project. +// +// The console used to assemble this itself: link the repository, then order a +// database, then trigger a build, three calls it had to keep in the right order +// and unwind by hand when the middle one failed. That is a sequence, not a +// decision, so it belongs on the server — and putting it here is what lets a +// catalog entry declare `needs: [postgres]` and have the database appear +// already bound to the app, instead of the customer reading a "now create a +// database" instruction under a project that does not work yet. +// +// It composes the existing cores rather than reimplementing them: linkGitRepo +// applies the same defaults and installation resolution the connect-repo +// endpoint applies, and createManagedDatabase generates the credential and +// seeds DATABASE_URL exactly as ordering a database by hand does. +// +// Failure is reported, not unwound. If the database order fails after the +// repository is linked, the link stays and the response says so: the link is +// the part the customer can see and reuse, and silently deleting it would turn +// a recoverable half-install into a mystery. +// +// @ID installSolution +// @Summary Install a ready-made project +// @Description Links the repository, orders any managed database the project declares it needs (bound to the app, with the connection string injected), and queues the first build — the sequence the console used to run as three calls. Accepts a catalog slug or any public repository. Requires write access. +// @Tags solutions +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param projectId path string true "Project UUID" +// @Param envId path string true "Environment UUID" +// @Param body body installSolutionRequest true "What to install" +// @Success 202 {object} map[string]interface{} "object with the app name, the queued build and any database operation" +// @Failure 400 {object} map[string]string +// @Failure 401 {object} map[string]string +// @Failure 403 {object} map[string]string +// @Failure 404 {object} map[string]string +// @Failure 409 {object} map[string]string +// @Router /projects/{projectId}/environments/{envId}/solutions/install [post] +func (h *Handler) InstallSolution(c *gin.Context) { + claims, ok := auth.GetClaims(c) + if !ok { + respondUnauthorized(c) + return + } + projectID, err := uuid.Parse(c.Param("projectId")) + if err != nil { + respondNotFound(c) + return + } + envID, err := uuid.Parse(c.Param("envId")) + if err != nil { + respondNotFound(c) + return + } + + role, err := h.effectiveRole(c.Request.Context(), claims, projectID) + if err == pgx.ErrNoRows { + respondNotFound(c) + return + } + if err != nil { + respondError(c, http.StatusInternalServerError, "failed to check project membership") + return + } + if !canWrite(role) { + respondForbidden(c) + return + } + + appAudit := "" + audit := func(outcome string, meta map[string]any) { + h.recordAudit(c.Request.Context(), claims.UserID, auditEntry{ + ProjectID: projectID, + EnvironmentID: envID, + Action: "InstallSolution", + ResourceKind: "Solution", + ResourceName: appAudit, + Outcome: outcome, + Metadata: meta, + }) + } + rejectErr := func(status int, reason, msg string) { + audit(auditOutcomeFailure, map[string]any{"reason": reason, "status": status}) + respondError(c, status, msg) + } + + var req installSolutionRequest + if err := c.ShouldBindJSON(&req); err != nil { + rejectErr(http.StatusBadRequest, "malformed_body", err.Error()) + return + } + + link := connectGitRepoRequest{ + Provider: "github", + ProductionBranch: req.Branch, + RootDir: req.RootDir, + Port: req.Port, + Profile: req.Profile, + AutoDeploy: true, + } + needsDatabase := false + + if req.Slug != "" { + s, found := solutions.Lookup(req.Slug) + if !found { + rejectErr(http.StatusNotFound, "unknown_solution", "no such ready-made project") + return + } + link.RepoFullName = s.Repo + if link.ProductionBranch == "" { + link.ProductionBranch = s.Branch + } + if link.RootDir == "" { + link.RootDir = s.RootDir + } + if link.Port == 0 { + link.Port = s.Port + } + if link.Profile == "" { + link.Profile = s.Profile + } + link.FrameworkOverride = s.Framework + for _, need := range s.Needs { + if need == "postgres" { + needsDatabase = true + } + } + } else { + if req.Repo == "" { + rejectErr(http.StatusBadRequest, "missing_repo", "slug or repo is required") + return + } + full, perr := solutions.ParseRepoURL(req.Repo) + if perr != nil { + rejectErr(http.StatusBadRequest, "invalid_repo", perr.Error()) + return + } + link.RepoFullName = full + link.FrameworkOverride = req.Framework + } + + if req.WithDatabase != nil { + needsDatabase = *req.WithDatabase + } + + link.AppName = appNameForInstall(req, link.RepoFullName) + appAudit = link.AppName + if err := validateKubeName(link.AppName); err != nil { + rejectErr(http.StatusBadRequest, "invalid_app_name", err.Error()) + return + } + + repo, fault := h.linkGitRepo(c.Request.Context(), claims.UserID, projectID, envID, &link) + if fault != nil { + rejectErr(fault.Status, fault.Reason, fault.Message) + return + } + + var dbOperation any + if needsDatabase { + resource, database := managedDatabaseNameFor(link.AppName) + res, dbFault := h.createManagedDatabase(c.Request.Context(), claims.UserID, projectID, envID, createServiceDatabaseRequest{ + Name: resource, + Database: database, + AppRef: link.AppName, + }) + if dbFault != nil { + audit(auditOutcomeFailure, map[string]any{ + "reason": dbFault.Reason, + "status": dbFault.Status, + "stage": "database", + "repo_linked": true, + "app": link.AppName, + }) + respondError(c, dbFault.Status, dbFault.Message) + return + } + dbOperation = res.Operation + } + + var b build + row := h.pool.QueryRow(c.Request.Context(), + `INSERT INTO builds + (git_repo_id, environment_id, app_name, commit_sha, branch, triggered_by, trigger, status) + VALUES ($1, $2, $3, $4, $5, $6, 'manual', 'queued') + RETURNING `+buildSelectCols, + repo.ID, envID, link.AppName, placeholderCommitSHA(), link.ProductionBranch, claims.UserID, + ) + if err := scanBuild(row, &b); err != nil { + audit(auditOutcomeFailure, map[string]any{ + "reason": "queue_failed", + "stage": "build", + "repo_linked": true, + "app": link.AppName, + }) + respondError(c, http.StatusInternalServerError, "failed to queue build") + return + } + + audit(auditOutcomeSuccess, map[string]any{ + "slug": req.Slug, + "repo": link.RepoFullName, + "branch": link.ProductionBranch, + "app": link.AppName, + "database": needsDatabase, + "build_id": b.ID.String(), + }) + h.notifyAuditEvent(claims, projectID, "InstallSolution", link.AppName) + + c.JSON(http.StatusAccepted, gin.H{ + "app_name": link.AppName, + "repo": *repo, + "build": b, + "database": dbOperation, + "installed": true, + }) +} + +// repoShortName is the repository half of "owner/name". +func repoShortName(full string) string { + if _, name, ok := strings.Cut(full, "/"); ok && name != "" { + return name + } + return full +} diff --git a/backend/internal/api/solutions_install_test.go b/backend/internal/api/solutions_install_test.go new file mode 100644 index 00000000..1b8630d5 --- /dev/null +++ b/backend/internal/api/solutions_install_test.go @@ -0,0 +1,316 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/dada-tuda/console/backend/internal/auth" + "github.com/dada-tuda/console/backend/internal/config" + "github.com/dada-tuda/console/backend/internal/crypto" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" +) + +// installTestKey is a throwaway 32-byte hex key: the install path encrypts the +// seeded database credentials, so a handler without one cannot run at all. +const installTestKey = "ab000000000000000000000000000000000000000000000000000000000000cd" + +func TestAppNameForInstall(t *testing.T) { + cases := []struct { + name string + req installSolutionRequest + repo string + want string + }{ + {"explicit wins", installSolutionRequest{AppName: "my-app", Slug: "excalidraw"}, "excalidraw/excalidraw", "my-app"}, + {"slug when no app name", installSolutionRequest{Slug: "it-tools"}, "CorentinTh/it-tools", "it-tools"}, + {"repo short name lowercased", installSolutionRequest{}, "freeCodeCamp/devdocs", "devdocs"}, + {"dots and underscores become hyphens", installSolutionRequest{}, "acme/My_Cool.App", "my-cool-app"}, + {"leading and trailing junk trimmed", installSolutionRequest{}, "acme/.hidden.", "hidden"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := appNameForInstall(tc.req, tc.repo); got != tc.want { + t.Fatalf("appNameForInstall = %q, want %q", got, tc.want) + } + }) + } +} + +// TestManagedDatabaseNameForIsAlwaysValid guards the derived names against the +// validators the database core applies: a name the customer never typed is +// still a name that has to pass validateKubeName and validatePgName, and a +// derivation that fails them turns one click into a 400 nobody can act on. +func TestManagedDatabaseNameForIsAlwaysValid(t *testing.T) { + for _, app := range []string{"n8n", "excalidraw", "9gag", "a", "my-long-app-name"} { + resource, database := managedDatabaseNameFor(app) + if err := validateKubeName(resource); err != nil { + t.Fatalf("resource name %q for app %q: %v", resource, app, err) + } + if err := validatePgName(database); err != nil { + t.Fatalf("database name %q for app %q: %v", database, app, err) + } + } + if resource, _ := managedDatabaseNameFor("n8n"); resource != "n8n-db" { + t.Fatalf("resource = %q, want n8n-db", resource) + } + if _, database := managedDatabaseNameFor("9gag"); database != "db-9gag" { + t.Fatalf("database = %q, want db-9gag", database) + } +} + +func testInstallPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("TEST_DATABASE_URL") + if dsn == "" { + t.Skip("TEST_DATABASE_URL not set; skipping solution-install DB integration test") + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("connect to TEST_DATABASE_URL: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func seedInstallProject(t *testing.T, pool *pgxpool.Pool, orgID, runtime string) (projectID, envID, userID uuid.UUID) { + t.Helper() + ctx := context.Background() + suffix := uuid.NewString()[:8] + + if err := pool.QueryRow(ctx, + `INSERT INTO users (username, email, password_hash, display_name, keycloak_sub) + VALUES ($1, $2, '', 'install test', $3) RETURNING id`, + "install-"+suffix, "install-"+suffix+"@example.test", uuid.NewString(), + ).Scan(&userID); err != nil { + t.Fatalf("seed user: %v", err) + } + t.Cleanup(func() { dropSeededUser(pool, userID) }) + + if err := pool.QueryRow(ctx, + `INSERT INTO projects (name, display_name, org_id) VALUES ($1, $1, $2) RETURNING id`, + "solution-install-test-"+suffix, orgID, + ).Scan(&projectID); err != nil { + t.Fatalf("seed project: %v", err) + } + t.Cleanup(func() { dropSeededProject(pool, projectID) }) + + if err := pool.QueryRow(ctx, + `INSERT INTO environments (project_id, name, namespace, type, runtime) VALUES ($1, 'prod', $2, 'prod', $3) RETURNING id`, + projectID, "ns-"+suffix, runtime, + ).Scan(&envID); err != nil { + t.Fatalf("seed environment: %v", err) + } + return projectID, envID, userID +} + +func newInstallCtx(projectID, envID uuid.UUID, body any, claims *auth.Claims) (*gin.Context, *httptest.ResponseRecorder) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + raw, _ := json.Marshal(body) + path := "/api/v1/projects/" + projectID.String() + "/environments/" + envID.String() + "/solutions/install" + c.Request = httptest.NewRequest(http.MethodPost, path, bytes.NewReader(raw)) + c.Request.Header.Set("Content-Type", "application/json") + c.Params = gin.Params{ + {Key: "projectId", Value: projectID.String()}, + {Key: "envId", Value: envID.String()}, + } + auth.SetClaims(c, claims) + return c, rec +} + +func newInstallHandler(pool *pgxpool.Pool) *Handler { + return &Handler{pool: pool, cfg: &config.Config{GitopsEncryptionKey: installTestKey}} +} + +// TestInstallSolution_CatalogEntryLinksAndBuilds is the newcomer's scenario: +// one call, and the project has a repository linked with the verified spec and +// a build already queued -- no second call the console could forget to make. +func TestInstallSolution_CatalogEntryLinksAndBuilds(t *testing.T) { + pool := testInstallPool(t) + projectID, envID, userID := seedInstallProject(t, pool, "acme", "k8s") + t.Cleanup(func() { dropSeededAudit(pool, "Solution", "excalidraw") }) + + h := newInstallHandler(pool) + claims := &auth.Claims{UserID: userID, Groups: []string{"/platform-admins"}} + c, rec := newInstallCtx(projectID, envID, installSolutionRequest{Slug: "excalidraw"}, claims) + h.InstallSolution(c) + + if rec.Code != http.StatusAccepted { + t.Fatalf("code=%d body=%s want 202", rec.Code, rec.Body.String()) + } + + ctx := context.Background() + var repoFullName, branch, rootDir string + var port int + if err := pool.QueryRow(ctx, + `SELECT repo_full_name, production_branch, root_dir, port FROM git_repos + WHERE project_id = $1 AND environment_id = $2 AND app_name = 'excalidraw'`, + projectID, envID, + ).Scan(&repoFullName, &branch, &rootDir, &port); err != nil { + t.Fatalf("linked repo not found: %v", err) + } + if repoFullName != "excalidraw/excalidraw" { + t.Fatalf("repo_full_name = %q", repoFullName) + } + if branch != "master" { + t.Fatalf("branch = %q, want the catalog's verified branch master", branch) + } + if port != 80 { + t.Fatalf("port = %d, want the catalog's verified port 80", port) + } + + var buildStatus, buildBranch string + if err := pool.QueryRow(ctx, + `SELECT status, branch FROM builds WHERE environment_id = $1 AND app_name = 'excalidraw'`, + envID, + ).Scan(&buildStatus, &buildBranch); err != nil { + t.Fatalf("build not queued: %v", err) + } + if buildStatus != "queued" { + t.Fatalf("build status = %q, want queued", buildStatus) + } + if buildBranch != "master" { + t.Fatalf("build branch = %q, want master", buildBranch) + } + + var dbCount int + if err := pool.QueryRow(ctx, + `SELECT COUNT(*) FROM operations WHERE environment_id = $1 AND action = 'CreateServiceDatabase'`, + envID, + ).Scan(&dbCount); err != nil { + t.Fatalf("count database operations: %v", err) + } + if dbCount != 0 { + t.Fatalf("ordered %d databases for a project that declares no needs", dbCount) + } +} + +// TestInstallSolution_WithDatabaseOnVMSeedsDSN is the whole point of item 4 on +// the VM track: the app comes up already able to reach its database, because +// the install seeded DATABASE_URL on it rather than telling the customer to go +// and wire one up. +func TestInstallSolution_WithDatabaseOnVMSeedsDSN(t *testing.T) { + pool := testInstallPool(t) + projectID, envID, userID := seedInstallProject(t, pool, "acme", "vm") + t.Cleanup(func() { dropSeededAudit(pool, "Solution", "devdocs") }) + t.Cleanup(func() { dropSeededAudit(pool, "ServiceDatabaseV2", "devdocs-db") }) + + h := newInstallHandler(pool) + claims := &auth.Claims{UserID: userID, Groups: []string{"/platform-admins"}} + withDB := true + c, rec := newInstallCtx(projectID, envID, installSolutionRequest{Slug: "devdocs", WithDatabase: &withDB}, claims) + h.InstallSolution(c) + + if rec.Code != http.StatusAccepted { + t.Fatalf("code=%d body=%s want 202", rec.Code, rec.Body.String()) + } + + ctx := context.Background() + var appRef, database string + if err := pool.QueryRow(ctx, + `SELECT payload->>'app_ref', payload->>'database' FROM operations + WHERE environment_id = $1 AND action = 'CreateServiceDatabase'`, + envID, + ).Scan(&appRef, &database); err != nil { + t.Fatalf("database operation not queued: %v", err) + } + if appRef != "devdocs" { + t.Fatalf("app_ref = %q, want the installed app so the chart binds them", appRef) + } + if database != "devdocs" { + t.Fatalf("database = %q", database) + } + + var encrypted []byte + if err := pool.QueryRow(ctx, + `SELECT value_encrypted FROM env_vars WHERE environment_id = $1 AND app_name = 'devdocs' AND key = 'DATABASE_URL'`, + envID, + ).Scan(&encrypted); err != nil { + t.Fatalf("DATABASE_URL not seeded on the app: %v", err) + } + dsn, err := crypto.DecryptToken(installTestKey, encrypted) + if err != nil { + t.Fatalf("decrypt DATABASE_URL: %v", err) + } + if want := "@devdocs-db:5432/devdocs"; !bytes.Contains(dsn, []byte(want)) { + t.Fatalf("DSN %q does not point at the database it just ordered (%q)", string(dsn), want) + } + + var pgPassword []byte + if err := pool.QueryRow(ctx, + `SELECT value_encrypted FROM env_vars WHERE environment_id = $1 AND app_name = 'devdocs-db' AND key = 'POSTGRES_PASSWORD'`, + envID, + ).Scan(&pgPassword); err != nil { + t.Fatalf("POSTGRES_PASSWORD not seeded on the database app: %v", err) + } +} + +func TestInstallSolution_UnknownSlugIsNotFound(t *testing.T) { + pool := testInstallPool(t) + projectID, envID, userID := seedInstallProject(t, pool, "acme", "k8s") + t.Cleanup(func() { dropSeededAudit(pool, "Solution", "") }) + + h := newInstallHandler(pool) + claims := &auth.Claims{UserID: userID, Groups: []string{"/platform-admins"}} + c, rec := newInstallCtx(projectID, envID, installSolutionRequest{Slug: "no-such-project"}, claims) + h.InstallSolution(c) + + if rec.Code != http.StatusNotFound { + t.Fatalf("code=%d body=%s want 404", rec.Code, rec.Body.String()) + } +} + +func TestInstallSolution_ReadOnlyRoleIsForbidden(t *testing.T) { + pool := testInstallPool(t) + projectID, envID, userID := seedInstallProject(t, pool, "acme", "k8s") + + h := newInstallHandler(pool) + claims := &auth.Claims{UserID: userID, Groups: []string{"/orgs/acme/projects/" + projectID.String() + "/ReadOnly"}} + c, rec := newInstallCtx(projectID, envID, installSolutionRequest{Slug: "excalidraw"}, claims) + h.InstallSolution(c) + + if rec.Code != http.StatusForbidden { + t.Fatalf("code=%d body=%s want 403", rec.Code, rec.Body.String()) + } +} + +// TestInstallSolution_PastedRepoInstalls covers the third audience: a link, +// no catalog entry, and the server-side defaults doing the rest. +func TestInstallSolution_PastedRepoInstalls(t *testing.T) { + pool := testInstallPool(t) + projectID, envID, userID := seedInstallProject(t, pool, "acme", "k8s") + t.Cleanup(func() { dropSeededAudit(pool, "Solution", "hello-world") }) + + h := newInstallHandler(pool) + claims := &auth.Claims{UserID: userID, Groups: []string{"/platform-admins"}} + c, rec := newInstallCtx(projectID, envID, installSolutionRequest{Repo: "https://github.com/octocat/hello-world"}, claims) + h.InstallSolution(c) + + if rec.Code != http.StatusAccepted { + t.Fatalf("code=%d body=%s want 202", rec.Code, rec.Body.String()) + } + + var repoFullName, branch string + var port int + if err := pool.QueryRow(context.Background(), + `SELECT repo_full_name, production_branch, port FROM git_repos + WHERE project_id = $1 AND environment_id = $2 AND app_name = 'hello-world'`, + projectID, envID, + ).Scan(&repoFullName, &branch, &port); err != nil { + t.Fatalf("linked repo not found: %v", err) + } + if repoFullName != "octocat/hello-world" { + t.Fatalf("repo_full_name = %q", repoFullName) + } + if branch != "main" || port != 8080 { + t.Fatalf("branch=%q port=%d, want the connect-repo defaults main/8080", branch, port) + } +} diff --git a/backend/internal/buildagent/client.go b/backend/internal/buildagent/client.go index 1789efc5..72197d72 100644 --- a/backend/internal/buildagent/client.go +++ b/backend/internal/buildagent/client.go @@ -188,3 +188,32 @@ func (c *Client) DetectFramework(ctx context.Context, installationID int64, repo } return &out, nil } + +// SearchHit mirrors the agent's github.SearchHit. +type SearchHit struct { + FullName string `json:"full_name"` + Description string `json:"description"` + Stars int `json:"stars"` + DefaultBranch string `json:"default_branch"` + AvatarURL string `json:"avatar_url"` + Archived bool `json:"archived"` + Fork bool `json:"fork"` + License string `json:"license"` + HTMLURL string `json:"html_url"` +} + +// SearchRepos proxies GET /github/search/repos on the agent. +func (c *Client) SearchRepos(ctx context.Context, query string, limit int) ([]SearchHit, error) { + q := url.Values{} + q.Set("q", query) + if limit > 0 { + q.Set("limit", fmt.Sprintf("%d", limit)) + } + var out struct { + Repositories []SearchHit `json:"repositories"` + } + if err := c.getJSON(ctx, "/github/search/repos?"+q.Encode(), &out); err != nil { + return nil, err + } + return out.Repositories, nil +} diff --git a/backend/internal/solutions/catalog.go b/backend/internal/solutions/catalog.go new file mode 100644 index 00000000..d0648631 --- /dev/null +++ b/backend/internal/solutions/catalog.go @@ -0,0 +1,404 @@ +// Package solutions holds the ready-made project catalog: real open-source +// projects a customer deploys into a cloud (Kubernetes) environment in one +// click, when they have nothing of their own to deploy yet. +// +// It replaces the starter templates (dada-nextjs-starter and friends, see +// api/demo_apps.go). A starter proves that a deploy happened and is then reaped; +// nobody opens it twice, because there is nothing inside it. A whiteboard, a +// toolbox, an offline documentation browser — those a customer might keep, and +// they arrive on the platform the same way the customer's own code will. +// +// The catalog is a frozen Go variable — same idiom, and same reason, as +// internal/boxcatalog and internal/profiles: edit this file and redeploy. +// +// # Built from source, on purpose +// +// Every entry is a REPOSITORY, not a published image. The install runs the +// ordinary customer path — link the public repo, detect the framework, build it +// in our pipeline, deploy the result — so a catalog card exercises exactly the +// machinery a customer's first repository will hit. Pulling `n8nio/n8n:2.34.2` +// would make prettier cards and prove nothing: it would be our platform running +// somebody else's build. +// +// That choice has a price, and it is the point. Real repositories are monorepos, +// build in two stages, read their port from an environment variable, and put +// their Dockerfile three directories down. Where the auto-detector misses, the +// card fails visibly and we fix the detector — which is worth more than a +// catalog that never touches it. tasks/autodeploy-benchmark-50-oss.md is that +// feedback loop written down. +// +// # Why every v1 entry is stateless +// +// An app created by the build pipeline takes its spec from git_repos +// (port/replicas/profile) and there is no volume in that spec, so a project +// that keeps state on disk would silently lose it on every redeploy. Until the +// build path can carry a volume, the catalog only lists projects whose state +// lives in the browser or nowhere at all. This is a real constraint, not an +// aesthetic: shipping a note-taking app that eats notes is worse than not +// shipping it. +package solutions + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// Category groups projects in the catalog UI. +type Category string + +const ( + CategoryDevTools Category = "dev-tools" + CategoryDocuments Category = "documents" + CategoryAI Category = "ai" +) + +// ParamKind is how the console renders one install parameter, and how the API +// validates it. +type ParamKind string + +const ( + ParamText ParamKind = "text" + ParamSecret ParamKind = "secret" + ParamSelect ParamKind = "select" +) + +// Param is one value the customer supplies at install time. EnvKey is the +// container environment variable it lands in. +type Param struct { + Key string + EnvKey string + Label string + Help string + Kind ParamKind + Required bool + Default string + Options []string + Placeholder string +} + +// Solution is one installable open-source project. +// +// Aliases are the other words a person types for this project: the English name +// when the product is known by a Russian one, the abbreviation, the thing it +// replaces. Without them the resolver only answers people who already know what +// an entry is called, which is the opposite of who the catalog exists for. +type Solution struct { + Slug string + Name string + Tagline string + About string + Bullets []string + Category Category + Homepage string + License string + Aliases []string + + // Repo is the public GitHub repository, "owner/name". It is cloned without + // an installation token: these are public projects, and requiring a customer + // to connect a GitHub account before they can see anything deploy is the + // exact wall the starter cards existed to get around. + Repo string + // Branch is the branch to build. Pinned by name rather than by commit + // because a card should ship what upstream currently calls released; a + // broken upstream build is a signal we want to see, not hide behind a pin. + Branch string + // RootDir is the directory to build from, "." for the repository root. + RootDir string + // Framework overrides auto-detection. Set to "dockerfile" for every entry + // that ships its own Dockerfile: the repository's own build is what upstream + // tests, and second-guessing it with a framework template is how a card + // starts failing on an upstream refactor nobody told us about. + Framework string + // Port is what the built image actually listens on, read from the + // repository's Dockerfile rather than assumed from the framework. + Port int + // Profile is the compute envelope (small | medium | large). + Profile string + + Params []Param + + // Needs are the managed resources this project cannot run without, by engine + // name ("postgres"). Declaring them here rather than telling the customer to + // go and create a database afterwards is what turns a two-step install into + // one button: the installer orders the database in the same call and binds it + // to the app, so the project comes up already connected. + Needs []string + + // Warning is the one thing a customer must read before deploying, rendered + // prominently rather than as fine print. + Warning string + // FirstRun is what to do once it is up. + FirstRun string + // BuildNote is what to expect from the BUILD, not the product: an honest + // heads-up that a real repository takes longer to build than a starter. + BuildNote string +} + +// V1 is the v1 catalog. Frozen at deploy time. +// +// Four entries, each verified to carry a Dockerfile at its repository root and +// to listen on the port recorded here. Small on purpose: a catalog of four +// projects that build is worth more than twenty assembled from READMEs, and the +// next entries earn their place by being deployed, not by being typed. +var V1 = []Solution{ + { + Slug: "excalidraw", + Name: "Excalidraw", + Tagline: "Доска для схем от руки — та самая, с «карандашным» стилем", + Category: CategoryDevTools, + Homepage: "https://excalidraw.com", + License: "MIT", + Aliases: []string{"экскалидроу", "доска", "whiteboard", "схемы", "диаграммы", "draw", "рисование", "miro"}, + About: "Виртуальная доска для схем, диаграмм и набросков: рисует так, будто чертили от " + + "руки на бумаге. Рисунки хранятся в браузере, экспорт в PNG и SVG, ссылка на " + + "доску открывается у коллеги без регистрации.", + Bullets: []string{ + "Схемы и наброски в «карандашном» стиле", + "Экспорт в PNG, SVG и файл проекта", + "Ничего не хранится на сервере — всё в браузере", + }, + Repo: "excalidraw/excalidraw", + Branch: "master", + RootDir: ".", + Framework: "dockerfile", + Port: 80, + Profile: "small", + FirstRun: "Открывайте и рисуйте — регистрация не нужна, доски сохраняются в браузере.", + BuildNote: "Сборка фронтенда занимает несколько минут: это настоящий репозиторий, а не заготовка.", + }, + { + Slug: "it-tools", + Name: "IT-Tools", + Tagline: "Больше сотни инструментов разработчика в одном месте", + Category: CategoryDevTools, + Homepage: "https://it-tools.tech", + License: "GPL-3.0", + Aliases: []string{"ittools", "tools", "инструменты", "утилиты", "jwt", "hash", "хеш", "uuid", "json", "конвертер"}, + About: "Инструменты, за которыми обычно идут на случайные сайты: разбор JWT, хеши и UUID, " + + "форматирование JSON, SQL и XML, конвертеры дат, кодировок и цветов, генератор " + + "паролей. Всё считается в браузере и никуда не отправляется.", + Bullets: []string{ + "Больше 100 инструментов: хеши, JWT, форматтеры, конвертеры", + "Всё вычисляется в браузере", + "Никаких данных на сервере", + }, + Repo: "CorentinTh/it-tools", + Branch: "main", + RootDir: ".", + Framework: "dockerfile", + Port: 80, + Profile: "small", + FirstRun: "Открывайте и пользуйтесь — учётная запись не нужна.", + }, + { + Slug: "gitingest", + Name: "Gitingest", + Tagline: "Превращает любой репозиторий в один текст для языковой модели", + Category: CategoryAI, + Homepage: "https://gitingest.com", + License: "MIT", + Aliases: []string{"ingest", "repo2text", "llm", "нейросеть", "контекст", "промпт", "ai"}, + About: "Принимает ссылку на репозиторий и собирает его в один структурированный текст, " + + "который можно целиком отдать модели: дерево файлов, содержимое, оценка размера " + + "в токенах. Полезно ровно тогда, когда нужно объяснить модели незнакомый проект.", + Bullets: []string{ + "Репозиторий → один текст с деревом файлов", + "Оценка размера в токенах", + "Фильтры по путям и расширениям", + }, + Repo: "cyclotruc/gitingest", + Branch: "main", + RootDir: ".", + Framework: "dockerfile", + Port: 8000, + Profile: "small", + FirstRun: "Вставьте ссылку на публичный репозиторий и получите текст для модели.", + Warning: "Приложение ходит в интернет за содержимым репозиториев, которые вы ему называете. " + + "Приватные репозитории оно без ваших ключей не увидит — и не должно.", + }, + { + Slug: "devdocs", + Name: "DevDocs", + Tagline: "Документация 500+ библиотек в одном интерфейсе, с поиском", + Category: CategoryDocuments, + Homepage: "https://devdocs.io", + License: "MPL-2.0", + Aliases: []string{"docs", "документация", "справочник", "api docs", "manual", "мануал"}, + About: "Собирает документацию сотен языков и библиотек в один быстрый интерфейс с общим " + + "поиском и горячими клавишами. Свой экземпляр удобен тем, что набор документаций " + + "и их версии выбираете вы.", + Bullets: []string{ + "Документация 500+ языков и библиотек", + "Мгновенный поиск по всему набору", + "Свой экземпляр — свой выбор версий", + }, + Repo: "freeCodeCamp/devdocs", + Branch: "main", + RootDir: ".", + Framework: "dockerfile", + Port: 9292, + Profile: "medium", + FirstRun: "Откройте приложение и включите нужные документации в настройках.", + BuildNote: "Образ большой: сборка идёт дольше остальных карточек каталога.", + }, +} + +// Lookup returns the Solution by slug. Second return is false if slug is unknown. +func Lookup(slug string) (Solution, bool) { + for _, s := range V1 { + if s.Slug == slug { + return s, true + } + } + return Solution{}, false +} + +// Slugs returns the catalog slugs in display order. +func Slugs() []string { + out := make([]string, len(V1)) + for i, s := range V1 { + out[i] = s.Slug + } + return out +} + +// IsCatalogRepo reports whether repoFullName is one of the catalog's own +// repositories. Case-insensitive, because GitHub owner/repo names are. +// +// This is what makes a catalog deploy a DEMO for the reaper (api/demo_apps.go): +// a project the platform offered, that nobody claimed, is exactly the app that +// used to idle for eighteen days. A repository the customer pasted themselves is +// never a demo — they chose it, and deleting their work on a timer would be a +// different product than the one we are building. Matching on the full name +// means a fork ("acme/it-tools") is the customer's, not ours. +func IsCatalogRepo(repoFullName string) bool { + for _, s := range V1 { + if strings.EqualFold(s.Repo, repoFullName) { + return true + } + } + return false +} + +// instanceNameRe is the accepted app name: lowercase, alphanumeric and dashes, +// starting with a letter — a legal Kubernetes resource name. +var instanceNameRe = regexp.MustCompile(`^[a-z]([-a-z0-9]*[a-z0-9])?$`) + +// ValidateInstanceName checks the name the deployed app will carry. +func ValidateInstanceName(name string) error { + if name == "" { + return fmt.Errorf("name is required") + } + if len(name) > 40 { + return fmt.Errorf("name must be at most 40 characters") + } + if !instanceNameRe.MatchString(name) { + return fmt.Errorf("name must be lowercase alphanumeric or '-', and start with a letter") + } + return nil +} + +// repoFullNameRe is a GitHub "owner/name" pair. GitHub allows alphanumerics, +// dash, underscore and dot in both halves. +var repoFullNameRe = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9_])?/[A-Za-z0-9._-]+$`) + +// ParseRepoURL turns what a person actually pastes into an "owner/name" pair. +// +// Accepts the browser URL, the clone URL, the SSH remote, and a bare +// "owner/name" — because all four are what ends up on a clipboard, and a form +// that takes only one of them is a form that rejects the customer's first +// attempt. Anything else is refused rather than guessed at: a wrong guess here +// deploys somebody else's repository under the customer's name. +func ParseRepoURL(raw string) (string, error) { + s := strings.TrimSpace(raw) + if s == "" { + return "", fmt.Errorf("paste a link to a public GitHub repository") + } + s = strings.TrimSuffix(s, "/") + s = strings.TrimPrefix(s, "git+") + + switch { + case strings.HasPrefix(s, "git@github.com:"): + s = strings.TrimPrefix(s, "git@github.com:") + case strings.HasPrefix(s, "ssh://git@github.com/"): + s = strings.TrimPrefix(s, "ssh://git@github.com/") + case strings.HasPrefix(s, "https://"), strings.HasPrefix(s, "http://"): + rest := s[strings.Index(s, "//")+2:] + host, path, found := strings.Cut(rest, "/") + if !found { + return "", fmt.Errorf("that link has no repository in it") + } + if !strings.EqualFold(host, "github.com") && !strings.EqualFold(host, "www.github.com") { + return "", fmt.Errorf("only public GitHub repositories are supported here") + } + s = path + } + s = strings.TrimSuffix(s, ".git") + + // A deep link (…/tree/main/apps/web) carries a branch and a subdirectory the + // caller has to choose deliberately, so keep only the repository and let + // them set branch and root directory in the form. + parts := strings.Split(s, "/") + if len(parts) > 2 { + parts = parts[:2] + } + full := strings.Join(parts, "/") + if !repoFullNameRe.MatchString(full) { + return "", fmt.Errorf("expected a link like https://github.com/owner/repository") + } + return full, nil +} + +// ResolveParams validates supplied values against the entry's parameters and +// returns the environment variables they produce. Unknown keys are rejected +// rather than ignored: a typo in a parameter name is a misconfigured deploy +// that would otherwise surface much later as a product behaving oddly. +func (s Solution) ResolveParams(in map[string]string) (map[string]string, error) { + known := make(map[string]Param, len(s.Params)) + for _, p := range s.Params { + known[p.Key] = p + } + unknown := make([]string, 0) + for k := range in { + if _, ok := known[k]; !ok { + unknown = append(unknown, k) + } + } + if len(unknown) > 0 { + sort.Strings(unknown) + return nil, fmt.Errorf("unknown parameter(s): %s", strings.Join(unknown, ", ")) + } + + env := make(map[string]string, len(s.Params)) + for _, p := range s.Params { + v := strings.TrimSpace(in[p.Key]) + if v == "" { + v = p.Default + } + if v == "" { + if p.Required { + return nil, fmt.Errorf("%s is required", p.Key) + } + continue + } + if p.Kind == ParamSelect && !contains(p.Options, v) { + return nil, fmt.Errorf("%s must be one of: %s", p.Key, strings.Join(p.Options, ", ")) + } + if strings.ContainsAny(v, "\n\r") { + return nil, fmt.Errorf("%s must not contain line breaks", p.Key) + } + env[p.EnvKey] = v + } + return env, nil +} + +func contains(list []string, v string) bool { + for _, item := range list { + if item == v { + return true + } + } + return false +} diff --git a/backend/internal/solutions/catalog_test.go b/backend/internal/solutions/catalog_test.go new file mode 100644 index 00000000..aa6485d2 --- /dev/null +++ b/backend/internal/solutions/catalog_test.go @@ -0,0 +1,181 @@ +package solutions + +import ( + "strings" + "testing" +) + +// The catalog is data a customer reads as a promise ("one click and this +// works"), so the invariants that keep the promise true are asserted here +// rather than left to review. +func TestV1CatalogInvariants(t *testing.T) { + validProfile := map[string]bool{"small": true, "medium": true, "large": true} + seenSlug := map[string]bool{} + seenRepo := map[string]bool{} + + for _, s := range V1 { + if s.Slug == "" || s.Name == "" || s.Tagline == "" || s.About == "" { + t.Fatalf("solution %q: slug, name, tagline and about are all required", s.Slug) + } + if seenSlug[s.Slug] { + t.Fatalf("duplicate slug %q", s.Slug) + } + seenSlug[s.Slug] = true + // The slug is the default app name, so it has to be a legal one. + if err := ValidateInstanceName(s.Slug); err != nil { + t.Fatalf("solution %q: slug is not a usable app name: %v", s.Slug, err) + } + if s.Homepage == "" || s.License == "" { + t.Fatalf("solution %q: a third-party project must name its homepage and license", s.Slug) + } + if s.FirstRun == "" { + t.Fatalf("solution %q: say what to do once it is up", s.Slug) + } + + // The repository is the whole point of an entry: it is what gets built. + if _, err := ParseRepoURL(s.Repo); err != nil { + t.Fatalf("solution %q: repo %q is not a usable owner/name: %v", s.Slug, s.Repo, err) + } + if seenRepo[strings.ToLower(s.Repo)] { + t.Fatalf("two entries build %q", s.Repo) + } + seenRepo[strings.ToLower(s.Repo)] = true + if s.Branch == "" { + t.Fatalf("solution %q: branch is required; the default is not the same on every repo", s.Slug) + } + if s.RootDir == "" { + t.Fatalf("solution %q: root dir is required (\".\" for the repository root)", s.Slug) + } + if s.Port < 1 || s.Port > 65535 { + t.Fatalf("solution %q: port %d is not a port; a wrong one deploys green and answers 502", s.Slug, s.Port) + } + if !validProfile[s.Profile] { + t.Fatalf("solution %q: profile %q is not one of small/medium/large", s.Slug, s.Profile) + } + + for _, p := range s.Params { + if p.Key == "" || p.EnvKey == "" || p.Label == "" { + t.Fatalf("solution %q param %q: key, env key and label are required", s.Slug, p.Key) + } + if p.Kind == ParamSecret && p.Default != "" { + t.Fatalf("solution %q param %q: a secret must never ship a default", s.Slug, p.Key) + } + if p.Kind == ParamSelect && len(p.Options) == 0 { + t.Fatalf("solution %q param %q: select needs options", s.Slug, p.Key) + } + } + } +} + +// v1 builds from source, and an app created by the build pipeline has no +// volume: a project that keeps state on disk would lose it on every redeploy. +// Until the build path can carry one, every entry must be stateless — this test +// is the reminder, so adding a stateful project is a deliberate act with a +// matching change to the build spec rather than an oversight. +func TestEveryEntryShipsItsOwnDockerfileBuild(t *testing.T) { + for _, s := range V1 { + if s.Framework != "dockerfile" { + t.Fatalf("solution %q builds via %q; every v1 entry was verified against its own root Dockerfile", s.Slug, s.Framework) + } + } +} + +func TestIsCatalogRepo(t *testing.T) { + if !IsCatalogRepo("CorentinTh/it-tools") { + t.Fatal("catalog repo not recognised; its deploys would never be reaped") + } + if !IsCatalogRepo("corentinth/IT-TOOLS") { + t.Fatal("match must be case-insensitive, like GitHub names") + } + // A fork is the customer's own work, not a demo we offered. + if IsCatalogRepo("acme/it-tools") { + t.Fatal("a fork must not be treated as a catalog demo") + } + if IsCatalogRepo("") { + t.Fatal("empty repo name matched") + } +} + +func TestParseRepoURL(t *testing.T) { + want := "excalidraw/excalidraw" + for _, in := range []string{ + "excalidraw/excalidraw", + "https://github.com/excalidraw/excalidraw", + "https://github.com/excalidraw/excalidraw/", + "https://github.com/excalidraw/excalidraw.git", + "http://www.github.com/excalidraw/excalidraw", + "git@github.com:excalidraw/excalidraw.git", + "ssh://git@github.com/excalidraw/excalidraw", + " https://github.com/excalidraw/excalidraw ", + // A deep link keeps only the repository: branch and subdirectory are + // choices the form asks for explicitly. + "https://github.com/excalidraw/excalidraw/tree/master/packages/excalidraw", + } { + got, err := ParseRepoURL(in) + if err != nil { + t.Fatalf("%q rejected: %v", in, err) + } + if got != want { + t.Fatalf("%q -> %q, want %q", in, got, want) + } + } + + for _, bad := range []string{ + "", " ", "https://gitlab.com/owner/repo", "https://github.com", + "https://github.com/owner", "not a url", "/leading-slash", + } { + if got, err := ParseRepoURL(bad); err == nil { + t.Fatalf("%q accepted as %q", bad, got) + } + } +} + +func TestValidateInstanceName(t *testing.T) { + for _, ok := range []string{"excalidraw", "it-tools", "a"} { + if err := ValidateInstanceName(ok); err != nil { + t.Fatalf("%q rejected: %v", ok, err) + } + } + for _, bad := range []string{"", "Excalidraw", "2fast", "my_app", "app-", strings.Repeat("a", 41)} { + if err := ValidateInstanceName(bad); err == nil { + t.Fatalf("%q accepted", bad) + } + } +} + +func TestResolveParams(t *testing.T) { + s := Solution{Params: []Param{ + {Key: "url", EnvKey: "URL", Kind: ParamText, Required: true, Default: "https://d"}, + {Key: "key", EnvKey: "KEY", Kind: ParamSecret, Required: true}, + {Key: "mode", EnvKey: "MODE", Kind: ParamSelect, Options: []string{"a", "b"}}, + }} + + env, err := s.ResolveParams(map[string]string{"key": " k "}) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if env["URL"] != "https://d" { + t.Fatalf("default not applied: %q", env["URL"]) + } + if env["KEY"] != "k" { + t.Fatalf("value not trimmed: %q", env["KEY"]) + } + if _, set := env["MODE"]; set { + t.Fatal("optional param with no value must not be written") + } + + if _, err := s.ResolveParams(map[string]string{}); err == nil { + t.Fatal("missing required param accepted") + } + if _, err := s.ResolveParams(map[string]string{"key": "k", "nope": "x"}); err == nil { + t.Fatal("unknown param accepted") + } + if _, err := s.ResolveParams(map[string]string{"key": "k", "mode": "c"}); err == nil { + t.Fatal("value outside the option set accepted") + } + // A newline would split one .env line in two and let the tail of a value + // become an attacker-chosen variable. + if _, err := s.ResolveParams(map[string]string{"key": "k\nEVIL=1"}); err == nil { + t.Fatal("newline in a value accepted") + } +} diff --git a/backend/internal/solutions/managed.go b/backend/internal/solutions/managed.go new file mode 100644 index 00000000..5aa9d654 --- /dev/null +++ b/backend/internal/solutions/managed.go @@ -0,0 +1,33 @@ +package solutions + +// ManagedResource is a platform resource the resolver can offer alongside +// applications: something the customer asks for by name and the platform runs +// for them, rather than an image or a repository we deploy. +// +// Engine is the value the databases API expects. Aliases are the other words +// people type for the same thing — the point of the whole list is that "post" +// finds Postgres before the third keystroke, which is the OSS-user scenario in +// tasks/ready-made-projects-unified-design.md. +type ManagedResource struct { + Slug string + Name string + Tagline string + Engine string + Aliases []string +} + +// ManagedResources is what the resolver can offer today. +// +// Postgres only, and deliberately so: the k8s track creates managed databases +// through ServiceDatabase, which is Postgres, and offering Redis here would +// mean a card that resolves on both runtimes but installs on one. A short +// honest list beats a long one that fails in the second half. +var ManagedResources = []ManagedResource{ + { + Slug: "postgres", + Name: "PostgreSQL", + Tagline: "Управляемая база данных: бэкапы, восстановление и строка подключения из консоли", + Engine: "postgres", + Aliases: []string{"postgresql", "postgre", "pg", "постгрес", "база", "база данных", "бд", "database", "db", "sql"}, + }, +} diff --git a/backend/internal/solutions/resolve.go b/backend/internal/solutions/resolve.go new file mode 100644 index 00000000..ec45f286 --- /dev/null +++ b/backend/internal/solutions/resolve.go @@ -0,0 +1,250 @@ +package solutions + +import ( + "sort" + "strings" +) + +// CandidateKind is what one row of the resolver's answer list is. +// +// CandidateSolution is a curated catalog entry: verified build spec, one click. +// CandidateRepo is a repository the customer named themselves, by link or by +// owner/name — never a demo, and so never reaped. CandidateManaged is a +// platform resource rather than an application (today a managed database); it +// appears in the same list with the same shape because "поднять postgres" is +// one action to the person asking, and the difference between "resource" and +// "application" is our vocabulary, not theirs. +type CandidateKind string + +// CandidateSolution marks a curated catalog entry. +const CandidateSolution CandidateKind = "solution" + +// CandidateRepo marks a repository the customer named themselves. +const CandidateRepo CandidateKind = "repo" + +// CandidateManaged marks a managed platform resource. +const CandidateManaged CandidateKind = "managed" + +// Candidate is one row of the resolver's answer. +// +// Icon is an absolute image URL, or "" when the console should draw its own +// glyph (managed resources); for repositories it is the owner's avatar, which +// is what every GitHub UI shows and costs us no asset pipeline. Engine is set +// only for CandidateManaged and names the resource to create. Score orders the +// list, higher first, ties broken by name. +type Candidate struct { + Kind CandidateKind + Slug string + Name string + Tagline string + Icon string + + Repo string + Branch string + RootDir string + Framework string + Port int + Profile string + + Engine string + + Score int +} + +// Result is what one Resolve call produced locally, plus whether the caller +// should still go and ask GitHub. +// +// SearchQuery is non-empty when the local answer is thin enough that a GitHub +// search is worth its rate-limit budget. Empty means "already answered": a +// pasted link needs no search, and neither does a query too short to mean +// anything. +type Result struct { + Candidates []Candidate + SearchQuery string +} + +// scorePastedLink outranks every other tier because pasting a link is an +// unambiguous statement of intent, and a ranking that answers it with a fuzzy +// catalog match is arguing with the customer about what they just typed. +const scorePastedLink = 1000 + +// scoreExact is a whole-string hit on a slug, name or alias. +const scoreExact = 500 + +// scorePrefix is what the customer is still in the middle of typing. +const scorePrefix = 300 + +// scoreAlias is a prefix hit on an alias rather than the entry's own name. +const scoreAlias = 200 + +// scoreSubstring is the weakest tier: a hit anywhere inside a name or tagline. +const scoreSubstring = 100 + +// minSearchQuery is the shortest string worth spending a GitHub search on. One +// or two characters match everything and inform nobody, and every such call +// comes out of a per-minute budget shared by the whole cluster. +const minSearchQuery = 3 + +// ownerAvatar is the owner's GitHub picture, which GitHub serves for any +// account without an API call or a token. +func ownerAvatar(repoFullName string) string { + owner, _, ok := strings.Cut(repoFullName, "/") + if !ok || owner == "" { + return "" + } + return "https://github.com/" + owner + ".png?size=160" +} + +// candidateFor renders a catalog entry as an answer row. +func candidateFor(s Solution) Candidate { + return Candidate{ + Kind: CandidateSolution, + Slug: s.Slug, + Name: s.Name, + Tagline: s.Tagline, + Icon: ownerAvatar(s.Repo), + Repo: s.Repo, + Branch: s.Branch, + RootDir: s.RootDir, + Framework: s.Framework, + Port: s.Port, + Profile: s.Profile, + } +} + +// Resolve turns one typed string into a ranked answer list. +// +// The console asks "what do you want to run?" exactly once, and that single +// string has to serve three people who share nothing but the keyboard: the +// beginner who types "n8n" and expects a picture, the person who types "post" +// and means a Postgres, and the professional who pastes a repository URL. +// Modes, tabs and a "choose a type first" step are how that one question ends +// up being asked three times. +// +// Ranking: a recognised link wins outright; then curated catalog entries and +// managed resources by how sure the match is. GitHub search never mixes in +// here — it is appended by the caller, below everything local, because a +// curated entry carries a build spec we verified and a search hit carries +// nothing but a name. +// +// A link to a catalog repository resolves to the CATALOG entry, so the customer +// gets the verified branch, root directory and port instead of our best guess. +// +// Pure and offline: it never calls GitHub. The network half belongs to the +// caller and only runs when Result.SearchQuery is set, which is what stops an +// interactive input from spending the search budget on keystrokes that already +// had a good answer. +func Resolve(query string) Result { + q := strings.TrimSpace(query) + if q == "" { + return Result{Candidates: []Candidate{}} + } + lower := strings.ToLower(q) + + out := make([]Candidate, 0, 8) + claimed := make(map[string]bool, 8) + + if full, err := ParseRepoURL(q); err == nil { + if s, ok := lookupByRepo(full); ok { + c := candidateFor(s) + c.Score = scorePastedLink + out = append(out, c) + claimed[s.Slug] = true + } else { + owner, name, _ := strings.Cut(full, "/") + out = append(out, Candidate{ + Kind: CandidateRepo, + Slug: full, + Name: name, + Tagline: owner, + Icon: ownerAvatar(full), + Repo: full, + RootDir: ".", + Score: scorePastedLink, + }) + } + } + + for _, s := range V1 { + if claimed[s.Slug] { + continue + } + if score := matchScore(lower, s.Slug, s.Name, s.Aliases, s.Tagline); score > 0 { + c := candidateFor(s) + c.Score = score + out = append(out, c) + } + } + + for _, m := range ManagedResources { + if score := matchScore(lower, m.Slug, m.Name, m.Aliases, m.Tagline); score > 0 { + out = append(out, Candidate{ + Kind: CandidateManaged, + Slug: m.Slug, + Name: m.Name, + Tagline: m.Tagline, + Engine: m.Engine, + Score: score, + }) + } + } + + sort.SliceStable(out, func(i, j int) bool { + if out[i].Score != out[j].Score { + return out[i].Score > out[j].Score + } + return out[i].Name < out[j].Name + }) + + res := Result{Candidates: out} + pasted := len(out) > 0 && out[0].Score == scorePastedLink + if !pasted && len([]rune(lower)) >= minSearchQuery { + res.SearchQuery = q + } + return res +} + +// lookupByRepo finds a catalog entry by its repository full name. +func lookupByRepo(repoFullName string) (Solution, bool) { + for _, s := range V1 { + if strings.EqualFold(s.Repo, repoFullName) { + return s, true + } + } + return Solution{}, false +} + +// matchScore rates one entry against an already-lowercased query. +// +// Prefix beats substring because typing runs left to right: someone three +// characters into "postgres" means Postgres, not the project that happens to +// carry "pos" in the middle of a word. The tagline only ever earns the bottom +// tier — it is prose, and prose matches everything eventually. +func matchScore(lowerQuery, slug, name string, aliases []string, tagline string) int { + slugL := strings.ToLower(slug) + nameL := strings.ToLower(name) + + if lowerQuery == slugL || lowerQuery == nameL { + return scoreExact + } + for _, a := range aliases { + if lowerQuery == strings.ToLower(a) { + return scoreExact + } + } + if strings.HasPrefix(slugL, lowerQuery) || strings.HasPrefix(nameL, lowerQuery) { + return scorePrefix + } + for _, a := range aliases { + if strings.HasPrefix(strings.ToLower(a), lowerQuery) { + return scoreAlias + } + } + if strings.Contains(slugL, lowerQuery) || strings.Contains(nameL, lowerQuery) { + return scoreSubstring + } + if len([]rune(lowerQuery)) >= minSearchQuery && strings.Contains(strings.ToLower(tagline), lowerQuery) { + return scoreSubstring + } + return 0 +} diff --git a/backend/internal/solutions/resolve_test.go b/backend/internal/solutions/resolve_test.go new file mode 100644 index 00000000..a1b614cf --- /dev/null +++ b/backend/internal/solutions/resolve_test.go @@ -0,0 +1,162 @@ +package solutions + +import ( + "strings" + "testing" +) + +// TestResolvePastedLinkOutranksEverything holds the resolver's one hard rule: +// a recognised link is a statement of intent, and no fuzzy catalog match may +// climb over it. When this breaks, the professional pasting a repository URL +// gets someone else's project offered first. +func TestResolvePastedLinkOutranksEverything(t *testing.T) { + res := Resolve("https://github.com/acme/docs-portal") + if len(res.Candidates) == 0 { + t.Fatal("no candidates for a pasted link") + } + top := res.Candidates[0] + if top.Kind != CandidateRepo || top.Repo != "acme/docs-portal" { + t.Fatalf("top candidate = %s %q, want repo acme/docs-portal", top.Kind, top.Repo) + } + if res.SearchQuery != "" { + t.Fatalf("SearchQuery = %q, want empty: a pasted link is already the answer", res.SearchQuery) + } +} + +// TestResolveCatalogLinkKeepsVerifiedSpec covers the case where the pasted link +// happens to be a catalog repository. The customer must get the entry we +// verified — branch, root dir, port — not a bare repository row that makes the +// pipeline guess all three again. +func TestResolveCatalogLinkKeepsVerifiedSpec(t *testing.T) { + res := Resolve("https://github.com/excalidraw/excalidraw/tree/master/packages") + if len(res.Candidates) == 0 { + t.Fatal("no candidates") + } + top := res.Candidates[0] + if top.Kind != CandidateSolution || top.Slug != "excalidraw" { + t.Fatalf("top candidate = %s %q, want the excalidraw catalog entry", top.Kind, top.Slug) + } + if top.Branch != "master" || top.Port != 80 { + t.Fatalf("verified spec lost: branch=%q port=%d", top.Branch, top.Port) + } +} + +// TestResolvePostFindsPostgresFirst is the OSS-user scenario from the design +// note, written as a test because it is the one behaviour the owner asked for +// by name: typing "post" offers Postgres before the third keystroke. +func TestResolvePostFindsPostgresFirst(t *testing.T) { + for _, q := range []string{"post", "postg", "pos", "бд", "база"} { + res := Resolve(q) + if len(res.Candidates) == 0 { + t.Fatalf("%q: no candidates", q) + } + top := res.Candidates[0] + if top.Kind != CandidateManaged || top.Engine != "postgres" { + t.Fatalf("%q: top candidate = %s/%s, want managed postgres", q, top.Kind, top.Engine) + } + } +} + +// TestResolveRanksExactAboveSubstring keeps the tiers from collapsing into one +// another: an entry the customer named outright cannot sit below one that +// merely mentions the word somewhere. +func TestResolveRanksExactAboveSubstring(t *testing.T) { + res := Resolve("devdocs") + if len(res.Candidates) == 0 { + t.Fatal("no candidates") + } + if got := res.Candidates[0].Slug; got != "devdocs" { + t.Fatalf("top candidate = %q, want devdocs", got) + } + if res.Candidates[0].Score != scoreExact { + t.Fatalf("score = %d, want scoreExact", res.Candidates[0].Score) + } +} + +// TestResolveAliasFindsRussianQuery is why aliases exist: the catalog is in +// Russian for people who have not met these projects before, and "доска" is +// what such a person types when they want Excalidraw. +func TestResolveAliasFindsRussianQuery(t *testing.T) { + res := Resolve("доска") + found := false + for _, c := range res.Candidates { + if c.Slug == "excalidraw" { + found = true + } + } + if !found { + t.Fatalf("доска did not find excalidraw; got %d candidates", len(res.Candidates)) + } +} + +// TestResolveShortQuerySkipsSearch guards the shared rate-limit budget: one and +// two character queries match everything, inform nobody, and would burn the +// cluster's per-minute search allowance on the way to the third keystroke. +func TestResolveShortQuerySkipsSearch(t *testing.T) { + for _, q := range []string{"n", "n8"} { + if got := Resolve(q).SearchQuery; got != "" { + t.Fatalf("%q: SearchQuery = %q, want empty", q, got) + } + } + if got := Resolve("n8n").SearchQuery; got != "n8n" { + t.Fatalf("SearchQuery = %q, want n8n", got) + } +} + +// TestResolveUnknownWordAsksForSearch is the beginner scenario: nothing local +// matches "n8n", so the caller is told to go and ask GitHub rather than +// showing an empty list. +func TestResolveUnknownWordAsksForSearch(t *testing.T) { + res := Resolve("n8n") + for _, c := range res.Candidates { + if c.Kind == CandidateSolution { + t.Fatalf("unexpected catalog match for n8n: %q", c.Slug) + } + } + if res.SearchQuery == "" { + t.Fatal("SearchQuery empty: an unknown word must fall through to search") + } +} + +// TestResolveEmptyQuery keeps the empty field from being an error the console +// has to special-case. +func TestResolveEmptyQuery(t *testing.T) { + res := Resolve(" ") + if len(res.Candidates) != 0 || res.SearchQuery != "" { + t.Fatalf("empty query produced %d candidates / search %q", len(res.Candidates), res.SearchQuery) + } +} + +// TestCatalogAliasesAreLowercase keeps matching honest: matchScore lowercases +// the query but compares aliases as written, so an uppercase alias would be +// dead weight nobody could ever hit. +func TestCatalogAliasesAreLowercase(t *testing.T) { + check := func(owner string, aliases []string) { + for _, a := range aliases { + if a != strings.ToLower(a) { + t.Errorf("%s: alias %q is not lowercase", owner, a) + } + if strings.TrimSpace(a) != a || a == "" { + t.Errorf("%s: alias %q has stray whitespace", owner, a) + } + } + } + for _, s := range V1 { + check(s.Slug, s.Aliases) + } + for _, m := range ManagedResources { + check(m.Slug, m.Aliases) + } +} + +// TestOwnerAvatarUsesOwner pins the icon source. Repository avatars do not +// exist on GitHub; the owner's picture is what every GitHub UI shows, and +// getting the owner half wrong silently renders a broken image everywhere. +func TestOwnerAvatarUsesOwner(t *testing.T) { + if got := ownerAvatar("freeCodeCamp/devdocs"); got != "https://github.com/freeCodeCamp.png?size=160" { + t.Fatalf("ownerAvatar = %q", got) + } + if got := ownerAvatar("nonsense"); got != "" { + t.Fatalf("ownerAvatar(nonsense) = %q, want empty", got) + } +} diff --git a/backend/internal/sourcedetect/detect.go b/backend/internal/sourcedetect/detect.go index 6e8cf604..7b73d9da 100644 --- a/backend/internal/sourcedetect/detect.go +++ b/backend/internal/sourcedetect/detect.go @@ -17,6 +17,7 @@ import ( "encoding/json" "fmt" "io" + "path" "regexp" "strconv" "strings" @@ -33,8 +34,8 @@ const ( // Result is what Detect resolves from an archive's manifest files. // // Framework is one of "docker", "nextjs", "vite", "react", "node", "fastapi", -// "flask", "django", "streamlit", "python", or "" when nothing matched. The -// names are the vocabulary dadaBuildPipeline.renderDockerfile switches on — +// "flask", "django", "streamlit", "python", "maven", "gradle", "go", or "" when +// nothing matched. The names are the vocabulary dadaBuildPipeline.renderDockerfile switches on — // keep them in sync with build-agent's GitHub-side detection // (build-agent/internal/server/server.go), or the pipeline finds no template // and the build fails with no_dockerfile. Port is 0 when unresolved, which @@ -47,16 +48,63 @@ type Result struct { // maxEntries caps how many table-of-contents entries Detect walks, so a // pathological archive (millions of tiny entries) can't stall the request. -const maxEntries = 500 +// +// The cap is generous because walking a header costs nothing: only entries +// whose base name is a known manifest have their bytes buffered (see +// isCandidate). A tight cap used to decide the answer for large repos — +// netdata carries 13k files — which made detection depend on where in the +// tree a manifest happened to sit. +const maxEntries = 40000 + +// candidateNames are the only files Detect ever reads. Everything else is +// walked as a header and dropped, which is what keeps a 40k-entry cap cheap. +var candidateNames = map[string]bool{ + "Dockerfile": true, + "Procfile": true, + "package.json": true, + "requirements.txt": true, + "pyproject.toml": true, + "docker-compose.yml": true, + "docker-compose.yaml": true, + "compose.yml": true, + "compose.yaml": true, + "pom.xml": true, + "build.gradle": true, + "build.gradle.kts": true, + "go.mod": true, + "railway.json": true, + "railway.toml": true, + "nixpacks.toml": true, +} + +// isCandidate reports whether an archive member is worth buffering. +// +// Variants like Dockerfile.debian count, because a root Dockerfile symlink +// usually points at one of them (vaultwarden ships Dockerfile -> +// docker/Dockerfile.debian) and a target that was never buffered resolves to +// nothing. +func isCandidate(name string) bool { + base := name + if i := strings.LastIndexByte(base, '/'); i >= 0 { + base = base[i+1:] + } + return candidateNames[base] || strings.HasPrefix(base, "Dockerfile.") +} // maxManifestBytes caps how much of any single candidate manifest file is // read into memory. const maxManifestBytes = 1024 * 1024 // entry is a format-agnostic view of one archive member, read on demand. +// +// link is non-empty for a symbolic link, and holds the raw link target as +// stored in the archive. A repo whose root Dockerfile is a symlink into a +// subdirectory (vaultwarden, netdata) is common enough that dropping such +// entries reads to the caller as "this repo ships no Dockerfile". type entry struct { name string size int64 + link string open func() (io.ReadCloser, error) } @@ -78,23 +126,37 @@ func Detect(data []byte) (Result, error) { result := Result{Format: format} root := detectRoot(entries) - if e, ok := findManifest(entries, root, "Dockerfile"); ok { + for _, pick := range []func() (entry, bool){ + func() (entry, bool) { return findManifest(entries, root, "Dockerfile") }, + func() (entry, bool) { return rootProductionDockerfile(entries, root) }, + func() (entry, bool) { return singleNestedDockerfile(entries, root) }, + } { + e, ok := pick() + if !ok { + continue + } raw, err := readEntry(e) - if err == nil { - result.Framework = "docker" - if p, ok := parseDockerfileExpose(raw); ok { - result.Port = p - } - return result, nil + if err != nil { + continue + } + result.Framework = "docker" + if p, ok := parseDockerfileExpose(raw); ok { + result.Port = p + } else if p, ok := composePort(entries, root); ok { + result.Port = p } + return result, nil } + platform := platformAssignsPort(entries, root) + compose, hasCompose := composePort(entries, root) + if e, ok := findManifest(entries, root, "package.json"); ok { raw, err := readEntry(e) if err == nil { if fw, port, ok := parsePackageJSON(raw); ok { result.Framework = fw - result.Port = port + result.Port = resolvePort(port, platform, compose, hasCompose) return result, nil } } @@ -111,14 +173,72 @@ func Detect(data []byte) (Result, error) { } if fw, port, ok := parsePythonManifest(raw); ok { result.Framework = fw - result.Port = port + result.Port = resolvePort(port, platform, compose, hasCompose) return result, nil } } + if fw, port, ok := detectCompiled(entries, root); ok { + result.Framework = fw + result.Port = resolvePort(port, platform, compose, hasCompose) + return result, nil + } + return result, nil } +// compiledManifests maps a build manifest to the framework name and default +// port the git-import path already uses (build-agent's frameworkDefaultPort). +// Without them the upload path answered "no manifest" for every Go and JVM +// repo, even though the pipeline knows how to build both. +var compiledManifests = []struct { + file string + framework string + port int +}{ + {"pom.xml", "maven", 8080}, + {"build.gradle", "gradle", 8080}, + {"build.gradle.kts", "gradle", 8080}, + {"go.mod", "go", 8080}, +} + +// platformFiles declare that the port is assigned by the host platform and read +// from $PORT: a Procfile, or a Railway/Nixpacks config. Naming a number for such +// a repo publishes a port nothing listens on. +var platformFiles = []string{"Procfile", "railway.json", "railway.toml", "nixpacks.toml"} + +func platformAssignsPort(entries []entry, root string) bool { + for _, name := range platformFiles { + if _, ok := findManifest(entries, root, name); ok { + return true + } + } + return false +} + +// resolvePort ranks the three answers a non-Dockerfile repo can give, from +// evidence to convention: a compose mapping states the port, a platform config +// states that there is no fixed port, and only then does the per-framework +// default apply. +func resolvePort(fallback int, platform bool, compose int, hasCompose bool) int { + if hasCompose { + return compose + } + if platform { + return 0 + } + return fallback +} + +func detectCompiled(entries []entry, root string) (string, int, bool) { + for _, m := range compiledManifests { + if _, ok := findManifest(entries, root, m.file); ok { + return m.framework, m.port, true + } + } + return "", 0, false +} + func detectFormat(data []byte) (Format, error) { if len(data) >= 4 && bytes.HasPrefix(data, []byte("PK\x03\x04")) { return FormatZip, nil @@ -153,7 +273,7 @@ func listZipEntries(data []byte) ([]entry, error) { if i >= maxEntries { break } - if f.FileInfo().IsDir() || strings.Contains(f.Name, "..") { + if f.FileInfo().IsDir() || strings.Contains(f.Name, "..") || !isCandidate(f.Name) { continue } f := f @@ -186,7 +306,16 @@ func listTarGzEntries(data []byte) ([]entry, error) { if err != nil { return nil, err } - if hdr.Typeflag != tar.TypeReg || strings.Contains(hdr.Name, "..") { + if strings.Contains(hdr.Name, "..") { + continue + } + if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink { + if isCandidate(hdr.Name) && !strings.Contains(hdr.Linkname, "..") { + out = append(out, entry{name: hdr.Name, link: hdr.Linkname}) + } + continue + } + if hdr.Typeflag != tar.TypeReg || !isCandidate(hdr.Name) { continue } limit := hdr.Size @@ -239,7 +368,129 @@ func findManifest(entries []entry, root, name string) (entry, bool) { for _, e := range entries { rel := strings.TrimPrefix(e.name, root) if rel == name { - return e, true + return resolveLink(entries, e, 0) + } + } + return entry{}, false +} + +// singleNestedDockerfile returns the archive's only Dockerfile when it lives +// outside the root, which is how a large share of real repos ship one +// (gotify keeps docker/Dockerfile, mealie docker/Dockerfile). +// +// "Only" is the whole rule: a repo with several Dockerfiles is describing +// several images, and choosing among them would be a guess whose cost lands on +// the user as an app that builds the wrong thing. Test and example images are +// excluded by path, since a repo that ships one app Dockerfile plus a test +// fixture is still unambiguous to a human. +func singleNestedDockerfile(entries []entry, root string) (entry, bool) { + if e, ok := nestedDockerfile(entries, root, true); ok { + return e, true + } + return nestedDockerfile(entries, root, false) +} + +// nestedDockerfile scans the archive's non-root Dockerfiles and returns the one +// candidate, if there is exactly one. +// +// With productionOnly set it looks only inside a production/ directory. Repos +// that lay their Dockerfiles out by environment (wger keeps base, development, +// demo and production side by side) are ambiguous only until that convention is +// read; the environment named production is the one that ships. +func nestedDockerfile(entries []entry, root string, productionOnly bool) (entry, bool) { + var found entry + count := 0 + for _, e := range entries { + rel := strings.TrimPrefix(e.name, root) + base := rel + if i := strings.LastIndexByte(base, '/'); i >= 0 { + base = base[i+1:] + } + if base != "Dockerfile" || !strings.Contains(rel, "/") || isExcludedPath(rel) { + continue + } + if productionOnly && !strings.Contains(rel, "production/") { + continue + } + resolved, ok := resolveLink(entries, e, 0) + if !ok { + continue + } + count++ + if count > 1 { + return entry{}, false + } + found = resolved + } + return found, count == 1 +} + +// excludedDirs are path segments whose Dockerfiles never describe the app +// itself, so their presence must not turn an unambiguous repo into an +// ambiguous one. +var excludedDirs = []string{ + "test/", "tests/", "e2e/", "example/", "examples/", "docs/", + "playwright/", "fixtures/", "contrib/", ".devcontainer/", ".github/", +} + +// productionDockerfileNames are the root Dockerfile variants that name the +// shipped image. A repo like Ghost carries only Dockerfile.production in its +// root; reading nothing there meant falling through to package.json and +// answering with the dev server's port instead of the image's. +var productionDockerfileNames = []string{"Dockerfile.production", "Dockerfile.prod"} + +// rootProductionDockerfile returns the root production Dockerfile when exactly +// one of the known names is present. Two candidates mean the repo, not the +// detector, decides which image ships, so it refuses. +func rootProductionDockerfile(entries []entry, root string) (entry, bool) { + var found entry + count := 0 + for _, name := range productionDockerfileNames { + e, ok := findManifest(entries, root, name) + if !ok { + continue + } + count++ + found = e + } + return found, count == 1 +} + +func isExcludedPath(rel string) bool { + for _, dir := range excludedDirs { + if strings.HasPrefix(rel, dir) || strings.Contains(rel, "/"+dir) { + return true + } + } + return false +} + +// maxLinkHops bounds symlink chasing; a cycle inside a hostile archive must +// not turn into an infinite loop. +const maxLinkHops = 4 + +// resolveLink follows a symlink entry to the regular file it names. The link +// target is tried both as an archive-absolute path (how hard links store it) +// and as a path relative to the link's own directory (how symlinks store it). +// A link that resolves to nothing is reported as "not found" rather than as an +// empty manifest, since an empty manifest would silently answer "no framework". +func resolveLink(entries []entry, e entry, hop int) (entry, bool) { + if e.link == "" { + return e, true + } + if hop >= maxLinkHops { + return entry{}, false + } + dir := "" + if i := strings.LastIndexByte(e.name, '/'); i >= 0 { + dir = e.name[:i+1] + } + for _, candidate := range []string{e.link, path.Clean(dir + e.link)} { + for _, other := range entries { + if other.name != candidate || other.name == e.name { + continue + } + return resolveLink(entries, other, hop+1) } } return entry{}, false @@ -258,14 +509,102 @@ func readEntry(e entry) ([]byte, error) { return io.ReadAll(io.LimitReader(r, limit)) } -var exposeRe = regexp.MustCompile(`(?im)^\s*EXPOSE\s+(\d+)`) +var ( + composeMappedRe = regexp.MustCompile(`(?m)^\s*-\s*"?[^"\n]*?:(\d+)(?:/(?:tcp|udp))?"?\s*$`) + composeDefaultRe = regexp.MustCompile(`\$\{[A-Za-z_][A-Za-z0-9_]*:-(\d+)\}`) + composeTargetRe = regexp.MustCompile(`(?m)^\s*target:\s*"?(\d+)`) +) + +// composePort recovers the container port from a root compose file, for repos +// that ship a Dockerfile without EXPOSE. +// +// It looks at the root compose file first and, only if that yields nothing, +// at compose files anywhere else in the archive: repos that keep the app's +// compose next to a nested Dockerfile (mealie, memos) are common, and the root +// file, when present, is the more authoritative of the two. +// +// It answers only when every published mapping agrees on one container port. +// A compose file with several distinct targets describes several services, and +// picking one of them would be guessing — and a guessed port is exactly what +// turns a healthy app into a failing readiness probe. Silence (port 0) leaves +// the decision to the build template, which is the safe default. +func composePort(entries []entry, root string) (int, bool) { + rootNames := map[string]bool{ + "docker-compose.yml": true, + "docker-compose.yaml": true, + "compose.yml": true, + "compose.yaml": true, + } + for _, scope := range []bool{true, false} { + found := map[int]bool{} + for _, e := range entries { + rel := strings.TrimPrefix(e.name, root) + base := rel + if i := strings.LastIndexByte(base, '/'); i >= 0 { + base = base[i+1:] + } + if !rootNames[base] || isExcludedPath(rel) { + continue + } + if scope && strings.Contains(rel, "/") { + continue + } + raw, err := readEntry(e) + if err != nil { + continue + } + raw = composeDefaultRe.ReplaceAll(raw, []byte("$1")) + for _, re := range []*regexp.Regexp{composeMappedRe, composeTargetRe} { + for _, m := range re.FindAllSubmatch(raw, -1) { + p, err := strconv.Atoi(string(m[1])) + if err == nil && p > 0 && p <= 65535 { + found[p] = true + } + } + } + } + if len(found) == 1 { + for p := range found { + return p, true + } + } + } + return 0, false +} + +var ( + exposeRe = regexp.MustCompile(`(?im)^\s*EXPOSE\s+(\S+)`) + dockerVarRe = regexp.MustCompile(`(?im)^\s*(?:ENV|ARG)\s+([A-Za-z_][A-Za-z0-9_]*)\s*[= ]\s*"?(\d+)"?\s*$`) + varRefRe = regexp.MustCompile(`^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$`) +) +// parseDockerfileExpose reads the first EXPOSE, resolving a variable reference +// against ENV and ARG defaults declared in the same Dockerfile. +// +// "EXPOSE $PORT" above "ENV PORT=3000" is how homepage, shiori and gotify +// declare their port; reading only literal digits treated all three as if they +// declared nothing, and the port then had to be guessed downstream. func parseDockerfileExpose(raw []byte) (int, bool) { m := exposeRe.FindSubmatch(raw) if m == nil { return 0, false } - p, err := strconv.Atoi(string(m[1])) + token := strings.TrimSpace(string(m[1])) + if i := strings.IndexByte(token, '/'); i >= 0 { + token = token[:i] + } + if ref := varRefRe.FindStringSubmatch(token); ref != nil { + token = "" + for _, v := range dockerVarRe.FindAllSubmatch(raw, -1) { + if string(v[1]) == ref[1] { + token = string(v[2]) + } + } + if token == "" { + return 0, false + } + } + p, err := strconv.Atoi(token) if err != nil || p <= 0 || p > 65535 { return 0, false } diff --git a/backend/internal/sourcedetect/detect_test.go b/backend/internal/sourcedetect/detect_test.go index 52d91e91..72958e00 100644 --- a/backend/internal/sourcedetect/detect_test.go +++ b/backend/internal/sourcedetect/detect_test.go @@ -212,3 +212,277 @@ func TestDetectZipSlipEntriesSkipped(t *testing.T) { t.Errorf("Framework = %q, want react (zip-slip entry must be skipped, not crash)", res.Framework) } } + +type tarLink struct { + name string + target string +} + +func buildTarGzWithLinks(t *testing.T, files []zipFile, links []tarLink) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + for _, l := range links { + hdr := &tar.Header{ + Name: l.name, + Linkname: l.target, + Typeflag: tar.TypeSymlink, + Mode: 0777, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tar symlink header %s: %v", l.name, err) + } + } + for _, f := range files { + hdr := &tar.Header{Name: f.name, Mode: 0644, Size: int64(len(f.body))} + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tar header %s: %v", f.name, err) + } + if _, err := tw.Write([]byte(f.body)); err != nil { + t.Fatalf("tar write %s: %v", f.name, err) + } + } + if err := tw.Close(); err != nil { + t.Fatalf("tar close: %v", err) + } + if err := gw.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + return buf.Bytes() +} + +func TestDetectFollowsRootDockerfileSymlink(t *testing.T) { + data := buildTarGzWithLinks(t, + []zipFile{{"app/docker/Dockerfile.debian", "FROM alpine\nEXPOSE 8080\n"}}, + []tarLink{{"app/Dockerfile", "docker/Dockerfile.debian"}}, + ) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "docker" { + t.Errorf("Framework = %q, want docker (root Dockerfile is a symlink, as in vaultwarden)", res.Framework) + } + if res.Port != 8080 { + t.Errorf("Port = %d, want 8080 from the link target's EXPOSE", res.Port) + } +} + +func TestDetectDanglingSymlinkIsNotAManifest(t *testing.T) { + data := buildTarGzWithLinks(t, + []zipFile{{"app/package.json", `{"dependencies":{"react-scripts":"5.0.0"}}`}}, + []tarLink{{"app/Dockerfile", "docker/Dockerfile.missing"}}, + ) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "react" { + t.Errorf("Framework = %q, want react: a symlink pointing at nothing must not answer for the Dockerfile", res.Framework) + } +} + +func TestDetectSymlinkCycleTerminates(t *testing.T) { + data := buildTarGzWithLinks(t, nil, []tarLink{ + {"app/Dockerfile", "Dockerfile.a"}, + {"app/Dockerfile.a", "Dockerfile"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "" { + t.Errorf("Framework = %q, want empty for a symlink cycle", res.Framework) + } +} + +func TestDetectComposePortWhenDockerfileHasNoExpose(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/Dockerfile", "FROM alpine\nCMD [\"sh\"]\n"}, + {"app/docker-compose.yml", "services:\n web:\n build: .\n ports:\n - \"3000:80\"\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "docker" || res.Port != 80 { + t.Errorf("got %q:%d, want docker:80 (container side of the compose mapping)", res.Framework, res.Port) + } +} + +func TestDetectComposeWithSeveralPortsStaysSilent(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/Dockerfile", "FROM alpine\n"}, + {"app/compose.yaml", "services:\n web:\n ports:\n - \"3000:80\"\n api:\n ports:\n - \"9000:9000\"\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Port != 0 { + t.Errorf("Port = %d, want 0: two services disagree, and a guessed port is worse than none", res.Port) + } +} + +func TestDetectExposeWinsOverCompose(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/Dockerfile", "FROM alpine\nEXPOSE 5000\n"}, + {"app/docker-compose.yml", "services:\n web:\n ports:\n - \"8080:8080\"\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Port != 5000 { + t.Errorf("Port = %d, want 5000: EXPOSE is the deploy contract, compose is only the fallback", res.Port) + } +} + +func TestDetectExposeResolvesEnvVar(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/Dockerfile", "FROM alpine\nENV PORT=3000\nEXPOSE $PORT\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Port != 3000 { + t.Errorf("Port = %d, want 3000: EXPOSE $PORT with ENV PORT=3000 above states the port", res.Port) + } +} + +func TestDetectExposeResolvesBracedArg(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/Dockerfile", "FROM alpine\nARG APP_PORT 8080\nEXPOSE ${APP_PORT}/tcp\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Port != 8080 { + t.Errorf("Port = %d, want 8080", res.Port) + } +} + +func TestDetectExposeUnresolvedVarStaysSilent(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/Dockerfile", "FROM alpine\nEXPOSE $MYSTERY_PORT\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Port != 0 { + t.Errorf("Port = %d, want 0: an unresolvable variable is not a port, and guessing one has already caused an outage", res.Port) + } +} + +func TestDetectIgnoresDevcontainerDockerfile(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/.devcontainer/Dockerfile", "FROM alpine\nEXPOSE 1234\n"}, + {"app/docker/Dockerfile", "FROM alpine\nENV APP_PORT=9000\nEXPOSE ${APP_PORT}\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "docker" || res.Port != 9000 { + t.Errorf("got %s:%d, want docker:9000: a devcontainer describes the dev box, not the app", res.Framework, res.Port) + } +} + +func TestDetectRootProductionDockerfile(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/Dockerfile.production", "FROM alpine\nEXPOSE 2368\n"}, + {"app/package.json", `{"dependencies":{"express":"4"}}`}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "docker" || res.Port != 2368 { + t.Errorf("got %s:%d, want docker:2368", res.Framework, res.Port) + } +} + +func TestDetectGoModule(t *testing.T) { + data := buildTarGz(t, []zipFile{{"app/go.mod", "module example.com/x\n\ngo 1.23\n"}}) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "go" || res.Port != 8080 { + t.Errorf("got %s:%d, want go:8080", res.Framework, res.Port) + } +} + +func TestDetectMavenProject(t *testing.T) { + data := buildTarGz(t, []zipFile{{"app/pom.xml", "\n"}}) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "maven" || res.Port != 8080 { + t.Errorf("got %s:%d, want maven:8080", res.Framework, res.Port) + } +} + +func TestDetectProcfileLeavesPortToPlatform(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/Procfile", "web: npm start\n"}, + {"app/package.json", `{"dependencies":{"express":"4"}}`}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "node" { + t.Errorf("Framework = %q, want node", res.Framework) + } + if res.Port != 0 { + t.Errorf("Port = %d, want 0: a Procfile app listens on the $PORT the platform assigns", res.Port) + } +} + +func TestDetectProcfileKeepsDockerfileEvidence(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/Procfile", "web: ./server\n"}, + {"app/Dockerfile", "FROM alpine\nEXPOSE 4567\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Port != 4567 { + t.Errorf("Port = %d, want 4567: EXPOSE is evidence, not a per-framework default", res.Port) + } +} + +func TestDetectComposeBeatsFrameworkDefault(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/package.json", `{"dependencies":{"express":"4"}}`}, + {"app/docker-compose.yml", "services:\n app:\n ports:\n - \"${LD_HOST_PORT:-9090}:9090\"\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "node" || res.Port != 9090 { + t.Errorf("got %s:%d, want node:9090: a compose mapping is evidence, the framework default is only a convention", res.Framework, res.Port) + } +} + +func TestDetectRailwayConfigLeavesPortToPlatform(t *testing.T) { + data := buildTarGz(t, []zipFile{ + {"app/railway.json", `{"build":{"builder":"NIXPACKS"}}`}, + {"app/requirements.txt", "Django==5.0\n"}, + }) + res, err := Detect(data) + if err != nil { + t.Fatalf("Detect: %v", err) + } + if res.Framework != "django" || res.Port != 0 { + t.Errorf("got %s:%d, want django:0", res.Framework, res.Port) + } +} diff --git a/build-agent/internal/github/app.go b/build-agent/internal/github/app.go index 09863a82..245ae49e 100644 --- a/build-agent/internal/github/app.go +++ b/build-agent/internal/github/app.go @@ -49,6 +49,9 @@ type App interface { // BranchHead resolves the current HEAD commit sha and message for a branch. // token may be empty for anonymous access to a public repo. BranchHead(ctx context.Context, token, repoFullName, branch string) (sha, message string, err error) + // SearchRepos searches public repositories by free text, so the console's + // one input field can answer "n8n" with repositories instead of nothing. + SearchRepos(ctx context.Context, query string, limit int) ([]SearchHit, error) } // InstallationAccount identifies the org/user a GitHub App installation belongs diff --git a/build-agent/internal/github/search.go b/build-agent/internal/github/search.go new file mode 100644 index 00000000..60231e22 --- /dev/null +++ b/build-agent/internal/github/search.go @@ -0,0 +1,136 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" +) + +// SearchHit is one public repository returned by a repository search. +// +// Everything here is what a person needs to decide whether this is the project +// they meant: the name, one line about it, how many people starred it, and +// whether it is archived. Stars are not a quality claim, they are the only +// ordering signal a stranger can read at a glance; Archived is here because +// deploying a dead project is a specific kind of bad afternoon. +type SearchHit struct { + FullName string `json:"full_name"` + Description string `json:"description"` + Stars int `json:"stars"` + DefaultBranch string `json:"default_branch"` + AvatarURL string `json:"avatar_url"` + Archived bool `json:"archived"` + Fork bool `json:"fork"` + License string `json:"license"` + HTMLURL string `json:"html_url"` +} + +// searchMaxLimit bounds one page. The console shows a handful of suggestions +// under an input; asking GitHub for more is bandwidth spent on rows nobody +// scrolls to. +const searchMaxLimit = 10 + +// SearchRepos searches public GitHub repositories by free text. +// +// Authentication is best-effort by design. GitHub's search endpoint allows 30 +// requests a minute with a token and 10 without, and both numbers are per +// source IP for the whole cluster — so the client mints an installation token +// when the App has any installation, and falls back to anonymous rather than +// failing. An interactive search that goes dark because a GitHub App +// installation was removed would be a strange way to lose the feature. +// +// The query is fenced to public, non-fork repositories: a fork is almost never +// what someone typing a product name meant, and a private repository we cannot +// clone is a result that only exists to disappoint. +func (c *Client) SearchRepos(ctx context.Context, query string, limit int) ([]SearchHit, error) { + q := strings.TrimSpace(query) + if q == "" { + return []SearchHit{}, nil + } + if limit <= 0 || limit > searchMaxLimit { + limit = searchMaxLimit + } + + endpoint := fmt.Sprintf("%s/search/repositories?q=%s&sort=stars&order=desc&per_page=%d", + apiBase, url.QueryEscape(q+" fork:false is:public"), limit) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + if token := c.searchToken(ctx); token != "" { + req.Header.Set("Authorization", "token "+token) + } + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("search repos: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("search repos: %s", readErr(resp)) + } + + var out struct { + Items []struct { + FullName string `json:"full_name"` + Description string `json:"description"` + Stars int `json:"stargazers_count"` + DefaultBranch string `json:"default_branch"` + Archived bool `json:"archived"` + Fork bool `json:"fork"` + HTMLURL string `json:"html_url"` + Owner struct { + AvatarURL string `json:"avatar_url"` + } `json:"owner"` + License struct { + SpdxID string `json:"spdx_id"` + } `json:"license"` + } `json:"items"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, fmt.Errorf("decode search: %w", err) + } + + hits := make([]SearchHit, 0, len(out.Items)) + for _, it := range out.Items { + hits = append(hits, SearchHit{ + FullName: it.FullName, + Description: truncate(it.Description, 200), + Stars: it.Stars, + DefaultBranch: it.DefaultBranch, + AvatarURL: it.Owner.AvatarURL, + Archived: it.Archived, + Fork: it.Fork, + License: it.License.SpdxID, + HTMLURL: it.HTMLURL, + }) + } + return hits, nil +} + +// searchToken returns an installation token to search with, or "" to search +// anonymously. +// +// Any installation will do: repository search over public repositories returns +// the same public index whichever installation asks, and the token is here for +// the rate limit rather than for access. Every failure path returns "" — a +// slower anonymous search is a better outcome than no search. +func (c *Client) searchToken(ctx context.Context) string { + if c.appID == "" || len(c.appKey) == 0 { + return "" + } + insts, err := c.ListInstallations(ctx) + if err != nil || len(insts) == 0 { + return "" + } + token, err := c.InstallToken(ctx, insts[0].InstallationID) + if err != nil { + return "" + } + return token +} diff --git a/build-agent/internal/server/framework_detect_live_test.go b/build-agent/internal/server/framework_detect_live_test.go index 225e8584..5a00aaf8 100644 --- a/build-agent/internal/server/framework_detect_live_test.go +++ b/build-agent/internal/server/framework_detect_live_test.go @@ -29,6 +29,9 @@ func (a *liveTokenApp) PostStatus(_ context.Context, _ int64, _, _, _, _, _ stri func (a *liveTokenApp) BranchHead(_ context.Context, _, _, _ string) (string, string, error) { return "", "", nil } +func (a *liveTokenApp) SearchRepos(_ context.Context, _ string, _ int) ([]github.SearchHit, error) { + return nil, nil +} func TestLiveDetectFrameworks(t *testing.T) { tok := os.Getenv("GITHUB_TOKEN") diff --git a/build-agent/internal/server/github_endpoints_test.go b/build-agent/internal/server/github_endpoints_test.go index 0edf4072..3fd1e6b8 100644 --- a/build-agent/internal/server/github_endpoints_test.go +++ b/build-agent/internal/server/github_endpoints_test.go @@ -22,6 +22,10 @@ type fakeApp struct { acct *github.InstallationAccount acctErr error insts []github.InstallationAccount + hits []github.SearchHit + searchQ string + searchN int + searchEr error } func (f *fakeApp) InstallToken(_ context.Context, _ int64) (string, error) { return "t", nil } @@ -44,6 +48,11 @@ func (f *fakeApp) PostStatus(_ context.Context, _ int64, _, _, _, _, _ string) e func (f *fakeApp) BranchHead(_ context.Context, _, _, _ string) (string, string, error) { return "", "", nil } +func (f *fakeApp) SearchRepos(_ context.Context, q string, limit int) ([]github.SearchHit, error) { + f.searchQ = q + f.searchN = limit + return f.hits, f.searchEr +} func newTestServer(gh github.App) http.Handler { s := New(":0", &Options{GitHub: gh}) diff --git a/build-agent/internal/server/server.go b/build-agent/internal/server/server.go index 374be548..1299c387 100644 --- a/build-agent/internal/server/server.go +++ b/build-agent/internal/server/server.go @@ -89,6 +89,7 @@ func (s *Server) Start(ctx context.Context) error { // exchange a user OAuth code → the installations that user can access. // The backend proxies here so the OAuth client secret stays in the agent. mux.HandleFunc("POST /github/oauth/exchange", s.handleOAuthExchange) + mux.HandleFunc("GET /github/search/repos", s.handleSearchRepos) } // Framework detection is best-effort here (no clone in the agent process — a // clone-based Nixpacks detect belongs in the build Job). Always 200 so the @@ -395,6 +396,41 @@ func (s *Server) handleAppInstallations(w http.ResponseWriter, r *http.Request) writeJSON(w, map[string]any{"installations": insts}) } +// handleSearchRepos searches public GitHub repositories by free text. +// GET /github/search/repos?q=n8n&limit=8 → {"repositories":[...]}. +// +// It lives in the agent because the agent is where the App key lives, and the +// search endpoint's rate limit triples with a token. The backend caches the +// answer before it ever gets here — this handler is deliberately not the place +// that thinks about budgets, it is the place that has the credential. +func (s *Server) handleSearchRepos(w http.ResponseWriter, r *http.Request) { + q := strings.TrimSpace(r.URL.Query().Get("q")) + if q == "" { + http.Error(w, "q is required", http.StatusBadRequest) + return + } + limit := 0 + if v := r.URL.Query().Get("limit"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + http.Error(w, "limit must be a positive integer", http.StatusBadRequest) + return + } + limit = n + } + + hits, err := s.gh.SearchRepos(r.Context(), q, limit) + if err != nil { + log.Warn().Err(err).Str("q", q).Msg("github repo search failed") + http.Error(w, "failed to search repositories", http.StatusBadGateway) + return + } + if hits == nil { + hits = []github.SearchHit{} + } + writeJSON(w, map[string]any{"repositories": hits}) +} + // handleOAuthExchange swaps a user OAuth code for the installations that user can // access. POST /github/oauth/exchange {"code":"..."} → {"login","installations":[...]}. // The OAuth client secret stays in the agent; the backend never sees it. diff --git a/build-agent/internal/worker/liveinstall_test.go b/build-agent/internal/worker/liveinstall_test.go index 3143ffa4..8f197740 100644 --- a/build-agent/internal/worker/liveinstall_test.go +++ b/build-agent/internal/worker/liveinstall_test.go @@ -30,6 +30,9 @@ func (f *fakeApp) PostStatus(context.Context, int64, string, string, string, str func (f *fakeApp) BranchHead(context.Context, string, string, string) (string, string, error) { return "", "", nil } +func (f *fakeApp) SearchRepos(context.Context, string, int) ([]github.SearchHit, error) { + return nil, nil +} func TestLiveInstallationForOwner(t *testing.T) { app := &fakeApp{installs: []github.InstallationAccount{ diff --git a/frontend/components/console/template-deploy-cards.tsx b/frontend/components/console/template-deploy-cards.tsx index 8fcc07db..0127650e 100644 --- a/frontend/components/console/template-deploy-cards.tsx +++ b/frontend/components/console/template-deploy-cards.tsx @@ -1,16 +1,12 @@ "use client"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { useRouter } from "next/navigation"; -import { gitApi, buildsApi } from "@/lib/api"; +import { gitApi, solutionsApi } from "@/lib/api"; +import type { Solution, SolutionCandidate } from "@/lib/types"; import { ResourceIcon } from "@/components/shell/icons"; import { Spinner } from "@/components/ui/spinner"; import { useT } from "@/lib/i18n/console/context"; import { trackBuildStart } from "@/lib/build-watch"; -import { STARTER_TEMPLATES, type StarterTemplate } from "@/lib/starter-templates"; - -type Template = StarterTemplate; - -const TEMPLATES: Template[] = STARTER_TEMPLATES; function toKubeName(s: string): string { return s @@ -18,7 +14,13 @@ function toKubeName(s: string): string { .replace(/[^a-z0-9-]+/g, "-") .replace(/-+/g, "-") .replace(/^-|-$/g, "") - .slice(0, 63); + .slice(0, 40); +} + +/** Random suffix so deploying the same project twice never collides on a name. */ +function uniqueAppName(base: string): string { + const suffix = Math.random().toString(36).slice(2, 8); + return toKubeName(`${toKubeName(base).slice(0, 30)}-${suffix}`); } export interface TemplateDeployCardsProps { @@ -36,60 +38,118 @@ export interface TemplateDeployCardsProps { } /** - * No-GitHub escape hatch: deploys one of the starter templates by app name - * directly, skipping the git-account connect flow entirely. Shared across the - * project overview, the apps empty state, and the git-import OAuth wall so the - * option is reachable everywhere a user would otherwise hit the GitHub gate. - * `hero` swaps in the activation-focused heading copy and a larger card for - * the placements where this is the primary onramp; the git-import wall keeps - * the default heading since Git is already the primary action there. + * No-GitHub escape hatch: builds and deploys a real open-source project — or + * any public repository the visitor pastes — without connecting a git account. + * + * Both paths run the ordinary customer flow (link the public repo, build it, + * deploy the image), which is the point: what the visitor sees on the empty + * screen is the same machinery their own first repository will go through. The + * catalog comes from the backend rather than a list in this file, so adding a + * project is a backend change and the console never disagrees with it about + * which branch or port an entry builds with. */ export function TemplateDeployCards({ projectId, envId, compact, hero, className }: TemplateDeployCardsProps) { const { t } = useT(); const router = useRouter(); + const [solutions, setSolutions] = useState(null); const [deployingKey, setDeployingKey] = useState(null); const [templateError, setTemplateError] = useState(null); + const [query, setQuery] = useState(""); + const [candidates, setCandidates] = useState(null); + const [resolving, setResolving] = useState(false); + const [searchFailed, setSearchFailed] = useState(false); + + useEffect(() => { + let cancelled = false; + solutionsApi + .list() + .then((res) => { + if (!cancelled) setSolutions(res.solutions ?? []); + }) + .catch(() => { + if (!cancelled) setSolutions([]); + }); + return () => { + cancelled = true; + }; + }, []); + + /** + * Resolves what the customer typed, one debounced request per pause in the + * typing. The delay is not cosmetic: every keystroke that reaches the backend + * can become a GitHub search, and that budget is 30 requests a minute for the + * whole platform, not per customer. + */ + useEffect(() => { + const typed = query.trim(); + if (typed.length < 2) return; + let cancelled = false; + const timer = setTimeout(() => { + setResolving(true); + solutionsApi + .resolve(projectId, typed) + .then((res) => { + if (cancelled) return; + setCandidates(res.candidates ?? []); + setSearchFailed(res.search_failed); + }) + .catch(() => { + if (!cancelled) { + setCandidates([]); + setSearchFailed(true); + } + }) + .finally(() => { + if (!cancelled) setResolving(false); + }); + }, 350); + return () => { + cancelled = true; + clearTimeout(timer); + }; + }, [query, projectId]); /** - * Links the template repo, triggers the first build, then lands the user on - * the app overview page. + * Installs one project: a single call that links the public repository, + * orders any managed database the catalog entry declares it needs, and + * queues the first build. No connected GitHub account is required — the + * repository is public and the backend derives the clone URL from its name. * - * DESTINATION. Overview, not `/deployments`. `TriggerBuild` is the single - * largest terminal action in the measured path graph, and every user in that - * cluster had `builds.status='success'` -- people leave right after their - * app works. The deployments feed has no live-URL surface at all, so the - * dominant new-user path never saw one: 64 successful builds in 14d against - * 38 lifetime views of a "your app is live" panel, by 2 distinct users. - * Overview carries `AppLatestBuildCard` (live URL + open CTAs), polls the - * app phase itself, and fires the Metrika deploy-success goal, which - * template deploys never reached before. + * The app name is minted here rather than server-side so deploying the same + * project twice never collides on a name. * - * The app row is created asynchronously and lags the build trigger, so the - * destination must tolerate a missing app: overview retries `appsApi.list` - * 40 times at 3s intervals (~120s) before rendering not-found. + * DESTINATION. The app overview, not `/deployments`. The deployments feed + * carries no live-URL surface, so the dominant new-user path never saw one; + * overview shows the live URL, polls the app phase and fires the Metrika + * deploy-success goal. The app row lags the build trigger, and overview + * tolerates that by retrying `appsApi.list` for ~120s before not-found. */ - async function deployTemplate(tpl: Template) { + async function deploy(opts: { + key: string; + appBase: string; + slug?: string; + repoFullName?: string; + branch?: string; + rootDir?: string; + framework?: string; + port?: number; + profile?: string; + }) { if (!envId || deployingKey) return; setTemplateError(null); - setDeployingKey(tpl.key); - const appName = toKubeName(`${tpl.key}-${Math.random().toString(36).slice(2, 8)}`); + setDeployingKey(opts.key); + const appName = uniqueAppName(opts.appBase); try { - try { - await gitApi.linkRepo(projectId, envId, { - installation_id: "", - repo_full_name: tpl.repo_full_name, - app_name: appName, - production_branch: "main", - root_dir: ".", - auto_deploy: false, - port: tpl.port, - profile: "small", - }); - } catch (err) { - const msg = err instanceof Error ? err.message : t("overview.templates.error"); - if (!/409|already/i.test(msg)) throw new Error(msg); - } - const { build } = await buildsApi.trigger(projectId, envId, appName); + const { build } = await solutionsApi.install(projectId, envId, { + slug: opts.slug, + repo: opts.repoFullName, + app_name: appName, + branch: opts.branch, + root_dir: opts.rootDir, + framework: opts.framework, + port: opts.port, + profile: opts.profile, + }); if (build?.id) trackBuildStart({ projectId, envId, appName, buildId: build.id }); router.push(`/projects/${projectId}/apps/${appName}?envId=${envId}`); } catch (err) { @@ -98,6 +158,57 @@ export function TemplateDeployCards({ projectId, envId, compact, hero, className } } + function deploySolution(s: Solution) { + return deploy({ key: s.slug, appBase: s.slug, slug: s.slug }); + } + + /** + * Deploys one resolver row. + * + * A catalog row already carries the build spec we verified, so it goes + * straight through. A repository row — pasted or found by search — carries a + * name and nothing else, so detection runs first: the port a repository + * actually listens on is what separates an app that answers from one that + * deploys green and returns 502, which reads as the platform being broken. + * A managed row is not a build at all and hands over to the databases page, + * where the customer picks size and backups. + */ + async function deployCandidate(c: SolutionCandidate) { + if (!envId || deployingKey) return; + if (c.kind === "managed") { + router.push(`/projects/${projectId}/databases?envId=${envId}`); + return; + } + if (c.kind === "solution") { + await deploy({ key: `cand:${c.slug}`, appBase: c.slug, slug: c.slug }); + return; + } + const key = `cand:${c.repo}`; + setTemplateError(null); + setDeployingKey(key); + let framework = c.framework || undefined; + let port = c.port || undefined; + try { + const detected = await gitApi.detectPublic(projectId, c.repo); + framework = detected.framework ?? framework; + port = detected.port ?? port; + } catch { + /* Best effort: the build pipeline detects again on the real checkout, which sees more than the GitHub API does. */ + } + setDeployingKey(null); + await deploy({ + key, + appBase: c.repo.split("/")[1] ?? "app", + repoFullName: c.repo, + branch: c.branch, + rootDir: c.root_dir || ".", + framework, + port, + }); + } + + const asking = query.trim().length >= 2; + const body = ( <> {!compact && ( @@ -127,18 +238,83 @@ export function TemplateDeployCards({ projectId, envId, compact, hero, className {templateError} )} +
- {TEMPLATES.map((tpl) => ( - ) + : solutions.map((s) => ( + deploySolution(s)} + /> + ))} +
+ +
+

+ {t("overview.templates.ask.title")} +

+

+ {t("overview.templates.ask.hint")} +

+
+ + setQuery(e.target.value)} + placeholder={t("overview.templates.ask.placeholder")} disabled={!!deployingKey || !envId} - onClick={() => deployTemplate(tpl)} + aria-label={t("overview.templates.ask.title")} + className="flex-1 bg-transparent text-sm text-gray-900 dark:text-gray-100 placeholder:text-gray-400 focus:outline-none disabled:opacity-60" /> - ))} + {asking && resolving && } +
+ + {asking && searchFailed && ( +

+ {t("overview.templates.ask.searchFailed")} +

+ )} + + {asking && candidates !== null && candidates.length === 0 && !resolving && ( +

+ {t("overview.templates.ask.empty")} +

+ )} + + {asking && candidates !== null && candidates.length > 0 && ( +
    + {candidates.map((c) => ( + void deployCandidate(c)} + /> + ))} +
+ )}
); @@ -151,23 +327,96 @@ export function TemplateDeployCards({ projectId, envId, compact, hero, className ? "rounded-2xl border-2 border-blue-200 dark:border-blue-900/60 bg-gradient-to-br from-blue-50 to-white dark:from-blue-950/20 dark:to-gray-900 p-6 shadow-sm sm:p-8" : "rounded-xl border border-gray-200 dark:border-gray-800 bg-white dark:bg-gray-900 p-5 shadow-sm"; + return
{body}
; +} + +/** + * One resolver suggestion. The badge is the honest part: a catalog row carries + * a build spec someone verified, a search row carries a name and a star count, + * and the customer choosing between them deserves to know which is which. + */ +function CandidateRow({ + candidate, + busy, + disabled, + cta, + badge, + archivedLabel, + onClick, +}: { + candidate: SolutionCandidate; + busy: boolean; + disabled: boolean; + cta: string; + badge: string; + archivedLabel: string; + onClick: () => void; +}) { + return ( +
  • + {candidate.icon ? ( + + ) : ( +
    + +
    + )} +
    +

    + {candidate.name} + + {badge} + + {candidate.archived && ( + + {archivedLabel} + + )} +

    +

    + {candidate.tagline || candidate.repo} +

    +
    + {typeof candidate.stars === "number" && candidate.stars > 0 && ( + ★ {candidate.stars} + )} + +
    • + ); +} + +function SolutionCardSkeleton() { return ( -
    - {body} +
    +
    +
    +
    +
    +
    ); } -function TemplateCard({ - title, - hint, +function SolutionCard({ + solution, cta, busy, disabled, onClick, }: { - title: string; - hint: string; + solution: Solution; cta: string; busy: boolean; disabled: boolean; @@ -178,8 +427,11 @@ function TemplateCard({
    -

    {title}

    -

    {hint}

    +

    {solution.name}

    +

    {solution.tagline}

    +

    + {solution.repo} · {solution.license} +