-
Notifications
You must be signed in to change notification settings - Fork 66
Schema sync api #353
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. Weβll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Schema sync api #353
Changes from all commits
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
9533526
added ub push in cli
Ayush4958 9f17e02
created the ub push command
Ayush4958 0a81c4e
function send collection payload
Ayush4958 9e9507b
created & added the syncschemapayload
Ayush4958 b2ece8f
controller for schema synching
Ayush4958 26942d0
added route for sync schema
Ayush4958 b957cc3
added the restriction for sync schema
Ayush4958 a320cd4
Merge remote-tracking branch 'upstream/main' into schema-sync-api
Ayush4958 88c05db
fix(sdk): resolve eslint any type and unused var errors
Ayush4958 a80ae5e
Fix for code rabbit AI review
Ayush4958 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
230 changes: 230 additions & 0 deletions
230
apps/dashboard-api/src/controllers/syncSchema.controller.js
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,230 @@ | ||
| const { z } = require("zod"); | ||
| const { | ||
| Project, | ||
| AppError, | ||
| ApiResponse, | ||
| syncSchemaPayload, | ||
| resolveEffectivePlan, | ||
| getPlanLimits, | ||
| deleteProjectById, | ||
| deleteProjectByApiKeyCache, | ||
| setProjectById, | ||
| getProjectAccessQuery, | ||
| } = require("@urbackend/common"); | ||
|
|
||
| // ββ Helpers (mirrored from project.controller.js) βββββββββββββββββββββββββββ | ||
|
|
||
| const normalizeFieldKey = (key) => | ||
| String(key || "") | ||
| .replace(/\uFEFF/g, "") | ||
| .trim(); | ||
|
|
||
| const normalizeFieldType = (type) => | ||
| String(type || "") | ||
| .trim() | ||
| .toLowerCase(); | ||
|
|
||
| const isRequiredField = (required) => | ||
| required === true || | ||
| required === 1 || | ||
| String(required).trim().toLowerCase() === "true" || | ||
| String(required).trim() === "1"; | ||
|
|
||
| const toPlainObject = (value) => { | ||
| if (!value || typeof value !== "object") return value; | ||
| if (typeof value.toObject === "function") { | ||
| return value.toObject({ depopulate: true }); | ||
| } | ||
| if (value._doc && typeof value._doc === "object") { | ||
| return { ...value._doc }; | ||
| } | ||
| return value; | ||
| }; | ||
|
|
||
| const sanitizeSchemaFields = (schema = []) => { | ||
| if (!Array.isArray(schema)) return []; | ||
| return schema | ||
| .map((rawField) => { | ||
| const field = toPlainObject(rawField); | ||
| if (!field || typeof field !== "object") return null; | ||
|
|
||
| const normalizedKey = normalizeFieldKey(field.key); | ||
| if (!normalizedKey) return null; | ||
|
|
||
| const next = { ...field, key: normalizedKey }; | ||
| if (field.default !== undefined) { | ||
| next.default = field.default; | ||
| } | ||
|
|
||
| if (Array.isArray(field.fields)) { | ||
| next.fields = sanitizeSchemaFields(field.fields); | ||
| } | ||
|
|
||
| if (field.items && typeof field.items === "object") { | ||
| next.items = { ...field.items }; | ||
| if (Array.isArray(field.items.fields)) { | ||
| next.items.fields = sanitizeSchemaFields(field.items.fields); | ||
| } | ||
| } | ||
|
|
||
| return next; | ||
| }) | ||
| .filter(Boolean); | ||
| }; | ||
|
|
||
| const getDefaultRlsForCollection = (collectionName, schema = []) => { | ||
| const normalizedName = String(collectionName || "").toLowerCase(); | ||
| const keys = sanitizeSchemaFields(schema).map((f) => f.key); | ||
|
|
||
| let ownerField = "userId"; | ||
| if (normalizedName === "users") { | ||
| ownerField = "_id"; | ||
| } else if (keys.includes("userId")) { | ||
| ownerField = "userId"; | ||
| } else if (keys.includes("ownerId")) { | ||
| ownerField = "ownerId"; | ||
| } | ||
|
|
||
| return { | ||
| enabled: false, | ||
| mode: "public-read", | ||
| ownerField, | ||
| requireAuthForWrite: true, | ||
| }; | ||
| }; | ||
|
|
||
| /** | ||
| * Validates that a users collection schema contains the required | ||
| * `email` (String, required) and `password` (String, required) fields. | ||
| */ | ||
| const validateUsersSchema = (schema) => { | ||
| if (!Array.isArray(schema)) return false; | ||
| const sanitized = sanitizeSchemaFields(schema); | ||
|
|
||
| const hasEmail = sanitized.find( | ||
| (f) => | ||
| normalizeFieldKey(f.key).toLowerCase() === "email" && | ||
| normalizeFieldType(f.type) === "string" && | ||
| isRequiredField(f.required), | ||
| ); | ||
|
|
||
| const hasPassword = sanitized.find( | ||
| (f) => | ||
| normalizeFieldKey(f.key).toLowerCase() === "password" && | ||
| normalizeFieldType(f.type) === "string" && | ||
| isRequiredField(f.required), | ||
| ); | ||
|
|
||
| return !!(hasEmail && hasPassword); | ||
| }; | ||
|
|
||
| // ββ Controller ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | ||
|
|
||
| /** | ||
| * PUT /projects/:projectId/sync-schema | ||
| * | ||
| * Atomically replaces the project's collection schema definitions. | ||
| * | ||
| * Behaviour: | ||
| * - Preserves RLS settings for collections that already exist. | ||
| * - Applies safe defaults for newly introduced collections. | ||
| * - Enforces plan-based collection limits. | ||
| * - Validates the `users` collection contract (email + password required). | ||
| * - Does NOT drop underlying MongoDB collections β only updates config. | ||
| */ | ||
| module.exports.syncSchema = async (req, res, next) => { | ||
| try { | ||
| // 1. Validate payload | ||
| const { collections: incoming } = syncSchemaPayload.parse(req.body); | ||
|
|
||
| const { projectId } = req.params; | ||
|
|
||
| // 2. Load project (authorizeProject middleware already attached req.project) | ||
| const project = req.project; | ||
| if (!project) { | ||
| return next(new AppError(404, "Project not found or access denied")); | ||
| } | ||
|
|
||
| // 3. Plan enforcement β check collection count limit | ||
| if (req.developer) { | ||
| const effectivePlan = resolveEffectivePlan(req.developer); | ||
| const limits = getPlanLimits({ | ||
| plan: effectivePlan, | ||
| customLimits: project.customLimits, | ||
| }); | ||
|
|
||
| if (limits.maxCollections !== -1 && incoming.length > limits.maxCollections) { | ||
| return next( | ||
| new AppError( | ||
| 403, | ||
| `Schema sync would create ${incoming.length} collections, but your plan allows ${limits.maxCollections}. Please upgrade your plan.`, | ||
| ), | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| // 4. Build a lookup of existing collections for RLS preservation | ||
| const existingByName = new Map(); | ||
| for (const col of project.collections || []) { | ||
| const plain = toPlainObject(col); | ||
| existingByName.set(plain.name, plain); | ||
| } | ||
|
|
||
| // 5. Merge: sanitize fields, preserve RLS, validate users contract | ||
| const merged = []; | ||
| for (const entry of incoming) { | ||
| const sanitizedModel = sanitizeSchemaFields(entry.model || []); | ||
|
|
||
| // Enforce users schema contract | ||
| if (entry.name.toLowerCase() === "users") { | ||
| if (!validateUsersSchema(sanitizedModel)) { | ||
| return next( | ||
| new AppError( | ||
| 422, | ||
| "The 'users' collection must have required 'email' and 'password' String fields.", | ||
| ), | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| const existing = existingByName.get(entry.name); | ||
|
|
||
| merged.push({ | ||
| name: entry.name, | ||
| model: sanitizedModel, | ||
| rls: existing?.rls || getDefaultRlsForCollection(entry.name, sanitizedModel), | ||
| }); | ||
| } | ||
|
|
||
| // 6. Atomic update β single $set, no partial states | ||
| const updated = await Project.findOneAndUpdate( | ||
| { _id: projectId, ...getProjectAccessQuery(req.user._id) }, | ||
| { $set: { collections: merged } }, | ||
| { new: true }, | ||
| ); | ||
|
|
||
| if (!updated) { | ||
| return next(new AppError(404, "Project not found or access denied")); | ||
| } | ||
|
|
||
| // 7. Invalidate caches so subsequent reads are fresh | ||
| await deleteProjectById(projectId); | ||
| await setProjectById(projectId, updated.toObject()); | ||
| await deleteProjectByApiKeyCache(updated.publishableKey); | ||
| await deleteProjectByApiKeyCache(updated.secretKey); | ||
|
|
||
| // 8. Respond | ||
| return new ApiResponse( | ||
| { | ||
| synced: merged.length, | ||
| collections: merged.map((c) => c.name), | ||
| }, | ||
| `Successfully synced ${merged.length} collection schema(s).`, | ||
| ).send(res); | ||
| } catch (err) { | ||
| if (err instanceof z.ZodError) { | ||
| return next(new AppError(400, err.issues[0]?.message || "Invalid schema payload")); | ||
| } | ||
| next(err); | ||
| } | ||
| }; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| import { loadWorkspaceConfig, getLocalSchemas } from "../../core/workspace.js"; | ||
| import { syncSchema } from "../../services/project.service.js"; | ||
| import { getToken } from "../../core/config.js"; | ||
| import { APIError } from "../../core/errors.js"; | ||
| import { logger } from "../../core/logger.js"; | ||
| import { label } from "../../utils/format.js"; | ||
|
|
||
| export async function pushCommand(): Promise<void> { | ||
| const token = getToken(); | ||
| if (!token) { | ||
| logger.error("You are not logged in. Run 'ub login' first."); | ||
| process.exitCode = 1; | ||
| return; | ||
| } | ||
|
|
||
| const workspaceConfig = loadWorkspaceConfig(); | ||
| if (!workspaceConfig || !workspaceConfig.projectId) { | ||
| logger.error( | ||
| "No project linked to this directory. Run 'ub init' to link a project first." | ||
| ); | ||
| process.exitCode = 1; | ||
| return; | ||
| } | ||
|
|
||
| const { projectId, projectName } = workspaceConfig; | ||
|
|
||
| // Read all .json files from .ub/schemas/ | ||
| const localSchemas = getLocalSchemas(); | ||
|
|
||
| if (localSchemas.length === 0) { | ||
| logger.error( | ||
| "No schema files found in .ub/schemas/. Run 'ub pull' first to fetch schemas, or create them manually." | ||
| ); | ||
| process.exitCode = 1; | ||
| return; | ||
| } | ||
|
|
||
| // Build the payload from local schema files | ||
| const collections = localSchemas.map((entry) => ({ | ||
| name: entry.schema.name ?? entry.name, | ||
| model: entry.schema.model ?? [], | ||
| })); | ||
|
|
||
| logger.info( | ||
| `Pushing ${collections.length} schema(s) to ${projectName ? projectName + " " : ""}(${projectId})...` | ||
| ); | ||
|
|
||
| try { | ||
| const result = await syncSchema(projectId, collections); | ||
|
|
||
| logger.success( | ||
| `Successfully synced ${result.synced} collection schema(s) to remote.` | ||
| ); | ||
| console.log(); | ||
| for (const name of result.collections) { | ||
| console.log(` ${label("synced")} ${name}`); | ||
| } | ||
| console.log( | ||
| `\nRemote project is now up to date with your local schemas.` | ||
| ); | ||
| } catch (error) { | ||
| if (error instanceof APIError) { | ||
| if (error.status === 401) { | ||
| logger.error( | ||
| "Token is invalid or expired. Run 'ub login' to re-authenticate." | ||
| ); | ||
| } else if (error.status === 403) { | ||
| logger.error(error.message || "You do not have permission to sync schemas for this project."); | ||
| } else if (error.status === 400 || error.status === 422) { | ||
| logger.error(`Schema validation failed: ${error.message}`); | ||
| } else { | ||
| logger.error(error.message); | ||
| } | ||
| process.exitCode = 1; | ||
| return; | ||
| } | ||
| const errorMessage = error instanceof Error ? error.message : String(error); | ||
| logger.error(`Unable to connect to the urBackend API. Error: ${errorMessage}`); | ||
| process.exitCode = 1; | ||
| } | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.