Feature | CMG-703 | stack-to-stack migration with audit report

.gitignore

@@ -372,3 +372,4 @@ app.json
 
 # Test coverage (global)
 coverage/
+export-stack/

api/src/constants/index.ts

@@ -38,13 +38,14 @@ export const regionalApiHosts = {
   GCP_EU: 'gcp-eu-api.contentstack.com',
 };
 export const CMS = {
-  CONTENTFUL: 'contentful',
-  SITECORE_V8: 'sitecore v8',
-  SITECORE_V9: 'sitecore v9',
-  SITECORE_V10: 'sitecore v10',
-  WORDPRESS: 'wordpress',
-  DRUPAL: 'drupal',
-  AEM: 'aem',
+  CONTENTSTACK: "contentstack",
+  CONTENTFUL: "contentful",
+  SITECORE_V8: "sitecore v8",
+  SITECORE_V9: "sitecore v9",
+  SITECORE_V10: "sitecore v10",
+  WORDPRESS: "wordpress",
+  DRUPAL: "drupal",
+  AEM: "aem",
 };
 export const MODULES = [
   'Project',
@@ -127,7 +128,7 @@ export const HTTP_TEXTS = {
   CONTENTMAPPER_NOT_FOUND:
     'Sorry, the requested content mapper id does not exists.',
   ADMIN_LOGIN_ERROR:
-    "Sorry, You Don't have admin access in any of the Organisation",
+    'You are not a member of any Contentstack organization in this region (or organization list is empty).',
   PROJECT_DELETE: 'Project Deleted Successfully',
   PROJECT_REVERT: 'Project Reverted Successfully',
   LOGS_NOT_FOUND: 'Sorry, no logs found for requested stack migration.',
@@ -175,9 +176,10 @@ export const PROJECT_STATUS = {
 export const STEPPER_STEPS: any = {
   LEGACY_CMS: 1,
   DESTINATION_STACK: 2,
-  CONTENT_MAPPING: 3,
-  TESTING: 4,
-  MIGRATION: 5,
+  AUDIT_REPORT: 3,
+  CONTENT_MAPPING: 4,
+  TESTING: 5,
+  MIGRATION: 6,
 };
 export const PREDEFINED_STATUS = [
   'Draft',
@@ -186,7 +188,7 @@ export const PREDEFINED_STATUS = [
   'Failed',
   'Success',
 ];
-export const PREDEFINED_STEPS = [1, 2, 3, 4, 5];
+export const PREDEFINED_STEPS = [1, 2, 3, 4, 5, 6];
 
 export const NEW_PROJECT_STATUS = {
   0: 0, //DRAFT
@@ -314,6 +316,11 @@ export const MIGRATION_DATA_CONFIG = {
 
   AEM_DAM_DIR: 'dam-downloads',
 };
+
+export const DB_CONFIG = {
+  AUDIT_THRESHOLD: 250,
+  AUDIT_DIR: "database/audit",
+};
 export const GET_AUDIT_DATA = {
   MIGRATION: 'migration-v2',
   API_DIR: 'api',

api/src/controllers/migration.controller.ts

@@ -16,6 +16,26 @@ const getAuditData = async (req: Request, res: Response): Promise<void> => {
   const resp = await migrationService.getAuditData(req);
   res.status(resp?.status).json(resp);
 };
+
+const exportSourceStack = async (req: Request, res: Response): Promise<void> => {
+  const resp = await migrationService.exportSourceStack(req);
+  res.status(resp?.status).json(resp);
+};
+
+const validateSourceExport = async (req: Request, res: Response): Promise<void> => {
+  const resp = await migrationService.validateSourceExport(req);
+  res.status(resp?.status).json(resp);
+};
+
+const runSourceAudit = async (req: Request, res: Response): Promise<void> => {
+  const resp = await migrationService.runSourceAudit(req);
+  res.status(resp?.status).json(resp);
+};
+
+const getSourceAuditSummary = async (req: Request, res: Response): Promise<void> => {
+  const resp = await migrationService.getSourceAuditSummary(req);
+  res.status(resp?.status).json(resp);
+};
 /**
  * Start Test Migartion.
  *
@@ -24,8 +44,8 @@ const getAuditData = async (req: Request, res: Response): Promise<void> => {
  * @returns {Promise<void>} - A Promise that resolves when the stack is deleted.
  */
 const startTestMigration = async (req: Request, res: Response): Promise<void> => {
-  const resp = migrationService.startTestMigration(req);
-  res.status(200).json(resp);
+  const resp = await migrationService.startTestMigration(req);
+  res.status(resp?.status ?? 200).json(resp);
 };
 
 
@@ -37,8 +57,8 @@ const startTestMigration = async (req: Request, res: Response): Promise<void> =>
  * @returns {Promise<void>} - A Promise that resolves when the stack is deleted.
  */
 const startMigration = async (req: Request, res: Response): Promise<void> => {
-  const resp = migrationService.startMigration(req);
-  res.status(200).json(resp);
+  const resp = await migrationService.startMigration(req);
+  res.status(resp?.status ?? 200).json(resp);
 };
 
 /**
@@ -70,6 +90,10 @@ const saveMappedLocales = async (req: Request, res: Response): Promise<void> =>
 
 export const migrationController = {
   createTestStack,
+  exportSourceStack,
+  validateSourceExport,
+  runSourceAudit,
+  getSourceAuditSummary,
   deleteTestStack,
   startTestMigration,
   startMigration,

api/src/controllers/projects.controller.ts

@@ -96,6 +96,11 @@ const updateFileFormat = async (req: Request, res: Response) => {
   res.status(resp.status).json(resp.data);
 };
 
+const updateSourceConfig = async (req: Request, res: Response) => {
+  const resp = await projectService.updateSourceConfig(req);
+  res.status(resp?.status).json(resp?.data);
+};
+
 /**
  * Handles the file format confirmation request.
  *
@@ -176,6 +181,14 @@ const getMigratedStacks = async (req: Request, res: Response): Promise<void> =>
   res.status(project.status).json(project);
 }
 
+/**
+ * Updates audit report selections for a project
+ */
+const updateAuditSelections = async (req: Request, res: Response): Promise<void> => {
+  const project = await projectService.updateAuditSelections(req);
+  res.status(200).json({ data: project, message: 'Audit selections updated successfully' });
+};
+
 export const projectController = {
   getAllProjects,
   getProject,
@@ -185,6 +198,7 @@ export const projectController = {
   updateAffix,
   affixConfirmation,
   updateFileFormat,
+  updateSourceConfig,
   fileformatConfirmation,
   updateDestinationStack,
   updateCurrentStep,
@@ -193,4 +207,5 @@ export const projectController = {
   updateStackDetails,
   updateMigrationExecution,
   getMigratedStacks,
+  updateAuditSelections,
 };

api/src/models/audit-lowdb.ts

@@ -0,0 +1,208 @@
+import path from 'path';
+import fs from 'fs';
+import { JSONFile } from 'lowdb/node';
+import LowWithLodash from '../utils/lowdb-lodash.utils.js';
+import { DB_CONFIG } from '../constants/index.js';
+
+type AuditType = 'assets' | 'content_types' | 'entries' | 'global_fields';
+
+type FlowIndex = {
+  project_id: string;
+  org_id: string;
+  stack_id: string;
+  created_at: string;
+  updated_at: string;
+  data_counts: Record<AuditType, number>;
+};
+
+type AuditIndexDocument = {
+  projects: FlowIndex[];
+};
+
+type DataTypeIndex = {
+  [key: string]: {
+    file: string;
+    uids: string[];
+  };
+};
+
+const DATA_TYPES: AuditType[] = [
+  'assets',
+  'content_types',
+  'entries',
+  'global_fields',
+];
+
+const AUDIT_DIR = path.join(process.cwd(), DB_CONFIG.AUDIT_DIR);
+const THRESHOLD = DB_CONFIG.AUDIT_THRESHOLD;
+
+const defaultData: AuditIndexDocument = { projects: [] };
+
+const auditIndexDb = new LowWithLodash(
+  new JSONFile<AuditIndexDocument>(path.join(AUDIT_DIR, 'audit-index.json')),
+  defaultData
+);
+
+const ensureProjectDir = (projectId: string) => {
+  const projectDir = path.join(AUDIT_DIR, projectId);
+  fs.mkdirSync(projectDir, { recursive: true });
+  for (const dataType of DATA_TYPES) {
+    const dataTypeDir = path.join(projectDir, dataType);
+    fs.mkdirSync(dataTypeDir, { recursive: true });
+    const indexPath = path.join(dataTypeDir, 'index.json');
+    if (!fs.existsSync(indexPath)) {
+      fs.writeFileSync(indexPath, JSON.stringify({}, null, 2));
+    }
+  }
+};
+
+const getUid = (item: any): string => {
+  if (typeof item === 'string') return item;
+  if (item?.uid) return String(item.uid);
+  if (item?.entryUid) return String(item.entryUid);
+  if (item?.id) return String(item.id);
+  return JSON.stringify(item);
+};
+
+const readTypeIndex = async (
+  projectId: string,
+  dataType: AuditType
+): Promise<DataTypeIndex> => {
+  const indexPath = path.join(AUDIT_DIR, projectId, dataType, 'index.json');
+  if (!fs.existsSync(indexPath)) return {};
+  const raw = await fs.promises.readFile(indexPath, 'utf8');
+  return JSON.parse(raw || '{}') as DataTypeIndex;
+};
+
+const writeTypeIndex = async (
+  projectId: string,
+  dataType: AuditType,
+  index: DataTypeIndex
+) => {
+  const indexPath = path.join(AUDIT_DIR, projectId, dataType, 'index.json');
+  await fs.promises.writeFile(indexPath, JSON.stringify(index, null, 2));
+};
+
+const writeSharded = async (
+  projectId: string,
+  dataType: AuditType,
+  items: any[]
+) => {
+  ensureProjectDir(projectId);
+  const typeDir = path.join(AUDIT_DIR, projectId, dataType);
+  const existing = await readTypeIndex(projectId, dataType);
+
+  for (const entry of Object.values(existing)) {
+    const fp = path.join(typeDir, entry?.file);
+    if (fs.existsSync(fp)) fs.rmSync(fp, { force: true });
+  }
+
+  const nextIndex: DataTypeIndex = {};
+  let shard = 0;
+  for (let i = 0; i < items.length; i += THRESHOLD) {
+    const chunk = items.slice(i, i + THRESHOLD);
+    const file = `${dataType}-${shard + 1}.json`;
+    await fs.promises.writeFile(
+      path.join(typeDir, file),
+      JSON.stringify(chunk, null, 2)
+    );
+    nextIndex[String(shard)] = {
+      file,
+      uids: chunk.map((item) => getUid(item)),
+    };
+    shard += 1;
+  }
+  await writeTypeIndex(projectId, dataType, nextIndex);
+  return items?.length;
+};
+
+const upsertAudit = async ({
+  project_id,
+  org_id,
+  stack_id,
+  assets = [],
+  content_types = [],
+  entries = [],
+  global_fields = [],
+}: {
+  project_id: string;
+  org_id: string;
+  stack_id: string;
+  assets?: any[];
+  content_types?: any[];
+  entries?: any[];
+  global_fields?: any[];
+}) => {
+  await auditIndexDb.read();
+  auditIndexDb.data ||= defaultData;
+
+  let project = auditIndexDb?.data?.projects?.find((p) => p?.project_id === project_id);
+  if (!project) {
+    project = {
+      project_id,
+      org_id,
+      stack_id,
+      created_at: new Date().toISOString(),
+      updated_at: new Date().toISOString(),
+      data_counts: {
+        assets: 0,
+        content_types: 0,
+        entries: 0,
+        global_fields: 0,
+      },
+    };
+    auditIndexDb?.data?.projects?.push(project);
+  }
+
+  project.org_id = org_id || project?.org_id;
+  project.stack_id = stack_id || project?.stack_id;
+  project.updated_at = new Date().toISOString();
+  project.data_counts.assets = await writeSharded(project_id, 'assets', assets);
+  project.data_counts.content_types = await writeSharded(
+    project_id,
+    'content_types',
+    content_types
+  );
+  project.data_counts.entries = await writeSharded(project_id, 'entries', entries);
+  project.data_counts.global_fields = await writeSharded(
+    project_id,
+    'global_fields',
+    global_fields
+  );
+
+  await auditIndexDb.write();
+  return project;
+};
+
+const getDataByType = async (projectId: string, dataType: AuditType) => {
+  const index = await readTypeIndex(projectId, dataType);
+  const allData: any[] = [];
+  for (const entry of Object.values(index)) {
+    const fp = path.join(AUDIT_DIR, projectId, dataType, entry?.file);
+    if (!fs.existsSync(fp)) continue;
+    const raw = await fs.promises.readFile(fp, 'utf8');
+    const parsed = JSON.parse(raw || '[]');
+    if (Array.isArray(parsed)) allData.push(...parsed);
+  }
+  return allData;
+};
+
+const getAuditByProjectId = async (projectId: string) => {
+  await auditIndexDb.read();
+  auditIndexDb.data ||= defaultData;
+  const metadata =
+    auditIndexDb?.data?.projects?.find((p) => p?.project_id === projectId) || null;
+  if (!metadata) return null;
+  const assets = await getDataByType(projectId, 'assets');
+  const content_types = await getDataByType(projectId, 'content_types');
+  const entries = await getDataByType(projectId, 'entries');
+  const global_fields = await getDataByType(projectId, 'global_fields');
+  return { metadata, assets, content_types, entries, global_fields };
+};
+
+export default {
+  upsertAudit,
+  getDataByType,
+  getAuditByProjectId,
+};
+