Add a program which verifies that an identity has been correctly deleted

.github/workflows/test.yml

@@ -173,7 +173,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        database: [sqlserver, postgres]
+        database: [ sqlserver, postgres ]
         test-project:
           - display-name: admin-api
             path: Applications/AdminApi/test/AdminApi.Tests.Integration
@@ -255,7 +255,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        database: [sqlserver, postgres]
+        database: [ sqlserver, postgres ]
     needs: image-test-builds
     steps:
       - name: Checkout backbone repository
@@ -357,3 +357,99 @@ jobs:
         run: dotnet restore /p:ContinuousIntegrationBuild=true ./Backbone.sln
       - name: Validate Licenses
         run: nuget-license --input ./Backbone.sln --allowed-license-types ./.ci/allowedLicenses.json --ignored-packages ./.ci/ignoredPackages.json --output table --error-only
+
+  verify-identity-deletion:
+    name: Validate Identity deletion (on ${{matrix.database.type}})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        database:
+          - type: postgres
+            env:
+              Database__Provider: Postgres
+              Database__ConnectionString: "Server=postgres;Database=enmeshed;User Id=devices;Password=Passw0rd;Port=5432"
+          - type: sqlserver
+            env:
+              Database__Provider: SqlServer
+              Database__ConnectionString: "Server=sqlserver;Database=enmeshed;User Id=devices;Password=Passw0rd;TrustServerCertificate=True"
+    needs: image-test-builds
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup Dotnet
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 9.x
+
+      - name: Cache NuGet packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.nuget/packages
+          key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }}
+          restore-keys: ${{ runner.os }}-nuget-
+
+      - name: Download cached Docker images
+        uses: actions/download-artifact@v5
+        with:
+          path: /tmp
+          pattern: docker-*
+          merge-multiple: true
+
+      - name: Load Docker images and build applications
+        run: |
+          {
+            ./.ci/loadDockerImages.sh
+          } &
+          {
+            # The following two lines are for the identity deletion job only
+            mv appsettings.override.json appsettings.override.json.bak
+            cp .ci/appsettings.override.${{matrix.database.type}}.local.json appsettings.override.json
+
+            dotnet restore ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj
+            dotnet restore ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj
+            dotnet restore ./Applications/AdminCli/src/AdminCli/AdminCli.csproj
+
+            dotnet build --no-restore ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj
+            dotnet build --no-restore ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj
+            dotnet build --no-restore ./Applications/AdminCli/src/AdminCli/AdminCli.csproj
+          }
+          wait
+
+      - name: Start compose stack
+        run: |
+          docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml down -v
+          docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml up --no-build --wait -d
+          docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml wait admin-cli
+
+      - name: Create identities, relationships etc. and initiate deletion processes
+        run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj init --consumerBaseUrl http://localhost:5000 --adminBaseUrl http://localhost:5173
+
+      - name: Run Identity deletion job
+        run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionJobs/src/Job.IdentityDeletion/Job.IdentityDeletion.csproj --Worker ActualDeletionWorker
+
+      - name: Export databse via Admin Cli
+        run: dotnet run --no-build --no-restore --project ./Applications/AdminCli/src/AdminCli/AdminCli.csproj database export --sensitive
+        env: ${{matrix.database.env}}
+
+      - name: Check exported database file
+        run: dotnet run --no-build --no-restore --project ./Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/IdentityDeletionVerifier.csproj check
+
+      - name: Save Docker Logs
+        if: failure()
+        run: docker compose -f ./.ci/compose.test.yml -f ./.ci/compose.test.${{matrix.database.type}}.yml logs > logs.txt
+
+      - name: Archive logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: identity-deletion-verifier-docker-${{matrix.database.type}}
+          path: logs.txt
+
+      - name: Archive exported database and identity files
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: identity-deletion-verifier-exported-db-${{matrix.database.type}}
+          path: /tmp/enmeshed/backbone

.run/Identity Deletion verifier.run.xml

@@ -0,0 +1,55 @@
+<component name="ProjectRunConfigurationManager">
+    <configuration default="false" name="Identity Deletion verifier" type="com.intellij.execution.configurations.multilaunch" factoryName="MultiLaunchConfiguration">
+        <rows>
+            <ExecutableRowSnapshot>
+                <option name="condition">
+                    <ConditionSnapshot>
+                        <option name="type" value="immediately" />
+                    </ConditionSnapshot>
+                </option>
+                <option name="executable">
+                    <ExecutableSnapshot>
+                        <option name="id" value="runConfig:.NET Launch Settings Profile.IdentityDeletionVerifier: Init" />
+                    </ExecutableSnapshot>
+                </option>
+            </ExecutableRowSnapshot>
+            <ExecutableRowSnapshot>
+                <option name="condition">
+                    <ConditionSnapshot>
+                        <option name="type" value="afterPreviousFinished" />
+                    </ConditionSnapshot>
+                </option>
+                <option name="executable">
+                    <ExecutableSnapshot>
+                        <option name="id" value="runConfig:.NET Launch Settings Profile.Job.IdentityDeletion: Job.IdentityDeletion.ActualDeletion" />
+                    </ExecutableSnapshot>
+                </option>
+            </ExecutableRowSnapshot>
+            <ExecutableRowSnapshot>
+                <option name="condition">
+                    <ConditionSnapshot>
+                        <option name="type" value="afterPreviousFinished" />
+                    </ConditionSnapshot>
+                </option>
+                <option name="executable">
+                    <ExecutableSnapshot>
+                        <option name="id" value="runConfig:.NET Launch Settings Profile.AdminCli: Export database" />
+                    </ExecutableSnapshot>
+                </option>
+            </ExecutableRowSnapshot>
+            <ExecutableRowSnapshot>
+                <option name="condition">
+                    <ConditionSnapshot>
+                        <option name="type" value="afterPreviousFinished" />
+                    </ConditionSnapshot>
+                </option>
+                <option name="executable">
+                    <ExecutableSnapshot>
+                        <option name="id" value="runConfig:.NET Launch Settings Profile.IdentityDeletionVerifier: Check" />
+                    </ExecutableSnapshot>
+                </option>
+            </ExecutableRowSnapshot>
+        </rows>
+        <method v="2" />
+    </configuration>
+</component>

Applications/IdentityDeletionVerifier/src/IdentityDeletionVerifier/Commands/CheckCommand.cs

@@ -0,0 +1,117 @@
+using System.CommandLine;
+using System.IO.Compression;
+using System.Text;
+using Spectre.Console;
+
+namespace Backbone.IdentityDeletionVerifier.Commands;
+
+public class CheckCommand : Command
+{
+    public CheckCommand() : base("check", "Check the exported database file for the given identity address in the temp directory")
+    {
+        SetAction(Handle);
+    }
+
+    private async Task<int> Handle(ParseResult _, CancellationToken cancellationToken)
+    {
+        if (!DirectoryExists())
+        {
+            AnsiConsole.MarkupLineInterpolated($"[red bold]The temp directory[/][grey bold]{FilePaths.PATH_TO_TEMP_DIR} [/][red bold]doesn't exist.[/]");
+            return -1;
+        }
+
+        if (!IdentitiesFileExists())
+        {
+            AnsiConsole.MarkupLine("[red bold]The deleted identities file doesn't exist. Run the Init command first.[/]");
+            return -1;
+        }
+
+        if (!ExportFileExists())
+        {
+            AnsiConsole.MarkupLine("[red bold]No exported database file found. Run the Admin Cli Export Database command first.[/]");
+            return -1;
+        }
+
+        var a = await GetIdentityToCheck();
+        if (a == null)
+        {
+            AnsiConsole.MarkupLineInterpolated($"[red bold]The identities file couldn't be read or has no Identity[/]");
+            return -1;
+        }
+
+        AnsiConsole.MarkupLineInterpolated($"[green bold]Identity to check:[/] [grey bold]{a}[/]");
+
+        return await CheckExportFileForIdentities(GetLatestExportFile(), a);
+    }
+
+    private bool DirectoryExists() => Directory.Exists(FilePaths.PATH_TO_TEMP_DIR);
+    private bool IdentitiesFileExists() => File.Exists(FilePaths.PATH_TO_IDENTITIES_FILE);
+    private bool ExportFileExists() => Directory.EnumerateFiles(FilePaths.PATH_TO_TEMP_DIR).Any(FilePaths.EXPORT_FILE_PATTERN.IsMatch);
+    private string GetLatestExportFile() => Directory.EnumerateFiles(FilePaths.PATH_TO_TEMP_DIR).Where(e => FilePaths.EXPORT_FILE_PATTERN.IsMatch(e)).Max()!;
+
+    private async Task<string?> GetIdentityToCheck()
+    {
+        using var reader = new StreamReader(File.OpenRead(FilePaths.PATH_TO_IDENTITIES_FILE), Encoding.UTF8);
+
+        return await reader.ReadLineAsync();
+    }
+
+    private async Task<int> CheckExportFileForIdentities(string exportFile, string identityToCheck)
+    {
+        return await AnsiConsole.Progress()
+            .AutoClear(false)
+            .HideCompleted(false)
+            .Columns(
+                new TaskDescriptionColumn(),
+                new ProgressBarColumn(),
+                new PercentageColumn(),
+                new RemainingTimeColumn(),
+                new SpinnerColumn(Spinner.Known.Clock)
+            )
+            .StartAsync(async ctx =>
+            {
+                using var archive = ZipFile.OpenRead(exportFile);
+
+                var found = 0;
+                var tasks = archive.Entries
+                    .Select(e => ctx.AddTask(e.Name, autoStart: false))
+                    .ToList();
+
+                foreach (var (index, entry) in archive.Entries.Index())
+                {
+                    var task = tasks[index];
+                    task.StartTask();
+                    found += await CheckCsvFileForIdentity(entry, identityToCheck, task);
+                }
+
+                return found;
+            });
+    }
+
+    private async Task<int> CheckCsvFileForIdentity(ZipArchiveEntry file, string identityToCheck, ProgressTask progressReporter)
+    {
+        using var reader = new StreamReader(file.Open(), Encoding.UTF8);
+        List<string> lines = [];
+        var found = 0;
+
+        while (!reader.EndOfStream)
+            lines.Add(await reader.ReadLineAsync() ?? string.Empty);
+
+        progressReporter.MaxValue = lines.Count;
+
+        foreach (var line in lines)
+        {
+            var count = line
+                .Split(',')
+                .Count(s => string.Equals(s, identityToCheck, StringComparison.OrdinalIgnoreCase));
+
+            if (count != 0)
+                AnsiConsole.MarkupLineInterpolated($"[red bold]Found[/] [grey bold]{count}[/] [red bold]occurrences in[/] [grey bold]{file.Name}[/][red bold]:[/] [grey bold]{line}[/]");
+
+            found += count;
+            progressReporter.Increment(1);
+        }
+
+        return found;
+    }
+}