Add JavaScript source map symbolication

docs/docs/source-maps.md

@@ -0,0 +1,31 @@
+---
+title: "JavaScript Source Maps"
+---
+
+# JavaScript Source Maps
+
+Exceptionless uses source maps to turn minified JavaScript stack frames into the original file names, line and column numbers, and function names. Symbolication happens before an event is assigned to a stack, so readable function names also improve stack grouping.
+
+## Automatic discovery
+
+No domain allowlist or project setup is required for public source maps. When an error contains an absolute HTTPS JavaScript URL, Exceptionless checks the generated file's `SourceMap` or `X-SourceMap` response header and its `sourceMappingURL` comment. If neither is present, it also checks the conventional `<generated-file>.map` URL.
+
+Downloaded maps are validated and cached in project-scoped file storage. Automatically downloaded maps are revalidated after one hour so a stable generated-file URL cannot retain a map from an older deployment indefinitely. If refresh fails, Exceptionless leaves the generated frame unchanged instead of risking a misleading stack trace from the stale map.
+
+Exceptionless only makes anonymous HTTPS requests to public network addresses. Redirects are revalidated, and downloads have time, redirect, size, concurrency, and per-project rate limits. Parsed maps use a bounded in-memory cache. Self-hosted installations can tune these safeguards under the `SourceMaps` configuration section, including `AutoDownloadRefreshIntervalMinutes`, `ParsedSourceMapCacheLifetimeMinutes`, and `MaximumParsedSourceMapCacheSize`.
+
+## Uploading a source map
+
+Upload a map when it is private or is not deployed next to the generated JavaScript:
+
+1. Open the project and select **Source Maps** under **Project Settings**.
+2. Enter the exact absolute URL that appears in the generated stack frame, including any path or query string used to identify the build.
+3. Select the corresponding source map and upload it.
+
+Uploading another map for the same generated file URL replaces the previous map. Uploaded and automatically discovered maps appear together on the Source Maps page and can be deleted there.
+
+Source maps must use the version 3 flat-map format. Indexed source maps with a `sections` property and authenticated automatic downloads are planned follow-up capabilities; private maps can be uploaded in the meantime.
+
+## Deployment guidance
+
+Generate source maps as part of the same build that produces the minified JavaScript. Content-hashed generated file names are preferred because the generated URL then identifies a specific build. You can publish the `.map` file for zero-configuration discovery or keep it private and upload it to Exceptionless during deployment.

src/Exceptionless.Core/Bootstrapper.cs

@@ -25,6 +25,7 @@
 using Exceptionless.Core.Seed;
 using Exceptionless.Core.Serialization;
 using Exceptionless.Core.Services;
+using Exceptionless.Core.Services.SourceMaps;
 using Exceptionless.Core.Utility;
 using Exceptionless.Core.Validation;
 using Foundatio.Caching;
@@ -193,6 +194,16 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
                 AllowAutoRedirect = false,
                 ConnectCallback = ConnectToPublicAddressAsync
             });
+        services.AddHttpClient(SourceMapService.HttpClientName)
+            .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler
+            {
+                AllowAutoRedirect = false,
+                AutomaticDecompression = DecompressionMethods.All,
+                ConnectCallback = ConnectToPublicAddressAsync,
+                UseCookies = false,
+                UseProxy = false
+            });
+        services.AddSingleton<SourceMapService>();
         services.AddSingleton<OAuthService>();
         services.AddSingleton<UsageService>();
         services.AddSingleton<SlackService>();
@@ -223,7 +234,7 @@ private static async ValueTask<Stream> ConnectToPublicAddressAsync(SocketsHttpCo
             }
         }
 
-        throw new HttpRequestException($"OAuth client metadata host '{context.DnsEndPoint.Host}' did not resolve to a reachable public address.", lastException);
+        throw new HttpRequestException($"Host '{context.DnsEndPoint.Host}' did not resolve to a reachable public address.", lastException);
     }
 
     public static void LogConfiguration(IServiceProvider serviceProvider, AppOptions appOptions, ILogger logger)

src/Exceptionless.Core/Configuration/AppOptions.cs

@@ -80,6 +80,7 @@ public class AppOptions
     public StripeOptions StripeOptions { get; internal set; } = null!;
     public AuthOptions AuthOptions { get; internal set; } = null!;
     public OAuthServerOptions OAuthServerOptions { get; internal set; } = null!;
+    public SourceMapOptions SourceMapOptions { get; internal set; } = null!;
 
     public static AppOptions ReadFromConfiguration(IConfiguration config)
     {
@@ -133,6 +134,7 @@ public static AppOptions ReadFromConfiguration(IConfiguration config)
         options.StripeOptions = StripeOptions.ReadFromConfiguration(config);
         options.AuthOptions = AuthOptions.ReadFromConfiguration(config);
         options.OAuthServerOptions = OAuthServerOptions.ReadFromConfiguration(config);
+        options.SourceMapOptions = SourceMapOptions.ReadFromConfiguration(config);
 
         return options;
     }

src/Exceptionless.Core/Configuration/SourceMapOptions.cs

@@ -0,0 +1,49 @@
+using Microsoft.Extensions.Configuration;
+
+namespace Exceptionless.Core.Configuration;
+
+public sealed class SourceMapOptions
+{
+    public bool EnableAutoDownload { get; internal set; }
+    public int RequestTimeoutMilliseconds { get; internal set; }
+    public int MaximumGeneratedFileSize { get; internal set; }
+    public int MaximumSourceMapSize { get; internal set; }
+    public int MaximumMappingSegments { get; internal set; }
+    public int MaximumRedirects { get; internal set; }
+    public int MaximumConcurrentDownloads { get; internal set; }
+    public int MaximumAutoDownloadsPerProjectPerHour { get; internal set; }
+    public int MaximumFramesPerError { get; internal set; }
+    public int MaximumProcessingTimeMilliseconds { get; internal set; }
+    public int AutoDownloadRefreshIntervalMinutes { get; internal set; }
+    public int ParsedSourceMapCacheLifetimeMinutes { get; internal set; }
+    public long MaximumParsedSourceMapCacheSize { get; internal set; }
+
+    public TimeSpan RequestTimeout => TimeSpan.FromMilliseconds(RequestTimeoutMilliseconds);
+    public TimeSpan MaximumProcessingTime => TimeSpan.FromMilliseconds(MaximumProcessingTimeMilliseconds);
+    public TimeSpan AutoDownloadRefreshInterval => TimeSpan.FromMinutes(AutoDownloadRefreshIntervalMinutes);
+    public TimeSpan ParsedSourceMapCacheLifetime => TimeSpan.FromMinutes(ParsedSourceMapCacheLifetimeMinutes);
+
+    public static SourceMapOptions ReadFromConfiguration(IConfiguration configuration)
+    {
+        var section = configuration.GetSection("SourceMaps");
+        return new SourceMapOptions
+        {
+            EnableAutoDownload = section.GetValue(nameof(EnableAutoDownload), true),
+            RequestTimeoutMilliseconds = ReadPositive(section, nameof(RequestTimeoutMilliseconds), 3000),
+            MaximumGeneratedFileSize = ReadPositive(section, nameof(MaximumGeneratedFileSize), 5 * 1024 * 1024),
+            MaximumSourceMapSize = ReadPositive(section, nameof(MaximumSourceMapSize), 20 * 1024 * 1024),
+            MaximumMappingSegments = ReadPositive(section, nameof(MaximumMappingSegments), 1_000_000),
+            MaximumRedirects = Math.Max(0, section.GetValue(nameof(MaximumRedirects), 3)),
+            MaximumConcurrentDownloads = ReadPositive(section, nameof(MaximumConcurrentDownloads), 4),
+            MaximumAutoDownloadsPerProjectPerHour = Math.Max(0, section.GetValue(nameof(MaximumAutoDownloadsPerProjectPerHour), 100)),
+            MaximumFramesPerError = ReadPositive(section, nameof(MaximumFramesPerError), 100),
+            MaximumProcessingTimeMilliseconds = ReadPositive(section, nameof(MaximumProcessingTimeMilliseconds), 5000),
+            AutoDownloadRefreshIntervalMinutes = ReadPositive(section, nameof(AutoDownloadRefreshIntervalMinutes), 60),
+            ParsedSourceMapCacheLifetimeMinutes = ReadPositive(section, nameof(ParsedSourceMapCacheLifetimeMinutes), 5),
+            MaximumParsedSourceMapCacheSize = Math.Max(1, section.GetValue(nameof(MaximumParsedSourceMapCacheSize), 100L * 1024 * 1024))
+        };
+    }
+
+    private static int ReadPositive(IConfiguration section, string name, int defaultValue)
+        => Math.Max(1, section.GetValue(name, defaultValue));
+}

src/Exceptionless.Core/Exceptionless.Core.csproj

@@ -31,6 +31,7 @@
     <PackageReference Include="MiniValidation" Version="0.10.0" />
     <PackageReference Include="Handlebars.Net" Version="2.1.6" />
     <PackageReference Include="McSherry.SemanticVersioning" Version="1.5.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="10.0.9" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.9" />
     <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="10.0.9" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="10.0.9" />

src/Exceptionless.Core/Jobs/CleanupDataJob.cs

@@ -341,6 +341,7 @@ private async Task RemoveProjectsAsync(Project project, JobContext context)
         await RenewLockAsync(context);
         long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(project.OrganizationId, project.Id);
 
+        await _fileStorage.DeleteFilesAsync($"source-maps/{project.Id}/*", context.CancellationToken);
         await _projectRepository.RemoveAsync(project);
         _logger.RemoveProjectComplete(project.Name, project.Id, removedStacks, removedEvents);
     }

src/Exceptionless.Core/Plugins/EventProcessor/Default/15_SourceMapPlugin.cs

@@ -0,0 +1,36 @@
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Pipeline;
+using Exceptionless.Core.Services.SourceMaps;
+using Foundatio.Serializer;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Plugins.EventProcessor.Default;
+
+[Priority(15)]
+public sealed class SourceMapPlugin : EventProcessorPluginBase
+{
+    private readonly SourceMapService _sourceMapService;
+    private readonly ITextSerializer _serializer;
+
+    public SourceMapPlugin(SourceMapService sourceMapService, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory)
+        : base(options, loggerFactory)
+    {
+        _sourceMapService = sourceMapService;
+        _serializer = serializer;
+        ContinueOnError = true;
+    }
+
+    public override async Task EventProcessingAsync(EventContext context)
+    {
+        if (!context.Event.IsError())
+            return;
+
+        var error = context.Event.GetError(_serializer, _logger);
+        if (error is null)
+            return;
+
+        if (await _sourceMapService.SymbolicateAsync(context.Project.Id, error))
+            context.Event.SetError(error);
+    }
+}

src/Exceptionless.Core/Services/SourceMaps/SourceMapArtifact.cs

@@ -0,0 +1,12 @@
+namespace Exceptionless.Core.Services.SourceMaps;
+
+public sealed record SourceMapArtifact
+{
+    public required string Id { get; init; }
+    public required string GeneratedFileUrl { get; init; }
+    public string? SourceMapUrl { get; init; }
+    public string? FileName { get; init; }
+    public required long Size { get; init; }
+    public required bool IsAutoDownloaded { get; init; }
+    public required DateTime CreatedUtc { get; init; }
+}

src/Exceptionless.Core/Services/SourceMaps/SourceMapContent.cs

@@ -0,0 +1,20 @@
+namespace Exceptionless.Core.Services.SourceMaps;
+
+internal static class SourceMapContent
+{
+    public static async Task<byte[]> ReadLimitedAsync(Stream stream, int maximumBytes, CancellationToken cancellationToken)
+    {
+        using var memoryStream = new MemoryStream(Math.Min(maximumBytes, 64 * 1024));
+        byte[] buffer = new byte[81920];
+        int read;
+        while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0)
+        {
+            if (memoryStream.Length + read > maximumBytes)
+                throw new InvalidOperationException("The file exceeded the configured maximum size.");
+
+            await memoryStream.WriteAsync(buffer.AsMemory(0, read), cancellationToken);
+        }
+
+        return memoryStream.ToArray();
+    }
+}