Dynamic GC based on host, API concurrency limiter, logging for 429 and configurable http worker queue timeout

Directory.Build.props

@@ -55,6 +55,14 @@
     <Authors>CyberDrain</Authors>
     <RepositoryUrl>https://github.com/CyberDrain/CRAFT</RepositoryUrl>
     <RepositoryType>git</RepositoryType>
+    <!--
+      Version stamping. Without this the assembly version defaults to 1.0.0, so the CRAFT version
+      reported in startup telemetry would be meaningless. Overridden at build time, e.g.
+      `dotnet build /p:Version=1.4.2+abc123` (CI feeds the real version). The default keeps a real,
+      non-1.0.0 marker for local builds. AssemblyInformationalVersion carries the full string
+      (including +metadata); the emitter reads that.
+    -->
+    <Version Condition="'$(Version)' == ''">0.0.0-dev</Version>
   </PropertyGroup>
 
 </Project>

Services/Bridges/WorkerMetricsBridge.cs

@@ -692,6 +692,27 @@ public static int CancelRun(string runName)
     public static bool DeleteJob(string jobId)
         => s_jobManager?.DeleteJob(jobId) ?? false;
 
+    /// <summary>
+    /// Empty the durable job queue — a maintenance/reset primitive. Returns the number of queue rows
+    /// removed, or -1 if the orchestrator is unavailable or the clear failed. In-flight work is
+    /// unaffected and Pending tasks may be re-driven, so pair with <see cref="CancelRun"/> when the
+    /// intent is to STOP work rather than clear a wedged or corrupted queue.
+    /// PS usage: <c>[Craft.Services.WorkerMetricsBridge]::ClearQueue()</c>.
+    /// </summary>
+    public static int ClearQueue()
+    {
+        if (s_orchestrator is not { } orchestrator) return -1;
+        try
+        {
+            return Task.Run(() => orchestrator.ClearQueueAsync(CancellationToken.None)).GetAwaiter().GetResult();
+        }
+        catch (Exception ex)
+        {
+            s_logger?.LogWarning(ex, "[WorkerMetrics] ClearQueue failed");
+            return -1;
+        }
+    }
+
     /// <summary>
     /// Change a queued job's priority. In the local buffer this re-enqueues at the new priority; for an
     /// unclaimed durable row it moves the row to the new priority bucket (keeping its age) and records

Services/Configuration/CraftSettings.cs

@@ -106,4 +106,7 @@ public class CraftSettings
 
     /// <summary>Realtime SSE channel (<c>/.craft/events</c>). See <see cref="RealtimeSettings"/>.</summary>
     public RealtimeSettings Realtime { get; set; } = new();
+
+    /// <summary>Startup phone-home usage telemetry. Off by default. See <see cref="TelemetrySettings"/>.</summary>
+    public TelemetrySettings Telemetry { get; set; } = new();
 }

Services/Configuration/OrchestratorSettings.cs

@@ -19,6 +19,16 @@ public class OrchestratorSettings
     /// </summary>
     public bool BatchStatusWrites { get; set; } = true;
 
+    /// <summary>
+    /// Coalesce SMALL task results (those that fit one Azure Table property) through the batched status
+    /// writer instead of a per-task upsert on the fan-out critical path. Each result is written BEFORE
+    /// its task's terminal marker in the same flush, so a result is always durable before the task is
+    /// counted done (and therefore before finalize/post-execution reads it). Large results keep the
+    /// directly-awaited chunked path. Default true; only applies when <see cref="BatchStatusWrites"/> is
+    /// also true. Set false to fall back to the original per-task awaited result write.
+    /// </summary>
+    public bool BatchResultWrites { get; set; } = true;
+
     /// <summary>
     /// When batching status writes, write the pre-invoke "Running" marker under a synchronous barrier so it
     /// is durable BEFORE the task invokes (batched with other concurrently-starting tasks). Preserves the

Services/Configuration/RateLimitSettings.cs

@@ -25,8 +25,44 @@ public class RateLimitSettings
     /// </summary>
     public int QueueLimit { get; set; }
 
+    /// <summary>
+    /// Maximum requests a single app-only API client (client-credentials caller) may have occupying
+    /// the HTTP worker system at once — counting both those queued waiting for a runspace and those
+    /// already executing, since the limiter's lease spans the whole downstream pipeline. Keyed per
+    /// client (its AppId), so one automation cannot monopolise the pool and starve the interactive UI,
+    /// which is never limited by this. Over-limit requests are rejected immediately with 429 (no
+    /// concurrency queue); the caller retries on <c>Retry-After</c>.
+    ///
+    /// <para>
+    /// 0 (default) = unlimited: the feature is off until a value is set. Distinct from
+    /// <see cref="PermitPerWindow"/>, which is a request-RATE cap; this is a simultaneous-in-flight cap.
+    /// Interactive (browser) callers are classified as UI and never counted here.
+    /// </para>
+    ///
+    /// Env override: <c>CRAFT_API_CONCURRENCY_LIMIT</c>.
+    /// </summary>
+    public int ApiConcurrencyLimit { get; set; }
+
     /// <summary>Resolved enabled state, honouring the CRAFT_RATELIMIT_ENABLED environment override.</summary>
     public bool IsEnabled =>
         Enabled
         || string.Equals(Environment.GetEnvironmentVariable("CRAFT_RATELIMIT_ENABLED"), "true", StringComparison.OrdinalIgnoreCase);
+
+    /// <summary>
+    /// Resolved per-client API concurrency cap, honouring the <c>CRAFT_API_CONCURRENCY_LIMIT</c>
+    /// environment override (which wins when it parses to a non-negative integer). 0 = unlimited/off.
+    /// </summary>
+    public int ResolvedApiConcurrencyLimit =>
+        int.TryParse(Environment.GetEnvironmentVariable("CRAFT_API_CONCURRENCY_LIMIT"),
+            System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture,
+            out var fromEnv) && fromEnv >= 0
+            ? fromEnv
+            : ApiConcurrencyLimit;
+
+    /// <summary>
+    /// Whether the rate-limiter middleware needs to run at all: either the per-client rate limiter is
+    /// enabled, or an API concurrency cap is configured. When both are off, the middleware is skipped
+    /// entirely (no per-request limiter cost).
+    /// </summary>
+    public bool RequiresLimiterMiddleware => IsEnabled || ResolvedApiConcurrencyLimit > 0;
 }

Services/Configuration/SkuProfile.cs

@@ -32,4 +32,25 @@ public class SkuProfile
 
     /// <summary>Background worker pool size to apply when this profile matches.</summary>
     public int BgPoolSize { get; set; }
+
+    /// <summary>
+    /// Optional GC heap hard limit, in MB, to apply when this profile matches. Three-way, mirroring the
+    /// <c>CRAFT_GC_HEAP_LIMIT_MB</c> override:
+    /// <list type="bullet">
+    /// <item><description>Omitted / null (or negative) = no opinion, keep the process baseline (typically
+    /// the DOTNET_GCHeapHardLimit env var baked into the image for the smallest tier).</description></item>
+    /// <item><description><c>0</c> = disable the cap entirely, so the GC uses the container's own memory
+    /// allowance — for tiers with more memory than the baked limit lets them use.</description></item>
+    /// <item><description>&gt; 0 = set that many MB.</description></item>
+    /// </list>
+    /// The baked env var is consumed by the CLR before any managed code runs, so this is applied after
+    /// the fact via <see cref="Craft.Hosting.GcHeapLimit"/> — raising or removing the limit is always
+    /// safe; a positive value the heap has already outgrown is refused and logged.
+    /// <para>
+    /// Per-instance override: the <c>CRAFT_GC_HEAP_LIMIT_MB</c> env var wins over this value (a positive
+    /// value sets the cap; <c>0</c> disables the cap entirely), so an operator can hand-tune one host
+    /// without editing the fleet-wide profile list.
+    /// </para>
+    /// </summary>
+    public int? GCHeapHardLimitMB { get; set; }
 }

Services/Configuration/TelemetrySettings.cs

@@ -0,0 +1,41 @@
+namespace Craft.Configuration;
+
+/// <summary>
+/// Startup phone-home telemetry (<c>App:Telemetry:*</c>). One usage report per process start, storm
+/// guarded so a crash loop cannot flood the ingest. See <c>StartupTelemetryService</c>.
+///
+/// <para>
+/// Off by default: nothing is sent until an operator both enables it and supplies an
+/// <see cref="AppId"/> and an <see cref="Endpoint"/>. The <c>CRAFT_TELEMETRY_OPTOUT=1</c> environment
+/// variable forces it off regardless of configuration.
+/// </para>
+/// </summary>
+public class TelemetrySettings
+{
+    /// <summary>Master switch. Off by default (privacy posture is an operator decision).</summary>
+    public bool Enabled { get; set; }
+
+    /// <summary>Ingest URL, e.g. <c>https://reporting.example.com/API/TelemetryIngest</c>. No send without it.</summary>
+    public string? Endpoint { get; set; }
+
+    /// <summary>Application id for this image (<c>cipp</c>, <c>geoipdb</c>, …). No send without it.</summary>
+    public string? AppId { get; set; }
+
+    /// <summary>Storm-guard floor: at most one report per this many hours per instance. Floored at 1.</summary>
+    public int MinIntervalHours { get; set; } = 6;
+
+    /// <summary>Outbound POST timeout in seconds. No retry — the next boot is the retry.</summary>
+    public int TimeoutSeconds { get; set; } = 10;
+
+    /// <summary>Lower bound of the jittered startup delay, in seconds.</summary>
+    public int MinStartupDelaySeconds { get; set; } = 60;
+
+    /// <summary>Upper bound of the jittered startup delay, in seconds.</summary>
+    public int MaxStartupDelaySeconds { get; set; } = 300;
+
+    /// <summary>Table holding the per-instance storm-guard state (<c>instanceId</c>, <c>lastSentUtc</c>).</summary>
+    public string GuardTable { get; set; } = "CraftTelemetryGuard";
+
+    /// <summary>Optional shared token sent as <c>X-Telemetry-Token</c> to a token-gated ingest.</summary>
+    public string? Token { get; set; }
+}

Services/Configuration/WorkerSettings.cs

@@ -35,6 +35,23 @@ public class WorkerSettings
     /// </summary>
     public bool IgnoreSkuProfiles { get; set; }
 
+    /// <summary>
+    /// A second SkuProfiles matrix, selected when the env var named by <see cref="SkuProfilesAltEnv"/> is
+    /// present (set to a non-empty value); otherwise <see cref="SkuProfiles"/> is used. Lets a deployment
+    /// ship one config with two sizings — e.g. a smaller per-instance matrix for instances packed onto a
+    /// shared App Service Plan — and pick between them with a single env var. Same matching rules as
+    /// <see cref="SkuProfiles"/>. Ignored (falls back to <see cref="SkuProfiles"/>) when empty.
+    /// </summary>
+    public List<SkuProfile> SkuProfilesAlt { get; set; } = [];
+
+    /// <summary>
+    /// Name of the env var whose presence selects <see cref="SkuProfilesAlt"/> instead of
+    /// <see cref="SkuProfiles"/> (e.g. "CIPP_HOSTED"). Null/empty = the second matrix is never used.
+    /// Only presence matters — set the var (to any non-empty value) on the instances that should use the
+    /// second matrix, and leave it unset on the rest. Configurable so each app picks its own flag.
+    /// </summary>
+    public string? SkuProfilesAltEnv { get; set; }
+
     /// <summary>
     /// Minimum .NET thread-pool worker/completion threads. <b>0 (default) = derive from the pool
     /// sizes</b>, which is almost always what you want; set a number only to pin it.
@@ -75,6 +92,29 @@ public class WorkerSettings
     /// </summary>
     public int BgTimeoutSeconds { get; set; }
 
+    /// <summary>
+    /// How long an incoming HTTP request waits for a free PowerShell runspace when every worker in the
+    /// HTTP pool is already busy, before it is shed with <c>503 "Server busy, please retry"</c>.
+    ///
+    /// <para>
+    /// This is a load-shedding bound, NOT a capacity or execution knob. It does not add throughput —
+    /// under sustained saturation it only changes how long callers wait before the 503 (longer waits
+    /// hold connections and lengthen the tail). Its value is in absorbing <b>brief</b> bursts: a
+    /// request that would have got a worker a few seconds later completes instead of failing spuriously.
+    /// When 503s appear under steady load the levers are <see cref="HttpPoolSize"/> /
+    /// <see cref="MinThreads"/> / a larger host, not this.
+    /// </para>
+    ///
+    /// <para>
+    /// Distinct from <see cref="HttpTimeoutSeconds"/> (which bounds how long a request may <i>execute</i>
+    /// once it holds a worker). 0 or negative = the built-in default of 30 seconds.
+    /// </para>
+    ///
+    /// Env override: <c>CRAFT_HTTP_QUEUE_TIMEOUT</c> (seconds), which wins over this setting.
+    /// Resolved by <c>CraftHostBuilderExtensions.ResolveHttpQueueTimeout</c>.
+    /// </summary>
+    public int HttpQueueTimeoutSeconds { get; set; }
+
     /// <summary>
     /// Environment variables to inject into every PowerShell runspace.
     /// Use "{ApiBasePath}" as a placeholder — it will be replaced with the resolved API directory at startup.

Services/Hosting/CallerClassifier.cs

@@ -0,0 +1,31 @@
+namespace Craft.Hosting;
+
+/// <summary>
+/// Classifies a request as an app-only API client versus an interactive (UI) caller, from the
+/// normalised principal headers <see cref="CraftAuthMiddleware"/> writes.
+/// <para>
+/// The distinction is load-bearing for the API concurrency cap: app-only automation must not be able
+/// to monopolise the shared worker pool and starve interactive users, who are never limited by it.
+/// The rule is exactly the one the hosted app already keys off — a client-credentials caller arrives
+/// with <c>x-ms-client-principal-idp: aad</c> and its AppId (a GUID) as the principal name, whereas an
+/// interactive Entra user is normalised to <c>azureStaticWebApps</c>. Both conditions are required so a
+/// stray <c>aad</c> idp on a non-GUID principal is never misread as an API client.
+/// </para>
+/// </summary>
+public static class CallerClassifier
+{
+    /// <summary>
+    /// True when <paramref name="context"/> is an app-only API client (idp is <c>aad</c> and the
+    /// principal name parses as a GUID AppId). Depends on running after <see cref="CraftAuthMiddleware"/>.
+    /// </summary>
+    public static bool IsApiClient(HttpContext context)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+
+        var idp = context.Request.Headers["x-ms-client-principal-idp"].ToString();
+        if (!string.Equals(idp, "aad", StringComparison.OrdinalIgnoreCase)) return false;
+
+        var name = context.Request.Headers["x-ms-client-principal-name"].ToString();
+        return Guid.TryParse(name, out _);
+    }
+}