diff --git a/agents/metadata-agent/README.md b/agents/metadata-agent/README.md index e03e2d2b01..a5ba7e15da 100644 --- a/agents/metadata-agent/README.md +++ b/agents/metadata-agent/README.md @@ -24,7 +24,7 @@ The Metadata Agent helps teams: 2. **Sync** labels and project fields with confidence scoring 3. **Validate** release readiness across Tier 1, Tier 2, and Tier 3 checks 4. **Discover** and learn label taxonomy -5. **Recover** from errors with intelligent retry and suggestions +5. **Recover** from errors with intelligent retry, adaptive backoff, and quota forecasting Perfect for: @@ -302,6 +302,12 @@ Auto-apply threshold: **85%+** confidence (customizable) Hit rate limit → Wait until reset → Retry → Continue ``` +The Phase 2.4 rate limit modules provide: + +- `lib/rate-limit/rate-limit-tracker.js` — independent tracking for core, GraphQL, and search quotas +- `lib/rate-limit/backoff-calculator.js` — adaptive exponential backoff with jitter and Retry-After support +- `lib/rate-limit/quota-monitor.js` — quota health states and recovery estimates + ### Missing Labels ``` diff --git a/agents/metadata-agent/__tests__/api/retry-strategy.test.js b/agents/metadata-agent/__tests__/api/retry-strategy.test.js index 084e83e77f..78d1dfab6b 100644 --- a/agents/metadata-agent/__tests__/api/retry-strategy.test.js +++ b/agents/metadata-agent/__tests__/api/retry-strategy.test.js @@ -158,6 +158,25 @@ describe("RetryStrategy", () => { expect(delay).toBeGreaterThan(3000); }); + test("uses adaptive delay for search rate limit pressure", () => { + const fastStrategy = new RetryStrategy({ + initialDelayMs: 100, + maxDelayMs: 5000, + jitterFactor: 0, + }); + + const error = new Error("Rate limit exceeded"); + error.status = 429; + error.response = { headers: { "x-ratelimit-resource": "search" } }; + + const delay = fastStrategy.getRetryDelay(error, 1, { + quotaRemainingPercent: 5, + }); + + const expectedDelay = 100 * 2 * 3 * (1 + 0.95); + expect(delay).toBeCloseTo(expectedDelay, 0); + }); + test("handles missing response headers", () => { const error = new Error("Generic error"); const delay = strategy.getRetryDelay(error, 0); diff --git a/agents/metadata-agent/__tests__/rate-limit/backoff-calculator.test.js b/agents/metadata-agent/__tests__/rate-limit/backoff-calculator.test.js new file mode 100644 index 0000000000..27e59420f2 --- /dev/null +++ b/agents/metadata-agent/__tests__/rate-limit/backoff-calculator.test.js @@ -0,0 +1,56 @@ +const BackoffCalculator = require("../../lib/rate-limit/backoff-calculator"); + +describe("BackoffCalculator", () => { + const originalRandom = Math.random; + + afterEach(() => { + Math.random = originalRandom; + }); + + test("calculates exponential delay with jitter", () => { + Math.random = jest.fn(() => 0.75); + const calculator = new BackoffCalculator({ + initialDelayMs: 1000, + maxDelayMs: 60000, + backoffFactor: 2, + jitterFactor: 0.1, + }); + + const delay = calculator.calculateDelay({ attemptNumber: 2 }); + + expect(delay).toBeGreaterThan(4000); + expect(delay).toBeLessThan(4500); + }); + + test("prefers Retry-After header over computed delay", () => { + const calculator = new BackoffCalculator({ maxDelayMs: 60000 }); + const error = new Error("Rate limited"); + error.response = { headers: { "retry-after": "30" } }; + + const delay = calculator.calculateDelay({ attemptNumber: 0, error }); + + expect(delay).toBe(30000); + }); + + test("adapts delay for rate limit pressure", () => { + Math.random = jest.fn(() => 0.5); + const calculator = new BackoffCalculator({ + initialDelayMs: 1000, + jitterFactor: 0, + maxDelayMs: 60000, + }); + + const error = new Error("API rate limit exceeded"); + error.status = 429; + + const normal = calculator.calculateDelay({ attemptNumber: 1, error }); + const pressured = calculator.calculateDelay({ + attemptNumber: 1, + error, + rateLimitType: "search", + quotaRemainingPercent: 2, + }); + + expect(pressured).toBeGreaterThan(normal); + }); +}); diff --git a/agents/metadata-agent/__tests__/rate-limit/quota-monitor.test.js b/agents/metadata-agent/__tests__/rate-limit/quota-monitor.test.js new file mode 100644 index 0000000000..97b4fe41ac --- /dev/null +++ b/agents/metadata-agent/__tests__/rate-limit/quota-monitor.test.js @@ -0,0 +1,47 @@ +const QuotaMonitor = require("../../lib/rate-limit/quota-monitor"); +const RateLimitTracker = require("../../lib/rate-limit/rate-limit-tracker"); + +describe("QuotaMonitor", () => { + let tracker; + let monitor; + + beforeEach(() => { + tracker = new RateLimitTracker(); + monitor = new QuotaMonitor(tracker); + }); + + test("throws when tracker is missing", () => { + expect(() => new QuotaMonitor()).toThrow("RateLimitTracker is required"); + }); + + test("returns warning state when quota is low", () => { + tracker.limits.search.remaining = 4; + + const status = monitor.getStatus("search"); + + expect(status.state).toBe("warning"); + expect(status.percentage).toBe(13); + }); + + test("returns exhausted state when no quota remains", () => { + tracker.limits.graphql.remaining = 0; + tracker.limits.graphql.reset = new Date(Date.now() + 20000); + + const status = monitor.getStatus("graphql", 1); + + expect(status.state).toBe("exhausted"); + expect(status.hasQuota).toBe(false); + expect(status.recoveryInMs).toBeGreaterThan(15000); + }); + + test("reports global bottleneck", () => { + tracker.limits.core.remaining = 4900; + tracker.limits.graphql.remaining = 500; + tracker.limits.search.remaining = 3; + + const status = monitor.getGlobalStatus(); + + expect(status.bottleneck).toBe("search"); + expect(status.search.state).toBe("warning"); + }); +}); diff --git a/agents/metadata-agent/__tests__/rate-limit/rate-limit-tracker.test.js b/agents/metadata-agent/__tests__/rate-limit/rate-limit-tracker.test.js new file mode 100644 index 0000000000..42c961da34 --- /dev/null +++ b/agents/metadata-agent/__tests__/rate-limit/rate-limit-tracker.test.js @@ -0,0 +1,55 @@ +const RateLimitTracker = require("../../lib/rate-limit/rate-limit-tracker"); + +describe("RateLimitTracker", () => { + let tracker; + + beforeEach(() => { + tracker = new RateLimitTracker(); + }); + + test("tracks core/graphql/search from rate limit response", () => { + const nowInSeconds = Math.floor(Date.now() / 1000); + const limits = tracker.updateFromRateLimitResponse({ + rate_limit: { limit: 5000, remaining: 4900, reset: nowInSeconds + 3600 }, + resources: { + graphql: { limit: 5000, remaining: 4700, reset: nowInSeconds + 3600 }, + search: { limit: 30, remaining: 20, reset: nowInSeconds + 60 }, + }, + }); + + expect(limits.core.remaining).toBe(4900); + expect(limits.graphql.remaining).toBe(4700); + expect(limits.search.remaining).toBe(20); + }); + + test("updates limit state from headers", () => { + const update = tracker.updateFromHeaders("search", { + "x-ratelimit-limit": "30", + "x-ratelimit-remaining": "5", + "x-ratelimit-reset": String(Math.floor(Date.now() / 1000) + 30), + }); + + expect(update.limit).toBe(30); + expect(update.remaining).toBe(5); + expect(update.reset).toBeInstanceOf(Date); + }); + + test("estimates quota recovery when quota is exhausted", () => { + tracker.limits.search.remaining = 0; + tracker.limits.search.reset = new Date(Date.now() + 15000); + + const estimate = tracker.estimateQuotaRecovery("search", 1); + + expect(estimate.hasQuota).toBe(false); + expect(estimate.recoveryInMs).toBeGreaterThan(10000); + expect(estimate.recoveryInMs).toBeLessThanOrEqual(15000); + }); + + test("identifies the most constrained quota pool", () => { + tracker.limits.core.remaining = 4500; + tracker.limits.graphql.remaining = 2500; + tracker.limits.search.remaining = 3; + + expect(tracker.getMostConstrainedType()).toBe("search"); + }); +}); diff --git a/agents/metadata-agent/lib/api/rate-limiter.js b/agents/metadata-agent/lib/api/rate-limiter.js index 4f68f11c54..539d7fa3ee 100644 --- a/agents/metadata-agent/lib/api/rate-limiter.js +++ b/agents/metadata-agent/lib/api/rate-limiter.js @@ -1,29 +1,19 @@ +const RateLimitTracker = require("../rate-limit/rate-limit-tracker"); + /** * Rate limit monitor for GitHub API. * Tracks core, GraphQL, and search rate limits independently. * Provides quota recovery estimation and threshold alerts. */ -class RateLimiter { +class RateLimiter extends RateLimitTracker { constructor(client) { if (!client) { throw new Error("Octokit client is required"); } + super(); this.client = client; - this.limits = { - core: { limit: 5000, remaining: 5000, reset: null }, - graphql: { limit: 5000, remaining: 5000, reset: null }, - search: { limit: 30, remaining: 30, reset: null }, - }; - - this.thresholds = { - core: 100, // Alert at 100 requests remaining - graphql: 100, - search: 5, // Alert at 5 searches remaining - }; - - this.lastUpdate = null; } /** @@ -33,35 +23,7 @@ class RateLimiter { async updateRateLimits() { try { const response = await this.client.rateLimit.get(); - const { rate_limit: rateLimit, resources } = response.data; - - // Update core limit - this.limits.core = { - limit: rateLimit.limit, - remaining: rateLimit.remaining, - reset: new Date(rateLimit.reset * 1000), - }; - - // Update GraphQL limit - if (resources.graphql) { - this.limits.graphql = { - limit: resources.graphql.limit, - remaining: resources.graphql.remaining, - reset: new Date(resources.graphql.reset * 1000), - }; - } - - // Update search limit - if (resources.search) { - this.limits.search = { - limit: resources.search.limit, - remaining: resources.search.remaining, - reset: new Date(resources.search.reset * 1000), - }; - } - - this.lastUpdate = new Date(); - return this.limits; + return this.updateFromRateLimitResponse(response.data); } catch (error) { throw new Error(`Failed to update rate limits: ${error.message}`, { cause: error, @@ -74,46 +36,6 @@ class RateLimiter { * @param {string} type - API type: 'core', 'graphql', or 'search' * @returns {Object} Rate limit information */ - getLimit(type = "core") { - if (!this.limits[type]) { - throw new Error(`Unknown rate limit type: ${type}`); - } - return { ...this.limits[type] }; - } - - /** - * Get all rate limits - * @returns {Object} All rate limits - */ - getAllLimits() { - return { - core: { ...this.limits.core }, - graphql: { ...this.limits.graphql }, - search: { ...this.limits.search }, - }; - } - - /** - * Check if rate limit is below threshold - * @param {string} type - API type - * @returns {boolean} True if below threshold - */ - isBelowThreshold(type = "core") { - const limit = this.getLimit(type); - return limit.remaining <= this.thresholds[type]; - } - - /** - * Get time until rate limit reset for a specific type - * @param {string} type - API type - * @returns {number} Milliseconds until reset - */ - getTimeUntilReset(type = "core") { - const limit = this.getLimit(type); - if (!limit.reset) return 0; - return Math.max(0, limit.reset - new Date()); - } - /** * Estimate quota recovery time (when we'll have quota available again) * @param {string} type - API type @@ -121,27 +43,7 @@ class RateLimiter { */ async estimateQuotaRecovery(type = "core") { await this.updateRateLimits(); - - const limit = this.getLimit(type); - - // If we have quota, return immediately - if (limit.remaining > 0) { - return 0; - } - - // Otherwise, return time until reset - return this.getTimeUntilReset(type); - } - - /** - * Get percentage of quota remaining - * @param {string} type - API type - * @returns {number} Percentage (0-100) - */ - getPercentageRemaining(type = "core") { - const limit = this.getLimit(type); - if (limit.limit === 0) return 0; - return Math.round((limit.remaining / limit.limit) * 100); + return super.estimateQuotaRecovery(type, 1).recoveryInMs; } /** @@ -191,21 +93,6 @@ class RateLimiter { }; } - /** - * Set custom threshold for a rate limit type - * @param {string} type - API type - * @param {number} threshold - Threshold value - */ - setThreshold(type, threshold) { - if (!this.thresholds[type]) { - throw new Error(`Unknown rate limit type: ${type}`); - } - if (threshold < 0) { - throw new Error("Threshold must be non-negative"); - } - this.thresholds[type] = threshold; - } - /** * Reset all rate limits to defaults */ diff --git a/agents/metadata-agent/lib/api/retry-strategy.js b/agents/metadata-agent/lib/api/retry-strategy.js index 60a3224a8d..914df60225 100644 --- a/agents/metadata-agent/lib/api/retry-strategy.js +++ b/agents/metadata-agent/lib/api/retry-strategy.js @@ -1,3 +1,6 @@ +const BackoffCalculator = require("../rate-limit/backoff-calculator"); +const { RATE_LIMIT_TYPES } = require("../rate-limit/rate-limit-types"); + /** * Exponential backoff retry strategy with jitter for GitHub API resilience. * Handles rate limit errors, network errors, and transient failures. @@ -5,11 +8,17 @@ class RetryStrategy { constructor(config = {}) { - this.maxRetries = config.maxRetries || 5; + this.maxRetries = config.maxRetries ?? 5; this.initialDelayMs = config.initialDelayMs ?? 1000; this.maxDelayMs = config.maxDelayMs ?? 60000; - this.backoffFactor = config.backoffFactor || 2; - this.jitterFactor = config.jitterFactor || 0.1; + this.backoffFactor = config.backoffFactor ?? 2; + this.jitterFactor = config.jitterFactor ?? 0.1; + this.backoffCalculator = new BackoffCalculator({ + initialDelayMs: this.initialDelayMs, + maxDelayMs: this.maxDelayMs, + backoffFactor: this.backoffFactor, + jitterFactor: this.jitterFactor, + }); // Error codes that should trigger retry this.retryableErrors = new Set([ @@ -62,22 +71,7 @@ class RetryStrategy { * @returns {number} Delay in milliseconds */ calculateDelay(attemptNumber) { - if (attemptNumber < 0) { - throw new Error("Attempt number must be non-negative"); - } - - // Exponential backoff: initialDelay * (backoffFactor ^ attemptNumber) - let delay = - this.initialDelayMs * Math.pow(this.backoffFactor, attemptNumber); - - // Cap at max delay - delay = Math.min(delay, this.maxDelayMs); - - // Add jitter: delay * (1 ± jitterFactor * random) - const jitter = delay * this.jitterFactor * (Math.random() - 0.5) * 2; - delay = Math.round(delay + jitter); - - return Math.max(0, delay); + return this.backoffCalculator.calculateDelay({ attemptNumber }); } /** @@ -86,25 +80,30 @@ class RetryStrategy { * @param {number} attemptNumber - Current attempt (0-indexed) * @returns {number} Delay in milliseconds */ - getRetryDelay(error, attemptNumber) { - // Check for explicit Retry-After header - if (error.response?.headers?.["retry-after"]) { - const retryAfter = error.response.headers["retry-after"]; - - // Retry-After can be seconds or HTTP-date - if (/^\d+$/.test(retryAfter)) { - return parseInt(retryAfter) * 1000; - } + getRetryDelay(error, attemptNumber, context = {}) { + return this.backoffCalculator.calculateDelay({ + attemptNumber, + error, + rateLimitType: context.rateLimitType || this._detectRateLimitType(error), + quotaRemainingPercent: context.quotaRemainingPercent ?? null, + }); + } - // Parse HTTP-date - const retryDate = new Date(retryAfter); - if (!isNaN(retryDate)) { - return Math.max(0, retryDate - new Date()); - } + _detectRateLimitType(error) { + const resourceHeader = + error?.response?.headers?.["x-ratelimit-resource"] || + error?.rateLimitType; + const normalised = String(resourceHeader || "").toLowerCase(); + + if (normalised === RATE_LIMIT_TYPES.GRAPHQL) { + return RATE_LIMIT_TYPES.GRAPHQL; + } + + if (normalised === RATE_LIMIT_TYPES.SEARCH) { + return RATE_LIMIT_TYPES.SEARCH; } - // Fall back to exponential backoff - return this.calculateDelay(attemptNumber); + return RATE_LIMIT_TYPES.CORE; } /** @@ -200,6 +199,12 @@ class RetryStrategy { this.jitterFactor = config.jitterFactor; this.validate(); + this.backoffCalculator = new BackoffCalculator({ + initialDelayMs: this.initialDelayMs, + maxDelayMs: this.maxDelayMs, + backoffFactor: this.backoffFactor, + jitterFactor: this.jitterFactor, + }); } } diff --git a/agents/metadata-agent/lib/rate-limit/backoff-calculator.js b/agents/metadata-agent/lib/rate-limit/backoff-calculator.js new file mode 100644 index 0000000000..0ffb17b4f3 --- /dev/null +++ b/agents/metadata-agent/lib/rate-limit/backoff-calculator.js @@ -0,0 +1,97 @@ +const { RATE_LIMIT_TYPES } = require("./rate-limit-types"); + +/** + * Calculates adaptive retry delays with exponential backoff and jitter. + */ +class BackoffCalculator { + constructor(config = {}) { + this.initialDelayMs = config.initialDelayMs ?? 1000; + this.maxDelayMs = config.maxDelayMs ?? 60000; + this.backoffFactor = config.backoffFactor ?? 2; + this.jitterFactor = config.jitterFactor ?? 0.1; + this.rateLimitMultipliers = { + core: config.coreMultiplier ?? 1.5, + graphql: config.graphqlMultiplier ?? 2, + search: config.searchMultiplier ?? 3, + }; + } + + calculateExponentialDelay(attemptNumber) { + if (!Number.isFinite(attemptNumber) || attemptNumber < 0) { + throw new Error("Attempt number must be non-negative"); + } + + return Math.min( + this.initialDelayMs * Math.pow(this.backoffFactor, attemptNumber), + this.maxDelayMs, + ); + } + + applyJitter(delayMs) { + if (!Number.isFinite(delayMs) || delayMs < 0) { + throw new Error("Delay must be non-negative"); + } + + if (this.jitterFactor === 0) { + return Math.round(delayMs); + } + + const randomValue = Math.max(Number.EPSILON, Math.random()); + const jitter = delayMs * this.jitterFactor * (randomValue * 2 - 1); + return Math.max(0, Math.round(delayMs + jitter)); + } + + getRetryAfterDelayMs(error) { + const retryAfter = error?.response?.headers?.["retry-after"]; + if (!retryAfter) { + return null; + } + + if (/^\d+$/.test(retryAfter)) { + return parseInt(retryAfter, 10) * 1000; + } + + const retryDate = new Date(retryAfter); + if (Number.isNaN(retryDate.getTime())) { + return null; + } + + return Math.max(0, retryDate.getTime() - Date.now()); + } + + calculateDelay({ + attemptNumber, + error = null, + rateLimitType = RATE_LIMIT_TYPES.CORE, + quotaRemainingPercent = null, + }) { + const retryAfterDelay = this.getRetryAfterDelayMs(error); + if (retryAfterDelay !== null) { + return Math.min(retryAfterDelay, this.maxDelayMs); + } + + let delay = this.calculateExponentialDelay(attemptNumber); + + if ( + error?.status === 429 || + error?.message?.toLowerCase().includes("rate limit") + ) { + const multiplier = this.rateLimitMultipliers[rateLimitType] || 1; + delay *= multiplier; + + if (Number.isFinite(quotaRemainingPercent)) { + const pressure = Math.max(0, 100 - quotaRemainingPercent) / 100; + delay *= 1 + pressure; + } + } + + delay = Math.min(delay, this.maxDelayMs); + if (delay === this.maxDelayMs) { + return this.maxDelayMs; + } + + return this.applyJitter(delay); + } +} + +module.exports = BackoffCalculator; diff --git a/agents/metadata-agent/lib/rate-limit/quota-monitor.js b/agents/metadata-agent/lib/rate-limit/quota-monitor.js new file mode 100644 index 0000000000..3bbce5daa9 --- /dev/null +++ b/agents/metadata-agent/lib/rate-limit/quota-monitor.js @@ -0,0 +1,73 @@ +const { RATE_LIMIT_TYPES } = require("./rate-limit-types"); + +/** + * Produces quota health and recovery estimates from a rate limit tracker. + */ +class QuotaMonitor { + constructor(tracker, config = {}) { + if (!tracker) { + throw new Error("RateLimitTracker is required"); + } + + this.tracker = tracker; + this.warningPercent = config.warningPercent ?? 15; + this.criticalPercent = config.criticalPercent ?? 5; + } + + _resolveState(limitPercent, remaining) { + if (remaining <= 0) { + return "exhausted"; + } + + if (limitPercent <= this.criticalPercent) { + return "critical"; + } + + if (limitPercent <= this.warningPercent) { + return "warning"; + } + + return "healthy"; + } + + getStatus(type = RATE_LIMIT_TYPES.CORE, requiredRequests = 1) { + const limit = this.tracker.getLimit(type); + const percentage = this.tracker.getPercentageRemaining(type); + const recovery = this.tracker.estimateQuotaRecovery(type, requiredRequests); + + return { + type, + state: this._resolveState(percentage, limit.remaining), + remaining: limit.remaining, + limit: limit.limit, + percentage, + requiredRequests, + recoveryInMs: recovery.recoveryInMs, + resetAt: recovery.resetAt, + hasQuota: recovery.hasQuota, + }; + } + + getGlobalStatus(requiredRequests = 1) { + const statuses = Object.values(RATE_LIMIT_TYPES).map((type) => + this.getStatus(type, requiredRequests), + ); + const byType = statuses.reduce((acc, status) => { + acc[status.type] = status; + return acc; + }, {}); + + const bottleneck = [...statuses].sort( + (a, b) => a.percentage - b.percentage || a.remaining - b.remaining, + )[0]; + + return { + core: byType[RATE_LIMIT_TYPES.CORE], + graphql: byType[RATE_LIMIT_TYPES.GRAPHQL], + search: byType[RATE_LIMIT_TYPES.SEARCH], + bottleneck: bottleneck.type, + }; + } +} + +module.exports = QuotaMonitor; diff --git a/agents/metadata-agent/lib/rate-limit/rate-limit-tracker.js b/agents/metadata-agent/lib/rate-limit/rate-limit-tracker.js new file mode 100644 index 0000000000..f6d6b317d5 --- /dev/null +++ b/agents/metadata-agent/lib/rate-limit/rate-limit-tracker.js @@ -0,0 +1,256 @@ +const { + RATE_LIMIT_TYPES, + DEFAULT_LIMITS, + DEFAULT_THRESHOLDS, + isValidRateLimitType, +} = require("./rate-limit-types"); + +/** + * Tracks GitHub API quotas for core, GraphQL, and search independently. + */ +class RateLimitTracker { + constructor(config = {}) { + const limits = config.limits || {}; + const thresholds = config.thresholds || {}; + + this.limits = { + core: { ...DEFAULT_LIMITS.core, ...(limits.core || {}) }, + graphql: { ...DEFAULT_LIMITS.graphql, ...(limits.graphql || {}) }, + search: { ...DEFAULT_LIMITS.search, ...(limits.search || {}) }, + }; + + this.thresholds = { + core: thresholds.core ?? DEFAULT_THRESHOLDS.core, + graphql: thresholds.graphql ?? DEFAULT_THRESHOLDS.graphql, + search: thresholds.search ?? DEFAULT_THRESHOLDS.search, + }; + + this.lastUpdate = null; + } + + _assertType(type) { + if (!isValidRateLimitType(type)) { + throw new Error(`Unknown rate limit type: ${type}`); + } + } + + _toResetDate(rawReset) { + if (!rawReset && rawReset !== 0) { + return null; + } + + if (rawReset instanceof Date) { + return rawReset; + } + + if (typeof rawReset === "number") { + return new Date(rawReset * 1000); + } + + if (typeof rawReset === "string" && /^\d+$/.test(rawReset)) { + return new Date(parseInt(rawReset, 10) * 1000); + } + + const parsed = new Date(rawReset); + return Number.isNaN(parsed.getTime()) ? null : parsed; + } + + _normaliseLimit(payload, fallbackType) { + if (!payload) { + return this.getLimit(fallbackType); + } + + const parsedLimit = Number(payload.limit); + const parsedRemaining = Number(payload.remaining); + const current = this.limits[fallbackType]; + const reset = payload.reset !== undefined ? this._toResetDate(payload.reset) : current.reset; + + return { + limit: Number.isFinite(parsedLimit) ? parsedLimit : current.limit, + remaining: Number.isFinite(parsedRemaining) ? parsedRemaining : current.remaining, + reset, + }; + } + + updateFromRateLimitResponse(responseData = {}) { + const resources = responseData.resources || {}; + const corePayload = responseData.rate_limit || resources.core; + + if (corePayload) { + this.limits.core = this._normaliseLimit( + corePayload, + RATE_LIMIT_TYPES.CORE, + ); + } + + if (resources.graphql) { + this.limits.graphql = this._normaliseLimit( + resources.graphql, + RATE_LIMIT_TYPES.GRAPHQL, + ); + } + + if (resources.search) { + this.limits.search = this._normaliseLimit( + resources.search, + RATE_LIMIT_TYPES.SEARCH, + ); + } + + this.lastUpdate = new Date(); + return this.getAllLimits(); + } + + updateFromHeaders(type, headers = {}) { + this._assertType(type); + + const limit = Number(headers["x-ratelimit-limit"]); + const remaining = Number(headers["x-ratelimit-remaining"]); + const reset = this._toResetDate(headers["x-ratelimit-reset"]); + + if (Number.isFinite(limit) && limit >= 0) { + this.limits[type].limit = limit; + } + + if (Number.isFinite(remaining) && remaining >= 0) { + this.limits[type].remaining = remaining; + } + + if (reset) { + this.limits[type].reset = reset; + } + + this.lastUpdate = new Date(); + return this.getLimit(type); + } + + getLimit(type = RATE_LIMIT_TYPES.CORE) { + this._assertType(type); + return { ...this.limits[type] }; + } + + getAllLimits() { + return { + core: this.getLimit(RATE_LIMIT_TYPES.CORE), + graphql: this.getLimit(RATE_LIMIT_TYPES.GRAPHQL), + search: this.getLimit(RATE_LIMIT_TYPES.SEARCH), + }; + } + + setThreshold(type, threshold) { + this._assertType(type); + + if (!Number.isFinite(threshold) || threshold < 0) { + throw new Error("Threshold must be non-negative"); + } + + this.thresholds[type] = threshold; + } + + isBelowThreshold(type = RATE_LIMIT_TYPES.CORE) { + const limit = this.getLimit(type); + return limit.remaining <= this.thresholds[type]; + } + + getTimeUntilReset(type = RATE_LIMIT_TYPES.CORE) { + const limit = this.getLimit(type); + if (!limit.reset) { + return 0; + } + + return Math.max(0, limit.reset.getTime() - Date.now()); + } + + getPercentageRemaining(type = RATE_LIMIT_TYPES.CORE) { + const limit = this.getLimit(type); + if (limit.limit <= 0) { + return 0; + } + + return Math.round((limit.remaining / limit.limit) * 100); + } + + estimateQuotaRecovery(type = RATE_LIMIT_TYPES.CORE, requiredRequests = 1) { + this._assertType(type); + + if (!Number.isFinite(requiredRequests) || requiredRequests < 0) { + throw new Error("requiredRequests must be non-negative"); + } + + const limit = this.getLimit(type); + const missingRequests = Math.max(0, requiredRequests - limit.remaining); + + return { + hasQuota: missingRequests === 0, + missingRequests, + recoveryInMs: missingRequests === 0 ? 0 : this.getTimeUntilReset(type), + resetAt: limit.reset, + remaining: limit.remaining, + requiredRequests, + }; + } + + getMostConstrainedType() { + const entries = Object.values(RATE_LIMIT_TYPES).map((type) => ({ + type, + percentage: this.getPercentageRemaining(type), + remaining: this.getLimit(type).remaining, + })); + + return entries.sort( + (a, b) => a.percentage - b.percentage || a.remaining - b.remaining, + )[0].type; + } + + recordRequest(type = RATE_LIMIT_TYPES.CORE, count = 1) { + this._assertType(type); + + if (!Number.isFinite(count) || count < 0) { + throw new Error("count must be non-negative"); + } + + this.limits[type].remaining = Math.max( + 0, + this.limits[type].remaining - count, + ); + return this.getLimit(type); + } + + reset() { + this.limits = { + core: { ...DEFAULT_LIMITS.core }, + graphql: { ...DEFAULT_LIMITS.graphql }, + search: { ...DEFAULT_LIMITS.search }, + }; + this.thresholds = { + core: DEFAULT_THRESHOLDS.core, + graphql: DEFAULT_THRESHOLDS.graphql, + search: DEFAULT_THRESHOLDS.search, + }; + this.lastUpdate = null; + } + + getSummary() { + return { + core: { + ...this.getLimit(RATE_LIMIT_TYPES.CORE), + percentage: this.getPercentageRemaining(RATE_LIMIT_TYPES.CORE), + resetIn: this.getTimeUntilReset(RATE_LIMIT_TYPES.CORE), + }, + graphql: { + ...this.getLimit(RATE_LIMIT_TYPES.GRAPHQL), + percentage: this.getPercentageRemaining(RATE_LIMIT_TYPES.GRAPHQL), + resetIn: this.getTimeUntilReset(RATE_LIMIT_TYPES.GRAPHQL), + }, + search: { + ...this.getLimit(RATE_LIMIT_TYPES.SEARCH), + percentage: this.getPercentageRemaining(RATE_LIMIT_TYPES.SEARCH), + resetIn: this.getTimeUntilReset(RATE_LIMIT_TYPES.SEARCH), + }, + lastUpdate: this.lastUpdate, + bottleneck: this.getMostConstrainedType(), + }; + } +} + +module.exports = RateLimitTracker; diff --git a/agents/metadata-agent/lib/rate-limit/rate-limit-types.js b/agents/metadata-agent/lib/rate-limit/rate-limit-types.js new file mode 100644 index 0000000000..6c475b73aa --- /dev/null +++ b/agents/metadata-agent/lib/rate-limit/rate-limit-types.js @@ -0,0 +1,32 @@ +/** + * Canonical GitHub API rate limit types and defaults. + */ + +const RATE_LIMIT_TYPES = Object.freeze({ + CORE: "core", + GRAPHQL: "graphql", + SEARCH: "search", +}); + +const DEFAULT_LIMITS = Object.freeze({ + [RATE_LIMIT_TYPES.CORE]: { limit: 5000, remaining: 5000, reset: null }, + [RATE_LIMIT_TYPES.GRAPHQL]: { limit: 5000, remaining: 5000, reset: null }, + [RATE_LIMIT_TYPES.SEARCH]: { limit: 30, remaining: 30, reset: null }, +}); + +const DEFAULT_THRESHOLDS = Object.freeze({ + [RATE_LIMIT_TYPES.CORE]: 100, + [RATE_LIMIT_TYPES.GRAPHQL]: 100, + [RATE_LIMIT_TYPES.SEARCH]: 5, +}); + +function isValidRateLimitType(type) { + return Object.values(RATE_LIMIT_TYPES).includes(type); +} + +module.exports = { + RATE_LIMIT_TYPES, + DEFAULT_LIMITS, + DEFAULT_THRESHOLDS, + isValidRateLimitType, +};