From f2d0076a4819a92bc6e595918958fcbb8c485d48 Mon Sep 17 00:00:00 2001 From: Christoph Rueger Date: Wed, 22 Jul 2026 18:19:23 +0200 Subject: [PATCH 1/3] Add negative caching for HTTP 404 responses Implement negative caching to avoid repeated requests for non-existent resources. When a 404 response is received, the metadata is cached with a notFound flag. Subsequent requests within the TTL window will return the cached 404 without making a network request. This reduces load on servers and improves performance for missing resource scenarios e.g. missing -sources.jar files of dependencies Signed-off-by: Christoph Rueger --- .../src/aQute/bnd/http/HttpClient.java | 18 ++++++++++++++++-- .../src/aQute/bnd/http/URLCache.java | 16 ++++++++++++++++ .../src/aQute/bnd/http/package-info.java | 2 +- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java b/biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java index 9c8f797885..6a55932426 100644 --- a/biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java +++ b/biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java @@ -477,6 +477,7 @@ private TaggedData doCached0() throws Exception { request.useCacheFile = info.file; if (info.isPresent()) { + // // We have a file in the cache, check if it is within // our accepted stale period @@ -504,7 +505,10 @@ private TaggedData doCached0() throws Exception { TaggedData tag = connect(); - if (tag.getState() == State.NOT_FOUND) { + if (tag.isNotFound()) { + // Negative cache: Just save the .json metadata + info.updateNegativeCache(); + // Clear the content file, keep metadata cache().clear(uri); } else if (tag.getState() == State.UPDATED) { // @@ -520,6 +524,11 @@ private TaggedData doCached0() throws Exception { } return new TaggedData(uri, HTTP_NOT_MODIFIED, info.file); } + else if (info.hasNegativeCache(request.maxStale)) { + // Has .json but no .content - negative cache hit + return new TaggedData(uri, HTTP_NOT_FOUND, info.file); + } + // // No entry in the cache, but we are cached // @@ -534,7 +543,12 @@ private TaggedData doCached0() throws Exception { TaggedData tag = connect(); - if (tag.isOk()) { + if (tag.isNotFound()) { + // Negative cache: Just save the .json metadata + info.updateNegativeCache(); + // Clear the content file, keep metadata + cache().clear(uri); + } else if (tag.isOk()) { info.update(tag.getInputStream(), tag.getTag(), tag.getModified()); } return tag; diff --git a/biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java b/biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java index 0d2712a592..fba073ff9b 100644 --- a/biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java +++ b/biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java @@ -37,6 +37,7 @@ public static class InfoDTO { public long modified; public URI uri; public String sha_256; + public boolean notFound = false; // true = cached 404 response } public class Info implements Closeable { @@ -92,6 +93,14 @@ public void update(String etag) throws Exception { .put(this.dto); } + public void updateNegativeCache() throws Exception { + this.dto.notFound = true; + this.dto.modified = System.currentTimeMillis(); + codec.enc() + .to(jsonFile) + .put(this.dto); + } + public boolean isPresent() { boolean f = file.isFile(); boolean j = jsonFile.isFile(); @@ -111,6 +120,13 @@ public long getModified() { return dto.modified; } + public boolean hasNegativeCache(long ttl) { + if (!jsonFile.isFile()) { + return false; + } + return dto.notFound && System.currentTimeMillis() - dto.modified < ttl; + } + @Override public String toString() { return "Info [file=" + file + ", etag=" + dto.etag + ", modified=" + Instant.ofEpochMilli(dto.modified) diff --git a/biz.aQute.bndlib/src/aQute/bnd/http/package-info.java b/biz.aQute.bndlib/src/aQute/bnd/http/package-info.java index d679690d05..d86a602067 100644 --- a/biz.aQute.bndlib/src/aQute/bnd/http/package-info.java +++ b/biz.aQute.bndlib/src/aQute/bnd/http/package-info.java @@ -1,4 +1,4 @@ -@Version("2.1.0") +@Version("2.2.0") package aQute.bnd.http; import org.osgi.annotation.versioning.Version; From 78c5009dbab5c03c7af8953a59b357fc55f0f6a3 Mon Sep 17 00:00:00 2001 From: Christoph Rueger Date: Thu, 23 Jul 2026 11:35:08 +0200 Subject: [PATCH 2/3] fix tests Signed-off-by: Christoph Rueger --- .../test/aQute/bnd/comm/tests/HttpClientTest.java | 3 ++- biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java | 4 ---- biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java | 3 ++- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/biz.aQute.bndlib.comm.tests/test/aQute/bnd/comm/tests/HttpClientTest.java b/biz.aQute.bndlib.comm.tests/test/aQute/bnd/comm/tests/HttpClientTest.java index e339868542..8fcb682a36 100644 --- a/biz.aQute.bndlib.comm.tests/test/aQute/bnd/comm/tests/HttpClientTest.java +++ b/biz.aQute.bndlib.comm.tests/test/aQute/bnd/comm/tests/HttpClientTest.java @@ -491,7 +491,8 @@ public void testClearCacheOn404(@InjectTemporaryDirectory .go(cheshire); assertNotNull(tag); assertEquals(404, tag.getResponseCode()); - assertThat(cache.isCached(cheshire)).isFalse(); + // we cache 404 NOT FOUND now + assertThat(cache.isCached(cheshire)).isTrue(); } } diff --git a/biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java b/biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java index 6a55932426..336a72b866 100644 --- a/biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java +++ b/biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java @@ -508,8 +508,6 @@ private TaggedData doCached0() throws Exception { if (tag.isNotFound()) { // Negative cache: Just save the .json metadata info.updateNegativeCache(); - // Clear the content file, keep metadata - cache().clear(uri); } else if (tag.getState() == State.UPDATED) { // // update the cache from the input stream @@ -546,8 +544,6 @@ else if (info.hasNegativeCache(request.maxStale)) { if (tag.isNotFound()) { // Negative cache: Just save the .json metadata info.updateNegativeCache(); - // Clear the content file, keep metadata - cache().clear(uri); } else if (tag.isOk()) { info.update(tag.getInputStream(), tag.getTag(), tag.getModified()); } diff --git a/biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java b/biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java index fba073ff9b..e4398668c3 100644 --- a/biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java +++ b/biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java @@ -95,6 +95,7 @@ public void update(String etag) throws Exception { public void updateNegativeCache() throws Exception { this.dto.notFound = true; + IO.mkdirs(this.jsonFile.getParentFile()); this.dto.modified = System.currentTimeMillis(); codec.enc() .to(jsonFile) @@ -130,7 +131,7 @@ public boolean hasNegativeCache(long ttl) { @Override public String toString() { return "Info [file=" + file + ", etag=" + dto.etag + ", modified=" + Instant.ofEpochMilli(dto.modified) - + ", url=" + url + ", lock=" + lock + "]"; + + ", url=" + url + ", lock=" + lock + ", notFound=" + dto.notFound + "]"; } } From fd5f5913a0585ca7960c8442e95a7f501cb6c38c Mon Sep 17 00:00:00 2001 From: Christoph Rueger Date: Thu, 23 Jul 2026 12:00:03 +0200 Subject: [PATCH 3/3] fix test Signed-off-by: Christoph Rueger --- .../test/aQute/maven/provider/RemoteRepoTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/biz.aQute.repository/test/aQute/maven/provider/RemoteRepoTest.java b/biz.aQute.repository/test/aQute/maven/provider/RemoteRepoTest.java index 6754862d8b..99c05d7fb0 100644 --- a/biz.aQute.repository/test/aQute/maven/provider/RemoteRepoTest.java +++ b/biz.aQute.repository/test/aQute/maven/provider/RemoteRepoTest.java @@ -78,7 +78,9 @@ public void testBasic() throws Exception { // Fetch it, must exist now // - assertEquals(State.UPDATED, repo.fetch("foo/bar", localFoobar) + // force cache refresh because we cache 404 NOT_FOUND now + boolean forceCacheRefresh = true; + assertEquals(State.UPDATED, repo.fetch("foo/bar", localFoobar, forceCacheRefresh ) .getState()); assertTrue(localFoobar.isFile()); assertEquals(3L, localFoobar.length());