Negative cache for 404 NOT_FOUND

biz.aQute.bndlib.comm.tests/test/aQute/bnd/comm/tests/HttpClientTest.java

@@ -491,7 +491,8 @@ public void testClearCacheOn404(@InjectTemporaryDirectory
 				.go(cheshire);
 			assertNotNull(tag);
 			assertEquals(404, tag.getResponseCode());
-			assertThat(cache.isCached(cheshire)).isFalse();
+			// we cache 404 NOT FOUND now
+			assertThat(cache.isCached(cheshire)).isTrue();
 		}
 	}
 

biz.aQute.bndlib/src/aQute/bnd/http/HttpClient.java

@@ -477,6 +477,7 @@ private TaggedData doCached0() throws Exception {
 
 				request.useCacheFile = info.file;
 				if (info.isPresent()) {
+
 					//
 					// We have a file in the cache, check if it is within
 					// our accepted stale period
@@ -504,8 +505,9 @@ private TaggedData doCached0() throws Exception {
 
 						TaggedData tag = connect();
 
-						if (tag.getState() == State.NOT_FOUND) {
-							cache().clear(uri);
+						if (tag.isNotFound()) {
+							// Negative cache: Just save the .json metadata
+							info.updateNegativeCache();
 						} else if (tag.getState() == State.UPDATED) {
 							//
 							// update the cache from the input stream
@@ -520,6 +522,11 @@ private TaggedData doCached0() throws Exception {
 					}
 					return new TaggedData(uri, HTTP_NOT_MODIFIED, info.file);
 				}
+				else if (info.hasNegativeCache(request.maxStale)) {
+					// Has .json but no .content - negative cache hit
+					return new TaggedData(uri, HTTP_NOT_FOUND, info.file);
+				}
+
 				//
 				// No entry in the cache, but we are cached
 				//
@@ -534,7 +541,10 @@ private TaggedData doCached0() throws Exception {
 
 				TaggedData tag = connect();
 
-				if (tag.isOk()) {
+				if (tag.isNotFound()) {
+					// Negative cache: Just save the .json metadata
+					info.updateNegativeCache();
+				} else if (tag.isOk()) {
 					info.update(tag.getInputStream(), tag.getTag(), tag.getModified());
 				}
 				return tag;

biz.aQute.bndlib/src/aQute/bnd/http/URLCache.java

@@ -37,6 +37,7 @@ public static class InfoDTO {
 		public long		modified;
 		public URI		uri;
 		public String	sha_256;
+		public boolean	notFound	= false;	// true = cached 404 response
 	}
 
 	public class Info implements Closeable {
@@ -92,6 +93,15 @@ public void update(String etag) throws Exception {
 				.put(this.dto);
 		}
 
+		public void updateNegativeCache() throws Exception {
+			this.dto.notFound = true;
+			IO.mkdirs(this.jsonFile.getParentFile());
+			this.dto.modified = System.currentTimeMillis();
+		    codec.enc()
+		        .to(jsonFile)
+		        .put(this.dto);
+		}
+
 		public boolean isPresent() {
 			boolean f = file.isFile();
 			boolean j = jsonFile.isFile();
@@ -111,10 +121,17 @@ public long getModified() {
 			return dto.modified;
 		}
 
+		public boolean hasNegativeCache(long ttl) {
+			if (!jsonFile.isFile()) {
+				return false;
+			}
+			return dto.notFound && System.currentTimeMillis() - dto.modified < ttl;
+		}
+
 		@Override
 		public String toString() {
 			return "Info [file=" + file + ", etag=" + dto.etag + ", modified=" + Instant.ofEpochMilli(dto.modified)
-				+ ", url=" + url + ", lock=" + lock + "]";
+				+ ", url=" + url + ", lock=" + lock + ", notFound=" + dto.notFound + "]";
 		}
 
 	}

biz.aQute.bndlib/src/aQute/bnd/http/package-info.java

@@ -1,4 +1,4 @@
-@Version("2.1.0")
+@Version("2.2.0")
 package aQute.bnd.http;
 
 import org.osgi.annotation.versioning.Version;

biz.aQute.repository/test/aQute/maven/provider/RemoteRepoTest.java

@@ -78,7 +78,9 @@ public void testBasic() throws Exception {
 		// Fetch it, must exist now
 		//
 
-		assertEquals(State.UPDATED, repo.fetch("foo/bar", localFoobar)
+		// force cache refresh because we cache 404 NOT_FOUND now
+		boolean forceCacheRefresh = true;
+		assertEquals(State.UPDATED, repo.fetch("foo/bar", localFoobar, forceCacheRefresh )
 			.getState());
 		assertTrue(localFoobar.isFile());
 		assertEquals(3L, localFoobar.length());