feat(oohelperd): implement caching

internal/cmd/oohelperd/dns.go

@@ -6,6 +6,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"sync"
 	"time"
 
@@ -24,6 +25,9 @@ type ctrlDNSResult = model.THDNSResult
 
 // dnsConfig configures the DNS check.
 type dnsConfig struct {
+	// Cache is the MANDATORY cache to use.
+	Cache model.KeyValueStore
+
 	// Domain is the MANDATORY domain to resolve.
 	Domain string
 
@@ -40,12 +44,94 @@ type dnsConfig struct {
 	Wg *sync.WaitGroup
 }
 
+// dnsCacheKey is the key used inside the DNS cache.
+type dnsCacheKey string
+
+// newDNSCacheKey creates a new dnsCacheKey
+func newDNSCacheKey(config *dnsConfig) dnsCacheKey {
+	return dnsCacheKey(config.Domain)
+}
+
+// asCacheKeyString returns the string used by the underlying cache as key.
+func (tck dnsCacheKey) asCacheKeyString() string {
+	return string(tck)
+}
+
+// dnsCacheEntry is an entry inside the DNS cache.
+type dnsCacheEntry struct {
+	// Created is when we created this entry.
+	Created time.Time
+
+	// Key is the domain we've resolved.
+	Key dnsCacheKey
+
+	// Result is the cached result.
+	Result ctrlDNSResult
+}
+
+// dnsCacheGet gets a list of results from the DNS cache key.
+func dnsCacheGet(cache model.KeyValueStore, key dnsCacheKey) ([]*dnsCacheEntry, error) {
+	rawdata, err := cache.Get(key.asCacheKeyString())
+	if err != nil {
+		return nil, err
+	}
+	var values []*dnsCacheEntry
+	if err := json.Unmarshal(rawdata, &values); err != nil {
+		return nil, err
+	}
+	const dnsCacheExpirationTime = 15 * time.Minute
+	var out []*dnsCacheEntry
+	for _, value := range values {
+		if value == nil || time.Since(value.Created) >= dnsCacheExpirationTime {
+			continue // this entry is malformed or has expired
+		}
+		out = append(out, value)
+	}
+	return out, nil
+}
+
+// dnsCacheEntriesFind searches for a given domain inside a set of entries.
+func dnsCacheEntriesFind(epv []*dnsCacheEntry, key dnsCacheKey) (*dnsCacheEntry, bool) {
+	for _, ep := range epv {
+		if ep != nil && key == ep.Key {
+			return ep, true
+		}
+	}
+	return nil, false
+}
+
+// dnsCacheWriteBack writes back into the cache.
+func dnsCacheWriteBack(cache model.KeyValueStore, key dnsCacheKey, epv []*dnsCacheEntry) error {
+	rawdata, err := json.Marshal(epv)
+	if err != nil {
+		return err
+	}
+	return cache.Set(key.asCacheKeyString(), rawdata)
+}
+
 // dnsDo performs the DNS check.
 func dnsDo(ctx context.Context, config *dnsConfig) {
+	defer config.Wg.Done()
+	key := newDNSCacheKey(config)
+	entries, _ := dnsCacheGet(config.Cache, key) // the error is not so relevant
+	entry, _ := dnsCacheEntriesFind(entries, key)
+	if entry == nil {
+		entry = &dnsCacheEntry{
+			Created: time.Now(),
+			Key:     key,
+			Result:  dnsDoWithoutCache(ctx, config),
+		}
+		entries = append(entries, entry)
+	}
+	config.Out <- entry.Result
+	_ = dnsCacheWriteBack(config.Cache, key, entries)
+}
+
+// dnsDoWithoutCache implements dnsDo.
+func dnsDoWithoutCache(ctx context.Context, config *dnsConfig) ctrlDNSResult {
 	const timeout = 4 * time.Second
 	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	defer config.Wg.Done()
 	reso := config.NewResolver(config.Logger)
 	defer reso.CloseIdleConnections()
 	ol := measurexlite.NewOperationLogger(config.Logger, "DNSLookup %s", config.Domain)
@@ -55,7 +141,7 @@ func dnsDo(ctx context.Context, config *dnsConfig) {
 		addrs = []string{} // fix: the old test helper did that
 	}
 	failure := dnsMapFailure(newfailure(err))
-	config.Out <- ctrlDNSResult{
+	return ctrlDNSResult{
 		Failure: failure,
 		Addrs:   addrs,
 		ASNs:    []int64{}, // unused by the TH and not serialized

internal/cmd/oohelperd/handler.go

@@ -23,6 +23,12 @@ type handler struct {
 	// BaseLogger is the MANDATORY logger to use.
 	BaseLogger model.Logger
 
+	// DNSCache is the MANDATORY cache for DNS.
+	DNSCache model.KeyValueStore
+
+	// HTTPCache is the MANDATORY cache for HTTP.
+	HTTPCache model.KeyValueStore
+
 	// Indexer is the MANDATORY atomic integer used to assign an index to requests.
 	Indexer *atomicx.Int64
 
@@ -40,6 +46,9 @@ type handler struct {
 
 	// NewTLSHandshaker is the MANDATORY factory for creating a new TLS handshaker.
 	NewTLSHandshaker func(model.Logger) model.TLSHandshaker
+
+	// TCPCache is the MANDATORY cache for TCP.
+	TCPCache model.KeyValueStore
 }
 
 var _ http.Handler = &handler{}

internal/cmd/oohelperd/http.go

@@ -6,6 +6,8 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"strings"
@@ -27,6 +29,9 @@ type ctrlHTTPResponse = model.THHTTPRequestResult
 
 // httpConfig configures the HTTP check.
 type httpConfig struct {
+	// Cache is the MANDATORY HTTP cache to use.
+	Cache model.KeyValueStore
+
 	// Headers is OPTIONAL and contains the request headers we should set.
 	Headers map[string][]string
 
@@ -49,30 +54,143 @@ type httpConfig struct {
 	Wg *sync.WaitGroup
 }
 
+// httpCacheKey is the key used by the HTTP cache
+type httpCacheKey struct {
+	// Accept is the value of the accept header.
+	Accept string
+
+	// AcceptLanguage is the value of the accept-language header.
+	AcceptLanguage string
+
+	// MaxAcceptableBody is the maximum acceptable body size.
+	MaxAcceptableBody int64
+
+	// URL is the MANDATORY URL to measure.
+	URL string
+
+	// UserAgent is the value of the user-agent header.
+	UserAgent string
+}
+
+// newHTTPCacheKey creates a new httpCacheKey from the given [config].
+func newHTTPCacheKey(config *httpConfig) *httpCacheKey {
+	headers := http.Header(config.Headers)
+	if headers == nil {
+		headers = http.Header{}
+	}
+	return &httpCacheKey{
+		Accept:            headers.Get("accept"),
+		AcceptLanguage:    headers.Get("accept-language"),
+		MaxAcceptableBody: config.MaxAcceptableBody,
+		URL:               config.URL,
+		UserAgent:         headers.Get("user-agent"),
+	}
+}
+
+// asCacheKeyString returns the string used by the underlying cache as key.
+func (tck *httpCacheKey) asCacheKeyString() string {
+	return fmt.Sprintf("%+v", tck)
+}
+
+// Equals returns whether two instances are equal
+func (tck *httpCacheKey) Equals(other *httpCacheKey) bool {
+	return tck.asCacheKeyString() == other.asCacheKeyString()
+}
+
+// httpCacheEntry is an entry inside the HTTP cache.
+type httpCacheEntry struct {
+	// Created is when we created this entry.
+	Created time.Time
+
+	// Key identifies this cache entry.
+	Key httpCacheKey
+
+	// Result is the cached result.
+	Result ctrlHTTPResponse
+}
+
+// httpCacheGet gets a list of results from the HTTP cache key.
+func httpCacheGet(cache model.KeyValueStore, key *httpCacheKey) ([]*httpCacheEntry, error) {
+	rawdata, err := cache.Get(key.asCacheKeyString())
+	if err != nil {
+		return nil, err
+	}
+	var values []*httpCacheEntry
+	if err := json.Unmarshal(rawdata, &values); err != nil {
+		return nil, err
+	}
+	const tcpCacheExpirationTime = 15 * time.Minute
+	var out []*httpCacheEntry
+	for _, value := range values {
+		if value == nil || time.Since(value.Created) >= tcpCacheExpirationTime {
+			continue // this entry is malformed or has expired
+		}
+		out = append(out, value)
+	}
+	return out, nil
+}
+
+// httpCacheEntriesFind searches for a given domain inside a set of entries.
+func httpCacheEntriesFind(epv []*httpCacheEntry, key *httpCacheKey) (*httpCacheEntry, bool) {
+	for _, ep := range epv {
+		if ep != nil && key.Equals(&ep.Key) {
+			return ep, true
+		}
+	}
+	return nil, false
+}
+
+// httpCacheWriteBack writes back into the cache.
+func httpCacheWriteBack(cache model.KeyValueStore, key *httpCacheKey, epv []*httpCacheEntry) error {
+	rawdata, err := json.Marshal(epv)
+	if err != nil {
+		return err
+	}
+	return cache.Set(key.asCacheKeyString(), rawdata)
+}
+
 // httpDo performs the HTTP check.
 func httpDo(ctx context.Context, config *httpConfig) {
+	defer config.Wg.Done()
+	key := newHTTPCacheKey(config)
+	entries, _ := httpCacheGet(config.Cache, key) // the error is not so relevant
+	entry, _ := httpCacheEntriesFind(entries, key)
+	if entry == nil {
+		entry = &httpCacheEntry{
+			Created: time.Now(),
+			Key:     *key,
+			Result:  httpDoWithoutCache(ctx, config),
+		}
+		entries = append(entries, entry)
+	}
+	config.Out <- entry.Result
+	_ = httpCacheWriteBack(config.Cache, key, entries)
+}
+
+// httpDoWithoutCache implements httpDo
+func httpDoWithoutCache(ctx context.Context, config *httpConfig) ctrlHTTPResponse {
 	ol := measurexlite.NewOperationLogger(config.Logger, "GET %s", config.URL)
 	const timeout = 15 * time.Second
 	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	defer config.Wg.Done()
 	req, err := http.NewRequestWithContext(ctx, "GET", config.URL, nil)
 	if err != nil {
+		ol.Stop(err)
 		// fix: emit -1 like the old test helper does
-		config.Out <- ctrlHTTPResponse{
+		return ctrlHTTPResponse{
 			BodyLength: -1,
 			Failure:    httpMapFailure(err),
 			Title:      "",
 			Headers:    map[string]string{},
 			StatusCode: -1,
 		}
-		ol.Stop(err)
-		return
 	}
 	// The original test helper failed with extra headers while here
 	// we're implementing (for now?) a more liberal approach.
 	for k, vs := range config.Headers {
 		switch strings.ToLower(k) {
+		// WARNING: if you enable more headers here then you must modify
+		// the caching code to use thise headers for the cache key
 		case "user-agent", "accept", "accept-language":
 			for _, v := range vs {
 				req.Header.Add(k, v)
@@ -83,16 +201,15 @@ func httpDo(ctx context.Context, config *httpConfig) {
 	defer clnt.CloseIdleConnections()
 	resp, err := clnt.Do(req)
 	if err != nil {
+		ol.Stop(err)
 		// fix: emit -1 like the old test helper does
-		config.Out <- ctrlHTTPResponse{
+		return ctrlHTTPResponse{
 			BodyLength: -1,
 			Failure:    httpMapFailure(err),
 			Title:      "",
 			Headers:    map[string]string{},
 			StatusCode: -1,
 		}
-		ol.Stop(err)
-		return
 	}
 	defer resp.Body.Close()
 	headers := make(map[string]string)
@@ -102,7 +219,7 @@ func httpDo(ctx context.Context, config *httpConfig) {
 	reader := &io.LimitedReader{R: resp.Body, N: config.MaxAcceptableBody}
 	data, err := netxlite.ReadAllContext(ctx, reader)
 	ol.Stop(err)
-	config.Out <- ctrlHTTPResponse{
+	return ctrlHTTPResponse{
 		BodyLength: int64(len(data)),
 		Failure:    httpMapFailure(err),
 		StatusCode: int64(resp.StatusCode),