From e3c35efc16f8df7186f7d718e9e3e531235e0089 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 12 Sep 2022 12:55:34 +0200 Subject: [PATCH] feat(oohelperd): implement caching See https://github.com/ooni/probe/issues/2289 The cache implementation comes from bassosimone/websteps-illustrated. --- internal/cmd/oohelperd/dns.go | 90 +++++++++++++- internal/cmd/oohelperd/handler.go | 9 ++ internal/cmd/oohelperd/http.go | 133 ++++++++++++++++++-- internal/cmd/oohelperd/main.go | 29 +++++ internal/cmd/oohelperd/measure.go | 3 + internal/cmd/oohelperd/tcpconnect.go | 114 ++++++++++++++++- internal/fscache/fscache.go | 177 +++++++++++++++++++++++++++ 7 files changed, 540 insertions(+), 15 deletions(-) create mode 100644 internal/fscache/fscache.go diff --git a/internal/cmd/oohelperd/dns.go b/internal/cmd/oohelperd/dns.go index 0501af4017..ab7d1e9e19 100644 --- a/internal/cmd/oohelperd/dns.go +++ b/internal/cmd/oohelperd/dns.go @@ -6,6 +6,7 @@ package main import ( "context" + "encoding/json" "sync" "time" @@ -24,6 +25,9 @@ type ctrlDNSResult = model.THDNSResult // dnsConfig configures the DNS check. type dnsConfig struct { + // Cache is the MANDATORY cache to use. + Cache model.KeyValueStore + // Domain is the MANDATORY domain to resolve. Domain string @@ -40,12 +44,94 @@ type dnsConfig struct { Wg *sync.WaitGroup } +// dnsCacheKey is the key used inside the DNS cache. +type dnsCacheKey string + +// newDNSCacheKey creates a new dnsCacheKey +func newDNSCacheKey(config *dnsConfig) dnsCacheKey { + return dnsCacheKey(config.Domain) +} + +// asCacheKeyString returns the string used by the underlying cache as key. +func (tck dnsCacheKey) asCacheKeyString() string { + return string(tck) +} + +// dnsCacheEntry is an entry inside the DNS cache. +type dnsCacheEntry struct { + // Created is when we created this entry. + Created time.Time + + // Key is the domain we've resolved. + Key dnsCacheKey + + // Result is the cached result. + Result ctrlDNSResult +} + +// dnsCacheGet gets a list of results from the DNS cache key. +func dnsCacheGet(cache model.KeyValueStore, key dnsCacheKey) ([]*dnsCacheEntry, error) { + rawdata, err := cache.Get(key.asCacheKeyString()) + if err != nil { + return nil, err + } + var values []*dnsCacheEntry + if err := json.Unmarshal(rawdata, &values); err != nil { + return nil, err + } + const dnsCacheExpirationTime = 15 * time.Minute + var out []*dnsCacheEntry + for _, value := range values { + if value == nil || time.Since(value.Created) >= dnsCacheExpirationTime { + continue // this entry is malformed or has expired + } + out = append(out, value) + } + return out, nil +} + +// dnsCacheEntriesFind searches for a given domain inside a set of entries. +func dnsCacheEntriesFind(epv []*dnsCacheEntry, key dnsCacheKey) (*dnsCacheEntry, bool) { + for _, ep := range epv { + if ep != nil && key == ep.Key { + return ep, true + } + } + return nil, false +} + +// dnsCacheWriteBack writes back into the cache. +func dnsCacheWriteBack(cache model.KeyValueStore, key dnsCacheKey, epv []*dnsCacheEntry) error { + rawdata, err := json.Marshal(epv) + if err != nil { + return err + } + return cache.Set(key.asCacheKeyString(), rawdata) +} + // dnsDo performs the DNS check. func dnsDo(ctx context.Context, config *dnsConfig) { + defer config.Wg.Done() + key := newDNSCacheKey(config) + entries, _ := dnsCacheGet(config.Cache, key) // the error is not so relevant + entry, _ := dnsCacheEntriesFind(entries, key) + if entry == nil { + entry = &dnsCacheEntry{ + Created: time.Now(), + Key: key, + Result: dnsDoWithoutCache(ctx, config), + } + entries = append(entries, entry) + } + config.Out <- entry.Result + _ = dnsCacheWriteBack(config.Cache, key, entries) +} + +// dnsDoWithoutCache implements dnsDo. +func dnsDoWithoutCache(ctx context.Context, config *dnsConfig) ctrlDNSResult { const timeout = 4 * time.Second ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - defer config.Wg.Done() reso := config.NewResolver(config.Logger) defer reso.CloseIdleConnections() ol := measurexlite.NewOperationLogger(config.Logger, "DNSLookup %s", config.Domain) @@ -55,7 +141,7 @@ func dnsDo(ctx context.Context, config *dnsConfig) { addrs = []string{} // fix: the old test helper did that } failure := dnsMapFailure(newfailure(err)) - config.Out <- ctrlDNSResult{ + return ctrlDNSResult{ Failure: failure, Addrs: addrs, ASNs: []int64{}, // unused by the TH and not serialized diff --git a/internal/cmd/oohelperd/handler.go b/internal/cmd/oohelperd/handler.go index 41292102f6..65ced39af3 100644 --- a/internal/cmd/oohelperd/handler.go +++ b/internal/cmd/oohelperd/handler.go @@ -23,6 +23,12 @@ type handler struct { // BaseLogger is the MANDATORY logger to use. BaseLogger model.Logger + // DNSCache is the MANDATORY cache for DNS. + DNSCache model.KeyValueStore + + // HTTPCache is the MANDATORY cache for HTTP. + HTTPCache model.KeyValueStore + // Indexer is the MANDATORY atomic integer used to assign an index to requests. Indexer *atomicx.Int64 @@ -40,6 +46,9 @@ type handler struct { // NewTLSHandshaker is the MANDATORY factory for creating a new TLS handshaker. NewTLSHandshaker func(model.Logger) model.TLSHandshaker + + // TCPCache is the MANDATORY cache for TCP. + TCPCache model.KeyValueStore } var _ http.Handler = &handler{} diff --git a/internal/cmd/oohelperd/http.go b/internal/cmd/oohelperd/http.go index 80402f281a..c81b406a70 100644 --- a/internal/cmd/oohelperd/http.go +++ b/internal/cmd/oohelperd/http.go @@ -6,6 +6,8 @@ package main import ( "context" + "encoding/json" + "fmt" "io" "net/http" "strings" @@ -27,6 +29,9 @@ type ctrlHTTPResponse = model.THHTTPRequestResult // httpConfig configures the HTTP check. type httpConfig struct { + // Cache is the MANDATORY HTTP cache to use. + Cache model.KeyValueStore + // Headers is OPTIONAL and contains the request headers we should set. Headers map[string][]string @@ -49,30 +54,143 @@ type httpConfig struct { Wg *sync.WaitGroup } +// httpCacheKey is the key used by the HTTP cache +type httpCacheKey struct { + // Accept is the value of the accept header. + Accept string + + // AcceptLanguage is the value of the accept-language header. + AcceptLanguage string + + // MaxAcceptableBody is the maximum acceptable body size. + MaxAcceptableBody int64 + + // URL is the MANDATORY URL to measure. + URL string + + // UserAgent is the value of the user-agent header. + UserAgent string +} + +// newHTTPCacheKey creates a new httpCacheKey from the given [config]. +func newHTTPCacheKey(config *httpConfig) *httpCacheKey { + headers := http.Header(config.Headers) + if headers == nil { + headers = http.Header{} + } + return &httpCacheKey{ + Accept: headers.Get("accept"), + AcceptLanguage: headers.Get("accept-language"), + MaxAcceptableBody: config.MaxAcceptableBody, + URL: config.URL, + UserAgent: headers.Get("user-agent"), + } +} + +// asCacheKeyString returns the string used by the underlying cache as key. +func (tck *httpCacheKey) asCacheKeyString() string { + return fmt.Sprintf("%+v", tck) +} + +// Equals returns whether two instances are equal +func (tck *httpCacheKey) Equals(other *httpCacheKey) bool { + return tck.asCacheKeyString() == other.asCacheKeyString() +} + +// httpCacheEntry is an entry inside the HTTP cache. +type httpCacheEntry struct { + // Created is when we created this entry. + Created time.Time + + // Key identifies this cache entry. + Key httpCacheKey + + // Result is the cached result. + Result ctrlHTTPResponse +} + +// httpCacheGet gets a list of results from the HTTP cache key. +func httpCacheGet(cache model.KeyValueStore, key *httpCacheKey) ([]*httpCacheEntry, error) { + rawdata, err := cache.Get(key.asCacheKeyString()) + if err != nil { + return nil, err + } + var values []*httpCacheEntry + if err := json.Unmarshal(rawdata, &values); err != nil { + return nil, err + } + const tcpCacheExpirationTime = 15 * time.Minute + var out []*httpCacheEntry + for _, value := range values { + if value == nil || time.Since(value.Created) >= tcpCacheExpirationTime { + continue // this entry is malformed or has expired + } + out = append(out, value) + } + return out, nil +} + +// httpCacheEntriesFind searches for a given domain inside a set of entries. +func httpCacheEntriesFind(epv []*httpCacheEntry, key *httpCacheKey) (*httpCacheEntry, bool) { + for _, ep := range epv { + if ep != nil && key.Equals(&ep.Key) { + return ep, true + } + } + return nil, false +} + +// httpCacheWriteBack writes back into the cache. +func httpCacheWriteBack(cache model.KeyValueStore, key *httpCacheKey, epv []*httpCacheEntry) error { + rawdata, err := json.Marshal(epv) + if err != nil { + return err + } + return cache.Set(key.asCacheKeyString(), rawdata) +} + // httpDo performs the HTTP check. func httpDo(ctx context.Context, config *httpConfig) { + defer config.Wg.Done() + key := newHTTPCacheKey(config) + entries, _ := httpCacheGet(config.Cache, key) // the error is not so relevant + entry, _ := httpCacheEntriesFind(entries, key) + if entry == nil { + entry = &httpCacheEntry{ + Created: time.Now(), + Key: *key, + Result: httpDoWithoutCache(ctx, config), + } + entries = append(entries, entry) + } + config.Out <- entry.Result + _ = httpCacheWriteBack(config.Cache, key, entries) +} + +// httpDoWithoutCache implements httpDo +func httpDoWithoutCache(ctx context.Context, config *httpConfig) ctrlHTTPResponse { ol := measurexlite.NewOperationLogger(config.Logger, "GET %s", config.URL) const timeout = 15 * time.Second ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - defer config.Wg.Done() req, err := http.NewRequestWithContext(ctx, "GET", config.URL, nil) if err != nil { + ol.Stop(err) // fix: emit -1 like the old test helper does - config.Out <- ctrlHTTPResponse{ + return ctrlHTTPResponse{ BodyLength: -1, Failure: httpMapFailure(err), Title: "", Headers: map[string]string{}, StatusCode: -1, } - ol.Stop(err) - return } // The original test helper failed with extra headers while here // we're implementing (for now?) a more liberal approach. for k, vs := range config.Headers { switch strings.ToLower(k) { + // WARNING: if you enable more headers here then you must modify + // the caching code to use thise headers for the cache key case "user-agent", "accept", "accept-language": for _, v := range vs { req.Header.Add(k, v) @@ -83,16 +201,15 @@ func httpDo(ctx context.Context, config *httpConfig) { defer clnt.CloseIdleConnections() resp, err := clnt.Do(req) if err != nil { + ol.Stop(err) // fix: emit -1 like the old test helper does - config.Out <- ctrlHTTPResponse{ + return ctrlHTTPResponse{ BodyLength: -1, Failure: httpMapFailure(err), Title: "", Headers: map[string]string{}, StatusCode: -1, } - ol.Stop(err) - return } defer resp.Body.Close() headers := make(map[string]string) @@ -102,7 +219,7 @@ func httpDo(ctx context.Context, config *httpConfig) { reader := &io.LimitedReader{R: resp.Body, N: config.MaxAcceptableBody} data, err := netxlite.ReadAllContext(ctx, reader) ol.Stop(err) - config.Out <- ctrlHTTPResponse{ + return ctrlHTTPResponse{ BodyLength: int64(len(data)), Failure: httpMapFailure(err), StatusCode: int64(resp.StatusCode), diff --git a/internal/cmd/oohelperd/main.go b/internal/cmd/oohelperd/main.go index 8760f09adc..dbdbd1042a 100644 --- a/internal/cmd/oohelperd/main.go +++ b/internal/cmd/oohelperd/main.go @@ -6,11 +6,13 @@ import ( "flag" "net" "net/http" + "path/filepath" "sync" "time" "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/atomicx" + "github.com/ooni/probe-cli/v3/internal/fscache" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" "github.com/ooni/probe-cli/v3/internal/runtimex" @@ -50,14 +52,36 @@ func main() { true: log.DebugLevel, false: log.InfoLevel, } + datadir := flag.String("datadir", "/var/lib/oohelperd", "Directory where to store data") prometheus := flag.String("prometheus", "127.0.0.1:9091", "Prometheus endpoint") debug := flag.Bool("debug", false, "Toggle debug mode") + flag.Parse() log.SetLevel(logmap[*debug]) defer srvCancel() + + dnscache := fscache.New(filepath.Join(*datadir, "cache", "dns")) + endpointcache := fscache.New(filepath.Join(*datadir, "cache", "endpoint")) + httpcache := fscache.New(filepath.Join(*datadir, "cache", "http")) + go func() { + ticker := time.NewTimer(15 * time.Minute) + for { + select { + case <-srvCtx.Done(): + return + case <-ticker.C: + dnscache.Trim() + endpointcache.Trim() + httpcache.Trim() + } + } + }() + mux := http.NewServeMux() mux.Handle("/", &handler{ BaseLogger: log.Log, + DNSCache: dnscache, + HTTPCache: httpcache, Indexer: &atomicx.Int64{}, MaxAcceptableBody: maxAcceptableBody, NewClient: func(logger model.Logger) model.HTTPClient { @@ -88,18 +112,23 @@ func main() { NewTLSHandshaker: func(logger model.Logger) model.TLSHandshaker { return netxlite.NewTLSHandshakerStdlib(logger) }, + TCPCache: endpointcache, }) + srv := &http.Server{Addr: *endpoint, Handler: mux} listener, err := net.Listen("tcp", *endpoint) runtimex.PanicOnError(err, "net.Listen failed") srvAddr <- listener.Addr().String() srvWg.Add(1) go srv.Serve(listener) + promMux := http.NewServeMux() promMux.Handle("/metrics", promhttp.Handler()) promSrv := &http.Server{Addr: *prometheus, Handler: promMux} go promSrv.ListenAndServe() + <-srvCtx.Done() + shutdown(srv) shutdown(promSrv) listener.Close() diff --git a/internal/cmd/oohelperd/measure.go b/internal/cmd/oohelperd/measure.go index 0d57651f6b..86d9adb2f9 100644 --- a/internal/cmd/oohelperd/measure.go +++ b/internal/cmd/oohelperd/measure.go @@ -44,6 +44,7 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp if net.ParseIP(URL.Hostname()) == nil { wg.Add(1) go dnsDo(ctx, &dnsConfig{ + Cache: config.DNSCache, Domain: URL.Hostname(), Logger: logger, NewResolver: config.NewResolver, @@ -85,6 +86,7 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp wg.Add(1) go tcpDo(ctx, &tcpConfig{ Address: endpoint.Addr, + Cache: config.TCPCache, EnableTLS: endpoint.TLS, Endpoint: endpoint.Epnt, Logger: logger, @@ -100,6 +102,7 @@ func measure(ctx context.Context, config *handler, creq *ctrlRequest) (*ctrlResp httpch := make(chan ctrlHTTPResponse, 1) wg.Add(1) go httpDo(ctx, &httpConfig{ + Cache: config.HTTPCache, Headers: creq.HTTPRequestHeaders, Logger: logger, MaxAcceptableBody: config.MaxAcceptableBody, diff --git a/internal/cmd/oohelperd/tcpconnect.go b/internal/cmd/oohelperd/tcpconnect.go index 1db8814b64..af9bbd17e5 100644 --- a/internal/cmd/oohelperd/tcpconnect.go +++ b/internal/cmd/oohelperd/tcpconnect.go @@ -7,6 +7,8 @@ package main import ( "context" "crypto/tls" + "encoding/json" + "fmt" "sync" "time" @@ -41,6 +43,9 @@ type tcpConfig struct { // Address is the MANDATORY address to measure. Address string + // Cache is the MANDATORY TCP cache to use. + Cache model.KeyValueStore + // EnableTLS OPTIONALLY enables TLS. EnableTLS bool @@ -66,21 +71,119 @@ type tcpConfig struct { Wg *sync.WaitGroup } +// tcpCacheKey is the key used by the TCP cache +type tcpCacheKey struct { + // EnableTLS OPTIONALLY enables TLS. + EnableTLS bool + + // Endpoint is the MANDATORY endpoint to connect to. + Endpoint string + + // URLHostname is the MANDATORY URL.Hostname() to use. + URLHostname string +} + +// newTCPCacheKey creates a new tcpCacheKey from the given [config]. +func newTCPCacheKey(config *tcpConfig) *tcpCacheKey { + return &tcpCacheKey{ + EnableTLS: config.EnableTLS, + Endpoint: config.Endpoint, + URLHostname: config.URLHostname, + } +} + +// asCacheKeyString returns the string used by the underlying cache as key. +func (tck *tcpCacheKey) asCacheKeyString() string { + return fmt.Sprintf("%+v", tck) +} + +// Equals returns whether two instances are equal +func (tck *tcpCacheKey) Equals(other *tcpCacheKey) bool { + return tck.EnableTLS == other.EnableTLS && tck.Endpoint == other.Endpoint && + tck.URLHostname == other.URLHostname +} + +// tcpCacheEntry is an entry inside the TCP cache. +type tcpCacheEntry struct { + // Created is when we created this entry. + Created time.Time + + // Key identifies this cache entry. + Key tcpCacheKey + + // Result is the cached result. + Result *tcpResultPair +} + +// tcpCacheGet gets a list of results from the TCP cache key. +func tcpCacheGet(cache model.KeyValueStore, key *tcpCacheKey) ([]*tcpCacheEntry, error) { + rawdata, err := cache.Get(key.asCacheKeyString()) + if err != nil { + return nil, err + } + var values []*tcpCacheEntry + if err := json.Unmarshal(rawdata, &values); err != nil { + return nil, err + } + const tcpCacheExpirationTime = 15 * time.Minute + var out []*tcpCacheEntry + for _, value := range values { + if value == nil || value.Result == nil || time.Since(value.Created) >= tcpCacheExpirationTime { + continue // this entry is malformed or has expired + } + out = append(out, value) + } + return out, nil +} + +// tcpCacheEntriesFind searches for a given domain inside a set of entries. +func tcpCacheEntriesFind(epv []*tcpCacheEntry, key *tcpCacheKey) (*tcpCacheEntry, bool) { + for _, ep := range epv { + if ep != nil && key.Equals(&ep.Key) { + return ep, true + } + } + return nil, false +} + +// tcpCacheWriteBack writes back into the cache. +func tcpCacheWriteBack(cache model.KeyValueStore, key *tcpCacheKey, epv []*tcpCacheEntry) error { + rawdata, err := json.Marshal(epv) + if err != nil { + return err + } + return cache.Set(key.asCacheKeyString(), rawdata) +} + // tcpDo performs the TCP check. func tcpDo(ctx context.Context, config *tcpConfig) { + defer config.Wg.Done() + key := newTCPCacheKey(config) + entries, _ := tcpCacheGet(config.Cache, key) // the error is not so relevant + entry, _ := tcpCacheEntriesFind(entries, key) + if entry == nil { + entry = &tcpCacheEntry{ + Created: time.Now(), + Key: *key, + Result: tcpDoWithoutCache(ctx, config), + } + entries = append(entries, entry) + } + config.Out <- entry.Result + _ = tcpCacheWriteBack(config.Cache, key, entries) +} + +// tcpDoWithoutCache implements tcpDo +func tcpDoWithoutCache(ctx context.Context, config *tcpConfig) *tcpResultPair { const timeout = 15 * time.Second ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - defer config.Wg.Done() out := &tcpResultPair{ Address: config.Address, Endpoint: config.Endpoint, TCP: model.THTCPConnectResult{}, TLS: nil, // means: not measured } - defer func() { - config.Out <- out - }() ol := measurexlite.NewOperationLogger( config.Logger, "TCPConnect %s EnableTLS=%v SNI=%s", @@ -96,7 +199,7 @@ func tcpDo(ctx context.Context, config *tcpConfig) { defer measurexlite.MaybeClose(conn) if err != nil || !config.EnableTLS { ol.Stop(err) - return + return out } tlsConfig := &tls.Config{ NextProtos: []string{"h2", "http/1.1"}, @@ -112,6 +215,7 @@ func tcpDo(ctx context.Context, config *tcpConfig) { Failure: newfailure(err), } measurexlite.MaybeClose(tlsConn) + return out } // tcpMapFailure attempts to map netxlite failures to the strings diff --git a/internal/fscache/fscache.go b/internal/fscache/fscache.go new file mode 100644 index 0000000000..c1a171329c --- /dev/null +++ b/internal/fscache/fscache.go @@ -0,0 +1,177 @@ +// Package fscache implements an on-disk cache. +package fscache + +// +// FSCache +// +// Contains a file system cache derived from golang build cache. +// + +import ( + "bytes" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/rogpeppe/go-internal/lockedfile" +) + +// Cache provides a simple cache-on-filesystem functionality. +type Cache struct { + // dirpath is the cache directory path + dirpath string + + // timeNow allows mocking time.Now for testing + timeNow func() time.Time +} + +var _ model.KeyValueStore = &Cache{} + +// New creates a new simpleCache instance. +func New(dirpath string) *Cache { + return &Cache{ + dirpath: dirpath, + timeNow: time.Now, + } +} + +var _ model.KeyValueStore = &Cache{} + +// Get implements model.KeyValueStore.Get. +func (sc *Cache) Get(key string) ([]byte, error) { + _, fpath := sc.fsmap(key) + return lockedfile.Read(fpath) +} + +// Set implements model.KeyValueStore.Set. +func (sc *Cache) Set(key string, value []byte) error { + dpath, fpath := sc.fsmap(key) + const dperms = 0700 + if err := os.MkdirAll(dpath, dperms); err != nil { + return err + } + const fperms = 0600 + if err := lockedfile.Write(fpath, bytes.NewReader(value), fperms); err != nil { + return err + } + sc.maybeMarkAsUsed(fpath) + return nil +} + +// fsmap maps a given key to a directory and a file paths. +func (sc *Cache) fsmap(key string) (dpath, fpath string) { + hs := sha256.Sum256([]byte(key)) + dpath = filepath.Join(sc.dirpath, fmt.Sprintf("%02x", hs[0])) + fpath = filepath.Join(dpath, fmt.Sprintf("%02x-d", hs)) + return +} + +// Time constants for cache expiration. +// +// We set the mtime on a cache file on each use, but at most one per cacheMtimeInterval, +// to avoid causing many unnecessary inode updates. The mtimes therefore +// roughly reflect "time of last use" but may in fact be older. +// +// We scan the cache for entries to delete at most once per cacheTrimInterval. +// +// When we do scan the cache, we delete entries that have not been used for +// at least cacheTrimLimit. This code was adapted from Go internals and the original +// code has numbers based on statistics. We should do the same for OONI. +// +// SPDX-License-Identifier: BSD-3-Clause +// +// Source: https://github.com/rogpeppe/go-internal/commit/797a764460877f0a4bd570a61d60d10815e728e6 +const ( + cacheMtimeInterval = 15 * time.Minute + cacheTrimInterval = 45 * time.Minute + cacheTrimLimit = 2 * time.Hour +) + +// maybeMarkAsUsed makes a best-effort attempt to update mtime on file, +// so that mtime reflects cache access time. +// +// Because the reflection only needs to be approximate, +// and to reduce the amount of disk activity caused by using +// cache entries, maybeMarkAsUsed only updates the mtime if the current +// mtime is more than an mtimeInterval old. This heuristic eliminates +// nearly all of the mtime updates that would otherwise happen, +// while still keeping the mtimes useful for cache trimming. +// +// SPDX-License-Identifier: BSD-3-Clause +// +// Source: https://github.com/rogpeppe/go-internal/commit/797a764460877f0a4bd570a61d60d10815e728e6 +func (sc *Cache) maybeMarkAsUsed(file string) { + info, err := os.Stat(file) + now := sc.timeNow() + if err == nil && now.Sub(info.ModTime()) < cacheMtimeInterval { + return + } + os.Chtimes(file, now, now) +} + +// Trim removes old cache entries that are likely not to be reused. +// +// SPDX-License-Identifier: BSD-3-Clause +// +// Source: https://github.com/rogpeppe/go-internal/commit/797a764460877f0a4bd570a61d60d10815e728e6 +func (sc *Cache) Trim() { + now := sc.timeNow() + + trimfilepath := filepath.Join(sc.dirpath, "trim.txt") + + // We maintain in dir/trim.txt the time of the last completed cache trim. + // If the cache has been trimmed recently enough, do nothing. + // This is the common case. + data, _ := os.ReadFile(trimfilepath) + lt, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err == nil && now.Sub(time.Unix(lt, 0)) < cacheTrimInterval { + return + } + + // Trim each of the 256 subdirectories. + // We subtract an additional mtimeInterval + // to account for the imprecision of our "last used" mtimes. + cutoff := now.Add(-cacheTrimLimit - cacheMtimeInterval) + for i := 0; i < 256; i++ { + subdir := filepath.Join(sc.dirpath, fmt.Sprintf("%02x", i)) + sc.trimSubdir(subdir, cutoff) + } + + os.WriteFile(trimfilepath, []byte(fmt.Sprintf("%d", now.Unix())), 0666) +} + +// trimSubdir trims a single cache subdirectory. +// +// SPDX-License-Identifier: BSD-3-Clause +// +// Source: https://github.com/rogpeppe/go-internal/commit/797a764460877f0a4bd570a61d60d10815e728e6 +func (sc *Cache) trimSubdir(subdir string, cutoff time.Time) { + // Read all directory entries from subdir before removing + // any files, in case removing files invalidates the file offset + // in the directory scan. Also, ignore error from df.Readdirnames, + // because we don't care about reporting the error and we still + // want to process any entries found before the error. + df, err := os.Open(subdir) + if err != nil { + return + } + names, _ := df.Readdirnames(-1) + df.Close() + + for _, name := range names { + // Remove only cache entries (xxxx-a and xxxx-d). + if !strings.HasSuffix(name, "-a") && !strings.HasSuffix(name, "-d") { + continue + } + entry := filepath.Join(subdir, name) + info, err := os.Stat(entry) + if err == nil && info.ModTime().Before(cutoff) { + os.Remove(entry) + } + } +}