diff --git a/.travis.yml b/.travis.yml index c6092f786c..c06fd94e6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ before_install: - sudo apt-get install --allow-downgrades -y docker-buildx-plugin mongodb-org=${MONGODB} mongodb-org-database=${MONGODB} mongodb-org-server=${MONGODB} mongodb-mongosh=${MONGOSH} mongodb-org-mongos=${MONGODB} mongodb-org-tools - mkdir -p /var/ramfs/mongodb/data - /usr/bin/mongod --dbpath /var/ramfs/mongodb/data --bind_ip 127.0.0.1 --replSet rs0 --logpath /var/ramfs/mongodb/mongod.log &> /dev/null & - - until nc -z localhost 27017; do echo Waiting for MongoDB; sleep 1; done + - until nc -z 127.0.0.1 27017; do echo Waiting for MongoDB; sleep 1; done - /usr/bin/mongosh --eval 'rs.initiate(); while (rs.status().startupStatus || (rs.status().hasOwnProperty("myState") && rs.status().myState != 1)) { printjson( rs.status() ); sleep(1000); }; printjson( rs.status() );' - echo -e "machine github.com\n login $GITHUB_TOKEN" > ~/.netrc diff --git a/Makefile b/Makefile index d4da95207d..d92862e5bd 100644 --- a/Makefile +++ b/Makefile @@ -240,7 +240,7 @@ vet: tmp @echo "go vet ./..." @cd $(ROOT_DIRECTORY) && \ { [ -z `go env GOWORK` ] || GOWORK_FLAGS=-mod=readonly; } && \ - go vet $${GOWORK_FLAGS:-} ./... > _tmp/govet.out 2>&1 || \ + $(TIMING_CMD) go vet $${GOWORK_FLAGS:-} ./... > _tmp/govet.out 2>&1 || \ (diff .govetignore _tmp/govet.out && exit 1) vet-ignore: diff --git a/README.md b/README.md index 30bc439961..7600c22fd6 100644 --- a/README.md +++ b/README.md @@ -159,3 +159,47 @@ Review all pending changes to all dependencies. If any changes could have a nega Ensure the `ci-build` and `ci-test` Makefile targets pass using the target Golang version. If you previously noted any changes or issues of concern, perform any explicit tests necessary. + +## Prometheus Metrics + +See source files for further details about and usage of each metric. + +### Summary Store + +* `tidepool_summary_queue_lag` - (histogram) - the current summary queue lag, in minutes +* `tidepool_summary_queue_length` - (gauge) - the current summary queue length, in number of summaries + +### C2C + +#### Abbott + +* `tidepool_abbott_api_request_count` - (counter) - Abbott API request count, sorted by method, path, and status +* `tidepool_abbott_api_request_duration_seconds` - (histogram) - Abbott API duration of each request, in seconds, sorted by method, path, and status + +#### Dexcom + +* `tidepool_dexcom_api_request_count` - (counter) - Dexcom API request count, sorted by method, path, and status +* `tidepool_dexcom_api_request_duration_seconds` - (histogram) - Dexcom API duration of each request, in seconds, sorted by method, path, and status +* `tidepool_dexcom_api_request_time_seconds` - (histogram) - Dexcom API duration of each request, as reported in the "request-time" response header from Dexcom, in seconds, sorted by method, path, and status + +#### Oura + +* `tidepool_oura_api_request_count` - (counter) - Oura API request count, sorted by method, path, and status +* `tidepool_oura_api_request_duration_seconds` - (histogram) - Oura API duration of each request, in seconds, sorted by method, path, and status + +### Task + +#### Queue + +* `tidepool_task_workers_total` - (gauge) - configured number of task queue workers, sorted by queue (per config) +* `tidepool_task_workers_available` - (gauge) - number of available task queue workers, sorted by queue +* `tidepool_task_runner_not_found_total` - (counter) - total number of task runs with no registered runner for the task type, sorted by type (ideally zero) +* `tidepool_task_run_duration_seconds` - (histogram) - duration of task runs in seconds, sorted by type +* `tidepool_task_runner_timeout_exceeded_total` - (counter) - total number of task runs that exceeded the runner timeout, sorted by type and disposition ("blocked", "recovered") (ideally zero) +* `tidepool_task_run_panic_total` - (counter) - total number of task runs that panicked, sorted by type (ideally 0) + +#### Store + +* `tidepool_task_type_state_total` - (counter) - total number of tasks run, sorted by type and state +* `tidepool_task_type_lost_completion_total` - (counter) - total number of task completions dropped because the claim-token compare-and-swap missed, sorted by type (ideally low-ish) +* `tidepool_task_type_revision_mismatch_total` - (counter) - total number of task revisions that do not match the task revision in the database, sorted by type (ideally zero) diff --git a/auth/client/external.go b/auth/client/external.go index d3901d2bdb..2292964ac7 100644 --- a/auth/client/external.go +++ b/auth/client/external.go @@ -3,7 +3,6 @@ package client import ( "context" "net/http" - "strconv" "sync" "time" @@ -16,6 +15,7 @@ import ( "github.com/tidepool-org/platform/auth" "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/config" + "github.com/tidepool-org/platform/duration" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/permission" @@ -380,13 +380,10 @@ func (l *externalConfigReporterLoader) Load(cfg *ExternalConfig) error { return err } cfg.ServerSessionTokenSecret = l.Reporter.GetWithDefault("server_session_token_secret", "") - if serverSessionTokenTimeoutString, err := l.Reporter.Get("server_session_token_timeout"); err == nil { - var serverSessionTokenTimeoutInteger int64 - serverSessionTokenTimeoutInteger, err = strconv.ParseInt(serverSessionTokenTimeoutString, 10, 0) - if err != nil { - return errors.New("server session token timeout is invalid") - } - cfg.ServerSessionTokenTimeout = time.Duration(serverSessionTokenTimeoutInteger) * time.Second + if serverSessionTokenTimeout, err := duration.Parse(l.Reporter.GetWithDefault("server_session_token_timeout", cfg.ServerSessionTokenTimeout.String()), time.Second); err != nil { + return errors.New("server session token timeout is invalid") + } else { + cfg.ServerSessionTokenTimeout = serverSessionTokenTimeout } return nil diff --git a/auth/service/api/v1/metrics.go b/auth/service/api/v1/metrics.go new file mode 100644 index 0000000000..4f8f9382ee --- /dev/null +++ b/auth/service/api/v1/metrics.go @@ -0,0 +1,21 @@ +package v1 + +import ( + "net/http" + + "github.com/ant0ine/go-json-rest/rest" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func (r *Router) MetricsRoutes() []*rest.Route { + return []*rest.Route{ + rest.Get("/v1/metrics", r.PrometheusMetrics), + } +} + +func (r *Router) PrometheusMetrics(res rest.ResponseWriter, req *rest.Request) { + // The default go-json-rest middleware gzips the content + promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{DisableCompression: true}). + ServeHTTP(res.(http.ResponseWriter), req.Request) +} diff --git a/auth/service/api/v1/router.go b/auth/service/api/v1/router.go index e62e7b2e40..e19e3c4d09 100644 --- a/auth/service/api/v1/router.go +++ b/auth/service/api/v1/router.go @@ -23,6 +23,7 @@ func NewRouter(svc service.Service) (*Router, error) { func (r *Router) Routes() []*rest.Route { routes := [][]*rest.Route{ + r.MetricsRoutes(), r.OAuthRoutes(), r.ProviderSessionsRoutes(), r.RestrictedTokensRoutes(), diff --git a/auth/service/service/client.go b/auth/service/service/client.go index 074b81483e..a8bca5e9a8 100644 --- a/auth/service/service/client.go +++ b/auth/service/service/client.go @@ -88,8 +88,8 @@ func (c *Client) CreateProviderSession(ctx context.Context, create *auth.Provide if err = prvdr.OnCreate(ctx, providerSession); err != nil { log.LoggerFromContext(ctx).WithError(err).Error("Unable to finalize creation of provider session") - if err := c.deleteProviderSession(ctx, repository, providerSession); err != nil { - log.LoggerFromContext(ctx).WithError(err).Warn("Unable to delete provider session") + if deleteErr := c.deleteProviderSession(ctx, repository, providerSession); deleteErr != nil { + log.LoggerFromContext(ctx).WithError(deleteErr).Warn("Unable to delete provider session") } return nil, err } diff --git a/client/client.go b/client/client.go index 635495844e..5e416c3427 100644 --- a/client/client.go +++ b/client/client.go @@ -9,6 +9,7 @@ import ( "net/url" "reflect" "strings" + "time" "unicode/utf8" "github.com/tidepool-org/platform/errors" @@ -31,8 +32,7 @@ type ErrorResponseParser interface { } type Client struct { - address string - userAgent string + config Config errorResponseParser ErrorResponseParser } @@ -48,14 +48,13 @@ func NewWithErrorParser(cfg *Config, errorResponseParser ErrorResponseParser) (* } return &Client{ - address: cfg.Address, - userAgent: cfg.UserAgent, + config: *cfg, errorResponseParser: errorResponseParser, }, nil } func (c *Client) ConstructURL(paths ...string) string { - return ConstructURL(c.address, paths...) + return ConstructURL(c.config.Address, paths...) } func (c *Client) AppendURLQuery(urlString string, query map[string]string) string { @@ -78,17 +77,37 @@ func (c *Client) AppendURLQuery(urlString string, query map[string]string) strin } func (c *Client) RequestStreamWithHTTPClient(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody interface{}, inspectors []request.ResponseInspector, httpClient *http.Client) (io.ReadCloser, error) { + if ctx == nil { + return nil, errors.New("context is missing") + } if httpClient == nil { return nil, errors.New("http client is missing") } + // The request must carry a cancelable context for the timeout to reach the transport. A deadline on that context + // cannot be used, though, as it would also abort any read of the returned response body, so instead cancel via a + // timer that is stopped once the response headers arrive. The cause preserves the deadline exceeded error. + ctx, cancel := context.WithCancelCause(ctx) + req, err := c.createRequest(ctx, method, url, mutators, requestBody) if err != nil { + cancel(nil) return nil, err } + var timer *time.Timer + if c.config.Timeout > 0 { + timer = time.AfterFunc(c.config.Timeout, func() { cancel(context.DeadlineExceeded) }) + } + res, err := httpClient.Do(req) + + if timer != nil { + timer.Stop() + } + if err != nil { + cancel(nil) return nil, errors.Wrapf(err, "unable to perform request to %s %s", method, url) } @@ -96,7 +115,16 @@ func (c *Client) RequestStreamWithHTTPClient(ctx context.Context, method string, inspector.InspectResponse(res) } - return c.handleResponse(ctx, res, req) + body, err := c.handleResponse(ctx, res, req) + if body == nil { + cancel(nil) + return nil, err + } + + return &ReadCloserWithCancelCause{ + ReadCloser: body, + CancelCauseFunc: cancel, + }, err } func (c *Client) RequestDataWithHTTPClient(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody interface{}, responseBody interface{}, inspectors []request.ResponseInspector, httpClient *http.Client) error { @@ -127,8 +155,8 @@ func (c *Client) createRequest(ctx context.Context, method string, url string, m return nil, errors.New("url is missing") } - if c.userAgent != "" { - mutators = append(mutators, request.NewHeaderMutator("User-Agent", c.userAgent)) + if c.config.UserAgent != "" { + mutators = append(mutators, request.NewHeaderMutator("User-Agent", c.config.UserAgent)) } var body io.Reader @@ -247,6 +275,16 @@ func drainAndClose(reader io.ReadCloser) { reader.Close() } +type ReadCloserWithCancelCause struct { + io.ReadCloser + context.CancelCauseFunc +} + +func (r *ReadCloserWithCancelCause) Close() error { + defer r.CancelCauseFunc(nil) + return r.ReadCloser.Close() +} + func NewSerializableErrorResponseParser() *SerializableErrorResponseParser { return &SerializableErrorResponseParser{} } diff --git a/client/client_test.go b/client/client_test.go index 03fe31d91a..f59adc5cc6 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -773,6 +774,40 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(HaveLen(1)) }) }) + + Context("with a timeout", func() { + var timeout time.Duration + + BeforeEach(func() { + timeout = 100 * time.Millisecond + config.Timeout = timeout + }) + + It("returns an error if the response is not received within the timeout", func() { + server.AppendHandlers(func(res http.ResponseWriter, req *http.Request) { + time.Sleep(3 * timeout) + }) + + reader, err = clnt.RequestStreamWithHTTPClient(ctx, method, url, mutators, requestBody, inspectors, httpClient) + Expect(errors.Is(errors.Cause(err), context.DeadlineExceeded)).To(BeTrue()) + Expect(reader).To(BeNil()) + }) + + It("does not apply the timeout to reading the response body", func() { + server.AppendHandlers(func(res http.ResponseWriter, req *http.Request) { + res.WriteHeader(http.StatusOK) + res.(http.Flusher).Flush() + time.Sleep(3 * timeout) + res.Write([]byte(responseString)) + }) + + reader, err = clnt.RequestStreamWithHTTPClient(ctx, method, url, mutators, requestBody, inspectors, httpClient) + Expect(err).ToNot(HaveOccurred()) + Expect(reader).ToNot(BeNil()) + Expect(io.ReadAll(reader)).To(Equal([]byte(responseString))) + Expect(server.ReceivedRequests()).To(HaveLen(1)) + }) + }) }) Context("RequestDataWithHTTPClient", func() { diff --git a/client/config.go b/client/config.go index 2d2d811a00..50fe101281 100644 --- a/client/config.go +++ b/client/config.go @@ -2,10 +2,12 @@ package client import ( "net/url" + "time" "github.com/kelseyhightower/envconfig" "github.com/tidepool-org/platform/config" + "github.com/tidepool-org/platform/duration" "github.com/tidepool-org/platform/errors" ) @@ -23,6 +25,9 @@ type Config struct { // // More info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent UserAgent string `envconfig:"TIDEPOOL_USER_AGENT"` + + // Timeout specifies the maximum amount of time a request can take. Zero means no timeout. + Timeout time.Duration `envconfig:"TIDEPOOL_CLIENT_TIMEOUT"` } func NewConfig() *Config { @@ -36,6 +41,11 @@ func (c *Config) Load(loader ConfigLoader) error { func (c *Config) LoadFromConfigReporter(reporter config.Reporter) error { c.Address = reporter.GetWithDefault("address", c.Address) c.UserAgent = reporter.GetWithDefault("user_agent", c.UserAgent) + if timeout, parseErr := duration.Parse(reporter.GetWithDefault("timeout", c.Timeout.String()), time.Second); parseErr != nil { + return errors.New("timeout is invalid") + } else { + c.Timeout = timeout + } return nil } @@ -45,6 +55,9 @@ func (c *Config) Validate() error { } else if _, err := url.Parse(c.Address); err != nil { return errors.New("address is invalid") } + if c.Timeout < 0 { + return errors.New("timeout is invalid") + } return nil } @@ -70,6 +83,11 @@ func NewConfigReporterLoader(reporter config.Reporter) *configReporterLoader { func (l *configReporterLoader) Load(cfg *Config) error { cfg.Address = l.Reporter.GetWithDefault("address", cfg.Address) cfg.UserAgent = l.Reporter.GetWithDefault("user_agent", cfg.UserAgent) + if timeout, parseErr := duration.Parse(l.Reporter.GetWithDefault("timeout", cfg.Timeout.String()), time.Second); parseErr != nil { + return errors.New("timeout is invalid") + } else { + cfg.Timeout = timeout + } return nil } diff --git a/client/config_test.go b/client/config_test.go index 7d1a1a0d6a..a1e918ade5 100644 --- a/client/config_test.go +++ b/client/config_test.go @@ -1,11 +1,15 @@ package client_test import ( + "strconv" + "time" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/tidepool-org/platform/client" configTest "github.com/tidepool-org/platform/config/test" + "github.com/tidepool-org/platform/test" testHttp "github.com/tidepool-org/platform/test/http" ) @@ -21,17 +25,34 @@ var _ = Describe("Config", func() { Expect(cfg).ToNot(BeNil()) Expect(cfg.Address).To(BeEmpty()) Expect(cfg.UserAgent).To(BeEmpty()) + Expect(cfg.Timeout).To(BeZero()) + }) + }) + + Context("NewConfigReporterLoader", func() { + It("returns successfully", func() { + Expect(client.NewConfigReporterLoader(configTest.NewReporter())).ToNot(BeNil()) + }) + }) + + Context("NewEnvconfigLoader", func() { + It("returns successfully", func() { + Expect(client.NewEnvconfigLoader()).ToNot(BeNil()) }) }) Context("with new config", func() { var address string var userAgent string + var timeoutSeconds int + var timeout time.Duration var cfg *client.Config BeforeEach(func() { address = testHttp.NewAddress() userAgent = testHttp.NewUserAgent() + timeoutSeconds = test.RandomIntFromRange(1, 3600) + timeout = time.Duration(timeoutSeconds) * time.Second cfg = client.NewConfig() Expect(cfg).ToNot(BeNil()) }) @@ -44,6 +65,7 @@ var _ = Describe("Config", func() { configReporter = configTest.NewReporter() configReporter.Config["address"] = address configReporter.Config["user_agent"] = userAgent + configReporter.Config["timeout"] = strconv.Itoa(timeoutSeconds) loader = client.NewConfigReporterLoader(configReporter) }) @@ -54,6 +76,7 @@ var _ = Describe("Config", func() { Expect(cfg.Load(loader)).To(Succeed()) Expect(cfg.Address).To(Equal(existingAddress)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) }) It("uses existing user agent if not set", func() { @@ -63,12 +86,72 @@ var _ = Describe("Config", func() { Expect(cfg.Load(loader)).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(existingUserAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) + }) + + It("uses existing timeout if not set", func() { + existingTimeout := time.Duration(test.RandomIntFromRange(1, 3600)) * time.Second + cfg.Timeout = existingTimeout + delete(configReporter.Config, "timeout") + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Address).To(Equal(address)) + Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(existingTimeout)) + }) + + It("interprets a timeout without units as seconds", func() { + configReporter.Config["timeout"] = "1.5" + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(1500 * time.Millisecond)) + }) + + It("interprets a timeout with units as a duration", func() { + configReporter.Config["timeout"] = "1m30s" + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(90 * time.Second)) + }) + + It("returns an error if the timeout is not a number or a duration", func() { + configReporter.Config["timeout"] = "invalid" + Expect(cfg.Load(loader)).To(MatchError("timeout is invalid")) + }) + + It("returns an error if the timeout is empty", func() { + configReporter.Config["timeout"] = "" + Expect(cfg.Load(loader)).To(MatchError("timeout is invalid")) }) It("returns successfully and uses values from config reporter", func() { Expect(cfg.Load(loader)).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) + }) + }) + + Context("Load with envconfig loader", func() { + var loader client.ConfigLoader + + BeforeEach(func() { + loader = client.NewEnvconfigLoader() + }) + + It("returns successfully and uses the timeout from the environment", func() { + GinkgoT().Setenv("TIDEPOOL_CLIENT_TIMEOUT", timeout.String()) + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(timeout)) + }) + + It("ignores the untagged TIMEOUT environment variable", func() { + GinkgoT().Setenv("TIMEOUT", timeout.String()) + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Timeout).To(BeZero()) + }) + + // Unlike the config reporter loaders, envconfig requires units - see CLIENT-016. + It("returns an error if the timeout has no units", func() { + GinkgoT().Setenv("TIDEPOOL_CLIENT_TIMEOUT", strconv.Itoa(timeoutSeconds)) + Expect(cfg.Load(loader)).To(MatchError(ContainSubstring("assigning TIDEPOOL_CLIENT_TIMEOUT to Timeout"))) }) }) @@ -79,6 +162,7 @@ var _ = Describe("Config", func() { configReporter = configTest.NewReporter() configReporter.Config["address"] = address configReporter.Config["user_agent"] = userAgent + configReporter.Config["timeout"] = strconv.Itoa(timeoutSeconds) }) It("uses existing address if not set", func() { @@ -88,6 +172,7 @@ var _ = Describe("Config", func() { Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) Expect(cfg.Address).To(Equal(existingAddress)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) }) It("uses existing user agent if not set", func() { @@ -97,12 +182,46 @@ var _ = Describe("Config", func() { Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(existingUserAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) + }) + + It("uses existing timeout if not set", func() { + existingTimeout := time.Duration(test.RandomIntFromRange(1, 3600)) * time.Second + cfg.Timeout = existingTimeout + delete(configReporter.Config, "timeout") + Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) + Expect(cfg.Address).To(Equal(address)) + Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(existingTimeout)) + }) + + It("interprets a timeout without units as seconds", func() { + configReporter.Config["timeout"] = "1.5" + Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(1500 * time.Millisecond)) + }) + + It("interprets a timeout with units as a duration", func() { + configReporter.Config["timeout"] = "1m30s" + Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(90 * time.Second)) + }) + + It("returns an error if the timeout is not a number or a duration", func() { + configReporter.Config["timeout"] = "invalid" + Expect(cfg.LoadFromConfigReporter(configReporter)).To(MatchError("timeout is invalid")) + }) + + It("returns an error if the timeout is empty", func() { + configReporter.Config["timeout"] = "" + Expect(cfg.LoadFromConfigReporter(configReporter)).To(MatchError("timeout is invalid")) }) It("returns successfully and uses values from config reporter", func() { Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) }) }) @@ -110,6 +229,7 @@ var _ = Describe("Config", func() { BeforeEach(func() { cfg.Address = address cfg.UserAgent = userAgent + cfg.Timeout = timeout }) Context("Validate", func() { @@ -123,10 +243,21 @@ var _ = Describe("Config", func() { Expect(cfg.Validate()).To(MatchError("address is invalid")) }) + It("returns an error if the timeout is negative", func() { + cfg.Timeout = -timeout + Expect(cfg.Validate()).To(MatchError("timeout is invalid")) + }) + + It("returns success if the timeout is zero", func() { + cfg.Timeout = 0 + Expect(cfg.Validate()).To(Succeed()) + }) + It("returns success", func() { Expect(cfg.Validate()).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) }) }) }) diff --git a/client/prometheus.go b/client/prometheus.go new file mode 100644 index 0000000000..a207076ec7 --- /dev/null +++ b/client/prometheus.go @@ -0,0 +1,147 @@ +package client + +import ( + "fmt" + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/tidepool-org/platform/pointer" +) + +const PathPatternAny = "/" + +var ( + DurationBucketsDefault = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60} + + PrometheusLabelNameMethod = "method" + PrometheusLabelNamePath = "path" + PrometheusLabelNameStatus = "status" + + PrometheusLabelValueError = "ERROR" +) + +func PrometheusLabelNames() []string { + return []string{ + PrometheusLabelNameMethod, + PrometheusLabelNamePath, + PrometheusLabelNameStatus, + } +} + +// Where there are numerous discrete paths possible, then must simplify via patterns to prevent overwhelming Prometheus. +// If no patterns are specified, then all paths are recorded as-is. If one or more patterns are specified, but a path +// does not match one of the patterns, then the path is NOT captured by Prometheus. If you wish to match "all other" paths +// and records those paths as-is, then add the pattern PathPatternAny at the end of your patterns. +// +// Uses standard Go HTTP pattern matching. See https://go.dev/src/net/http/pattern.go. +// +// For example: /one/{id}, /two/{id} +type PrometheusRequestURLPathMatcher struct { + pathPatternMux *http.ServeMux +} + +func NewPrometheusRequestURLPathPatternMatcher(pathPatterns ...string) *PrometheusRequestURLPathMatcher { + var pathPatternMux *http.ServeMux + + if len(pathPatterns) > 0 { + pathPatternMux = http.NewServeMux() + for _, pattern := range pathPatterns { + pathPatternMux.HandleFunc(pattern, func(http.ResponseWriter, *http.Request) {}) + } + } + + return &PrometheusRequestURLPathMatcher{ + pathPatternMux: pathPatternMux, + } +} + +func (p *PrometheusRequestURLPathMatcher) MatchPath(req *http.Request) *string { + path := req.URL.Path + if p.pathPatternMux != nil { + if _, pattern := p.pathPatternMux.Handler(req); pattern == "" { + return nil + } else if pattern != PathPatternAny { + path = pattern + } + } + return &path +} + +type PrometheusRequestRoundTripper struct { + *RoundTripper + *PrometheusRequestURLPathMatcher +} + +func NewPrometheusRequestRoundTripper(pathPatterns ...string) *PrometheusRequestRoundTripper { + return &PrometheusRequestRoundTripper{ + RoundTripper: NewRoundTripper(nil), + PrometheusRequestURLPathMatcher: NewPrometheusRequestURLPathPatternMatcher(pathPatterns...), + } +} + +type PrometheusRequestMetricsRoundTripper struct { + *PrometheusRequestRoundTripper + requestCountCounterVec *prometheus.CounterVec + requestDurationHistogramVec *prometheus.HistogramVec +} + +func NewPrometheusRequestMetricsRoundTripper(name string, help string) *PrometheusRequestMetricsRoundTripper { + return NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name, help, nil, nil) +} + +func NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name string, help string, pathPatterns []string, durationBuckets []float64) *PrometheusRequestMetricsRoundTripper { + return &PrometheusRequestMetricsRoundTripper{ + PrometheusRequestRoundTripper: NewPrometheusRequestRoundTripper(pathPatterns...), + requestCountCounterVec: promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: fmt.Sprintf("%s_request_count", name), + Help: fmt.Sprintf("%s request count, sorted by method, path, and status", help), + }, + PrometheusLabelNames(), + ), + requestDurationHistogramVec: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: fmt.Sprintf("%s_request_duration_seconds", name), + Help: fmt.Sprintf("%s request duration, in seconds, sorted by method, path, and status", help), + Buckets: pointer.DefaultArray(durationBuckets, DurationBucketsDefault), + }, + PrometheusLabelNames(), + ), + } +} + +func (p *PrometheusRequestMetricsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + start := time.Now() + res, err := p.PrometheusRequestRoundTripper.RoundTrip(req) + duration := time.Since(start) + + if labels := p.Labels(req, res); labels != nil { + p.requestCountCounterVec.With(*labels).Inc() + p.requestDurationHistogramVec.With(*labels).Observe(duration.Seconds()) + } + + return res, err +} + +func (p *PrometheusRequestMetricsRoundTripper) Labels(req *http.Request, res *http.Response) *prometheus.Labels { + path := p.MatchPath(req) + if path == nil { + return nil + } + + labels := prometheus.Labels{ + PrometheusLabelNameMethod: req.Method, + PrometheusLabelNamePath: *path, + } + if res != nil { + labels[PrometheusLabelNameStatus] = strconv.Itoa(res.StatusCode) + } else { + labels[PrometheusLabelNameStatus] = PrometheusLabelValueError + } + + return &labels +} diff --git a/client/prometheus_test.go b/client/prometheus_test.go new file mode 100644 index 0000000000..af75a91f25 --- /dev/null +++ b/client/prometheus_test.go @@ -0,0 +1,281 @@ +package client_test + +import ( + "net/http" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/tidepool-org/platform/client" + errorsTest "github.com/tidepool-org/platform/errors/test" + prometheusTest "github.com/tidepool-org/platform/prometheus/test" + "github.com/tidepool-org/platform/test" + testHttp "github.com/tidepool-org/platform/test/http" +) + +var _ = Describe("Prometheus", func() { + It("PathPatternAny is expected", func() { + Expect(client.PathPatternAny).To(Equal("/")) + }) + + It("DurationBucketsDefault is expected", func() { + Expect(client.DurationBucketsDefault).To(Equal([]float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60})) + }) + + It("PrometheusLabelNameMethod is expected", func() { + Expect(client.PrometheusLabelNameMethod).To(Equal("method")) + }) + + It("PrometheusLabelNamePath is expected", func() { + Expect(client.PrometheusLabelNamePath).To(Equal("path")) + }) + + It("PrometheusLabelNameStatus is expected", func() { + Expect(client.PrometheusLabelNameStatus).To(Equal("status")) + }) + + It("PrometheusLabelValueError is expected", func() { + Expect(client.PrometheusLabelValueError).To(Equal("ERROR")) + }) + + Context("PrometheusLabelNames", func() { + It("returns the expected label names", func() { + Expect(client.PrometheusLabelNames()).To(Equal([]string{ + client.PrometheusLabelNameMethod, + client.PrometheusLabelNamePath, + client.PrometheusLabelNameStatus, + })) + }) + }) + + Context("PrometheusRequestURLPathPatternMatcher", func() { + Context("NewPrometheusRequestURLPathPatternMatcher", func() { + It("returns successfully with no path patterns", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher() + Expect(matcher).ToNot(BeNil()) + }) + It("returns successfully with path patterns", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher("/one/{id}", client.PathPatternAny) + Expect(matcher).ToNot(BeNil()) + }) + }) + + Context("MatchPath", func() { + It("returns the request path unchanged when there are no path patterns", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher() + request := testHttp.NewRequest() + Expect(matcher.MatchPath(request)).To(PointTo(Equal(request.URL.Path))) + }) + + It("returns nil when the path does not match any path pattern", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher("/one/{id}") + request := test.Must(http.NewRequest(http.MethodGet, "http://example.com/two/456", nil)) + Expect(matcher.MatchPath(request)).To(BeNil()) + }) + + It("returns the matched path pattern when the path matches a specific pattern", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher("/one/{id}", client.PathPatternAny) + request := test.Must(http.NewRequest(http.MethodGet, "http://example.com/one/123", nil)) + Expect(matcher.MatchPath(request)).To(PointTo(Equal("/one/{id}"))) + }) + + It("returns the request path unchanged when the path matches the any path pattern", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher("/one/{id}", client.PathPatternAny) + request := test.Must(http.NewRequest(http.MethodGet, "http://example.com/two/456", nil)) + Expect(matcher.MatchPath(request)).To(PointTo(Equal("/two/456"))) + }) + }) + }) + + Context("PrometheusRequestRoundTripper", func() { + Context("NewPrometheusRequestRoundTripper", func() { + It("returns successfully", func() { + roundTripper := client.NewPrometheusRequestRoundTripper() + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.RoundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestURLPathMatcher).ToNot(BeNil()) + }) + + It("returns successfully with path patterns", func() { + roundTripper := client.NewPrometheusRequestRoundTripper("/one/{id}", client.PathPatternAny) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.RoundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestURLPathMatcher).ToNot(BeNil()) + }) + }) + }) + + Context("PrometheusRequestMetricsRoundTripper", func() { + Context("NewPrometheusRequestMetricsRoundTripper", func() { + It("returns successfully", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(prometheusTest.RandomMetricName(), prometheusTest.RandomMetricHelp()) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestRoundTripper).ToNot(BeNil()) + }) + }) + + Context("NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets", func() { + It("returns successfully", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(prometheusTest.RandomMetricName(), prometheusTest.RandomMetricHelp(), []string{"/one/{id}"}, []float64{1, 2, 3}) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestRoundTripper).ToNot(BeNil()) + }) + }) + + Context("with a metrics round tripper", func() { + var testRoundTripper *testHttp.RoundTripper + var name string + var help string + var request *http.Request + + BeforeEach(func() { + testRoundTripper = testHttp.NewRoundTripper() + name = prometheusTest.RandomMetricName() + help = prometheusTest.RandomMetricHelp() + request = testHttp.NewRequest() + }) + + Context("RoundTrip", func() { + It("returns the response from the resolved round tripper", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + + result := test.Must(roundTripper.RoundTrip(request)) + Expect(result).To(BeIdenticalTo(testRoundTripper.Response)) + Expect(testRoundTripper.Request).To(BeIdenticalTo(request)) + }) + + It("returns the error from the resolved round tripper", func() { + testErr := errorsTest.RandomError() + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Error = testErr + + result, err := roundTripper.RoundTrip(request) + Expect(err).To(Equal(testErr)) + Expect(result).To(BeNil()) + }) + + It("records request count and duration metrics for a successful request", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + roundTripper.WithRoundTripper(testRoundTripper) + statusCode := testHttp.NewStatusCode() + testRoundTripper.Response = &http.Response{StatusCode: statusCode} + + _ = test.Must(roundTripper.RoundTrip(request)) + + expectedLabels := map[string]string{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: strconv.Itoa(statusCode), + } + + countFamily := prometheusTest.MetricFamilyFromName(name + "_request_count") + Expect(countFamily).ToNot(BeNil()) + Expect(countFamily.GetMetric()).To(HaveLen(1)) + countMetric := countFamily.GetMetric()[0] + Expect(countMetric.GetCounter().GetValue()).To(Equal(float64(1))) + Expect(prometheusTest.LabelPairsToMap(countMetric.GetLabel())).To(Equal(expectedLabels)) + + durationFamily := prometheusTest.MetricFamilyFromName(name + "_request_duration_seconds") + Expect(durationFamily).ToNot(BeNil()) + Expect(durationFamily.GetMetric()).To(HaveLen(1)) + durationMetric := durationFamily.GetMetric()[0] + Expect(durationMetric.GetHistogram().GetSampleCount()).To(Equal(uint64(1))) + Expect(prometheusTest.LabelPairsToMap(durationMetric.GetLabel())).To(Equal(expectedLabels)) + }) + + It("records request count and duration metrics with an ERROR status for a failed request", func() { + testErr := errorsTest.RandomError() + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Error = testErr + + _, err := roundTripper.RoundTrip(request) + Expect(err).To(Equal(testErr)) + + expectedLabels := map[string]string{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: client.PrometheusLabelValueError, + } + + countFamily := prometheusTest.MetricFamilyFromName(name + "_request_count") + Expect(countFamily).ToNot(BeNil()) + Expect(countFamily.GetMetric()).To(HaveLen(1)) + Expect(prometheusTest.LabelPairsToMap(countFamily.GetMetric()[0].GetLabel())).To(Equal(expectedLabels)) + + durationFamily := prometheusTest.MetricFamilyFromName(name + "_request_duration_seconds") + Expect(durationFamily).ToNot(BeNil()) + Expect(durationFamily.GetMetric()).To(HaveLen(1)) + Expect(prometheusTest.LabelPairsToMap(durationFamily.GetMetric()[0].GetLabel())).To(Equal(expectedLabels)) + }) + + It("does not record metrics when the path does not match a path pattern", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name, help, []string{"/one/{id}"}, nil) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + request = test.Must(http.NewRequest(http.MethodGet, "http://example.com/two/456", nil)) + + _ = test.Must(roundTripper.RoundTrip(request)) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_count")).To(BeNil()) + Expect(prometheusTest.MetricFamilyFromName(name + "_request_duration_seconds")).To(BeNil()) + }) + + It("uses the custom duration buckets when specified", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name, help, nil, []float64{1, 2, 3}) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + + _ = test.Must(roundTripper.RoundTrip(request)) + + durationFamily := prometheusTest.MetricFamilyFromName(name + "_request_duration_seconds") + Expect(durationFamily).ToNot(BeNil()) + Expect(durationFamily.GetMetric()).To(HaveLen(1)) + buckets := durationFamily.GetMetric()[0].GetHistogram().GetBucket() + Expect(buckets).To(HaveLen(3)) + Expect(buckets[0].GetUpperBound()).To(Equal(1.0)) + Expect(buckets[1].GetUpperBound()).To(Equal(2.0)) + Expect(buckets[2].GetUpperBound()).To(Equal(3.0)) + }) + }) + + Context("Labels", func() { + It("returns labels with the response status code", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + statusCode := testHttp.NewStatusCode() + response := &http.Response{StatusCode: statusCode} + + Expect(roundTripper.Labels(request, response)).To(PointTo(Equal(prometheus.Labels{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: strconv.Itoa(statusCode), + }))) + }) + + It("returns labels with an ERROR status when the response is nil", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + + Expect(roundTripper.Labels(request, nil)).To(PointTo(Equal(prometheus.Labels{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: client.PrometheusLabelValueError, + }))) + }) + + It("returns nil when the path does not match a path pattern", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name, help, []string{"/one/{id}"}, nil) + request := test.Must(http.NewRequest(http.MethodGet, "http://example.com/two/456", nil)) + + Expect(roundTripper.Labels(request, nil)).To(BeNil()) + }) + }) + }) + }) +}) diff --git a/client/round_tripper.go b/client/round_tripper.go new file mode 100644 index 0000000000..4fd484b3d2 --- /dev/null +++ b/client/round_tripper.go @@ -0,0 +1,32 @@ +package client + +import "net/http" + +type RoundTripper struct { + roundTripper http.RoundTripper +} + +func NewRoundTripper(roundTripper http.RoundTripper) *RoundTripper { + return &RoundTripper{ + roundTripper: roundTripper, + } +} + +func (p *RoundTripper) ResolvedRoundTripper() http.RoundTripper { + roundTripper := p.roundTripper + if roundTripper == nil { + roundTripper = http.DefaultClient.Transport + if roundTripper == nil { + roundTripper = http.DefaultTransport + } + } + return roundTripper +} + +func (p *RoundTripper) WithRoundTripper(roundTripper http.RoundTripper) { + p.roundTripper = roundTripper +} + +func (p *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return p.ResolvedRoundTripper().RoundTrip(req) +} diff --git a/client/round_tripper_test.go b/client/round_tripper_test.go new file mode 100644 index 0000000000..0cef7f2deb --- /dev/null +++ b/client/round_tripper_test.go @@ -0,0 +1,113 @@ +package client_test + +import ( + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tidepool-org/platform/client" + errorsTest "github.com/tidepool-org/platform/errors/test" + testHttp "github.com/tidepool-org/platform/test/http" +) + +var _ = Describe("RoundTripper", func() { + var testRoundTripper *testHttp.RoundTripper + var roundTripper *client.RoundTripper + + BeforeEach(func() { + testRoundTripper = testHttp.NewRoundTripper() + }) + + Context("NewRoundTripper", func() { + It("returns successfully with a round tripper", func() { + roundTripper = client.NewRoundTripper(testRoundTripper) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + + It("returns successfully without a round tripper", func() { + roundTripper = client.NewRoundTripper(nil) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.ResolvedRoundTripper()).ToNot(BeNil()) + }) + }) + + Context("ResolvedRoundTripper", func() { + var originalDefaultTransport http.RoundTripper + var originalDefaultClientTransport http.RoundTripper + + BeforeEach(func() { + originalDefaultTransport = http.DefaultTransport + originalDefaultClientTransport = http.DefaultClient.Transport + }) + + AfterEach(func() { + http.DefaultClient.Transport = originalDefaultClientTransport + http.DefaultTransport = originalDefaultTransport + }) + + It("returns the round tripper if it is set", func() { + roundTripper = client.NewRoundTripper(testRoundTripper) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + + It("returns http.DefaultClient.Transport if the round tripper is not set and http.DefaultClient.Transport is set", func() { + http.DefaultClient.Transport = testRoundTripper + roundTripper = client.NewRoundTripper(nil) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + + It("returns http.DefaultTransport if the round tripper is not set and http.DefaultClient.Transport is not set", func() { + http.DefaultClient.Transport = nil + http.DefaultTransport = testRoundTripper + roundTripper = client.NewRoundTripper(nil) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + }) + + Context("WithRoundTripper", func() { + BeforeEach(func() { + roundTripper = client.NewRoundTripper(nil) + }) + + It("sets the round tripper when it was previously unset", func() { + roundTripper.WithRoundTripper(testRoundTripper) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + + It("replaces the round tripper when it was previously set", func() { + roundTripper.WithRoundTripper(testRoundTripper) + replacement := testHttp.NewRoundTripper() + roundTripper.WithRoundTripper(replacement) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(replacement))) + }) + }) + + Context("RoundTrip", func() { + var request *http.Request + + BeforeEach(func() { + request = testHttp.NewRequest() + }) + + It("returns the response from the resolved round tripper", func() { + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + roundTripper = client.NewRoundTripper(testRoundTripper) + + result, err := roundTripper.RoundTrip(request) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeIdenticalTo(testRoundTripper.Response)) + Expect(testRoundTripper.Request).To(BeIdenticalTo(request)) + }) + + It("returns the error from the resolved round tripper", func() { + testRoundTripper.Error = errorsTest.RandomError() + roundTripper = client.NewRoundTripper(testRoundTripper) + + result, err := roundTripper.RoundTrip(request) + Expect(err).To(Equal(testRoundTripper.Error)) + Expect(result).To(BeNil()) + }) + }) +}) diff --git a/data/service/api/v1/metrics.go b/data/service/api/v1/metrics.go new file mode 100644 index 0000000000..13328e14d7 --- /dev/null +++ b/data/service/api/v1/metrics.go @@ -0,0 +1,25 @@ +package v1 + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + dataService "github.com/tidepool-org/platform/data/service" +) + +func MetricsRoutes() []dataService.Route { + return []dataService.Route{ + dataService.Get("/v1/metrics", PrometheusMetrics), + } +} + +func PrometheusMetrics(dataServiceContext dataService.Context) { + res := dataServiceContext.Response() + req := dataServiceContext.Request() + + // The default go-json-rest middleware gzips the content + promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{DisableCompression: true}). + ServeHTTP(res.(http.ResponseWriter), req.Request) +} diff --git a/data/service/api/v1/v1.go b/data/service/api/v1/v1.go index 3707db8104..387e43b7a6 100644 --- a/data/service/api/v1/v1.go +++ b/data/service/api/v1/v1.go @@ -30,6 +30,7 @@ func Routes() []service.Route { service.Post("/v1/partners/twiist/data/:tidepoolLinkId", NewTwiistDataCreateHandler(DataSetsDataCreate), api.RequireAuth), } + routes = append(routes, MetricsRoutes()...) routes = append(routes, DataSetsRoutes()...) routes = append(routes, SourcesRoutes()...) routes = append(routes, SummaryRoutes()...) diff --git a/data/service/service/standard.go b/data/service/service/standard.go index 0d5151504a..a309a01d5f 100644 --- a/data/service/service/standard.go +++ b/data/service/service/standard.go @@ -197,7 +197,7 @@ func (s *Standard) Initialize(provider application.Provider) error { func (s *Standard) Terminate() { if s.server != nil { if err := s.server.Shutdown(); err != nil { - s.Logger().Errorf("Error while terminating the the server: %v", err) + s.Logger().Errorf("Error while terminating the server: %v", err) } s.server = nil } diff --git a/data/source/store/structured/mongo/mongo_test.go b/data/source/store/structured/mongo/mongo_test.go index 8092205470..a56b7275ad 100644 --- a/data/source/store/structured/mongo/mongo_test.go +++ b/data/source/store/structured/mongo/mongo_test.go @@ -112,6 +112,7 @@ var _ = Describe("Mongo", func() { store, err = dataSourceStoreStructuredMongo.NewStore(config) Expect(err).ToNot(HaveOccurred()) Expect(store).ToNot(BeNil()) + Expect(store.Ping(context.Background())).ToNot(HaveOccurred()) mongoCollection = store.GetCollection("data_sources") }) diff --git a/data/store/mongo/mongo_datum.go b/data/store/mongo/mongo_datum.go index 02260c661d..f080fc8102 100644 --- a/data/store/mongo/mongo_datum.go +++ b/data/store/mongo/mongo_datum.go @@ -437,7 +437,7 @@ func (d *DatumRepository) ArchiveDeviceDataUsingHashesFromDataSet(ctx context.Co var updateInfo *mongo.UpdateResult - // Note that the "DeduplicatorHash" index is NOT used here as the fields in the query don't match the the index definition. On average an upload only has one device anyways (P90 ~ 1). However the "DeduplicatorHash" index is still useful for the UpdateMany operation that follows. + // Note that the "DeduplicatorHash" index is NOT used here as the fields in the query don't match the index definition. On average an upload only has one device anyways (P90 ~ 1). However the "DeduplicatorHash" index is still useful for the UpdateMany operation that follows. selector := bson.M{ "_userId": dataSet.UserID, "uploadId": dataSet.UploadID, diff --git a/dexcom/alert.go b/dexcom/alert.go index 4cfa6ff4b9..5295d42734 100644 --- a/dexcom/alert.go +++ b/dexcom/alert.go @@ -174,7 +174,7 @@ func (a *Alert) Validate(validator structure.Validator) { logger.Warnf("AlertState is '%s'", *a.AlertState) } if a.TransmitterID != nil && *a.TransmitterID == "" { - logger.Warnf("TransmitterID is empty", *a.TransmitterID) + logger.Warn("TransmitterID is empty") } } diff --git a/dexcom/calibration.go b/dexcom/calibration.go index b2efaee055..a5cc1ecb01 100644 --- a/dexcom/calibration.go +++ b/dexcom/calibration.go @@ -161,7 +161,7 @@ func (c *Calibration) Validate(validator structure.Validator) { logger.Warnf("Unit is '%s'", *c.Unit) } if c.TransmitterID != nil && *c.TransmitterID == "" { - logger.Warnf("TransmitterID is empty", *c.TransmitterID) + logger.Warn("TransmitterID is empty") } if c.DisplayDevice != nil && *c.DisplayDevice == DeviceDisplayDeviceUnknown { logger.Warnf("DisplayDevice is '%s'", *c.DisplayDevice) diff --git a/dexcom/client/client.go b/dexcom/client/client.go index 83af300841..55ad46d203 100644 --- a/dexcom/client/client.go +++ b/dexcom/client/client.go @@ -2,15 +2,20 @@ package client import ( "context" + "fmt" + "net/http" + "strings" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/dexcom" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/oauth" oauthClient "github.com/tidepool-org/platform/oauth/client" - "github.com/tidepool-org/platform/request" ) type Client struct { @@ -18,12 +23,29 @@ type Client struct { } func New(cfg *client.Config, tknSrcSrc oauth.TokenSourceSource) (*Client, error) { + if cfg == nil { + return nil, errors.New("config is missing") + } else if err := cfg.Validate(); err != nil { + return nil, errors.Wrap(err, "config is invalid") + } + + if cfg.Timeout == 0 { + cfg.Timeout = 1 * time.Minute + } + + httpClient := &http.Client{ + Transport: prometheusRequestMetricsRoundTripper, + CheckRedirect: http.DefaultClient.CheckRedirect, + Jar: http.DefaultClient.Jar, + Timeout: http.DefaultClient.Timeout, + } + baseClient, err := client.New(cfg) if err != nil { return nil, err } - clnt, err := oauthClient.NewWithClient(baseClient, tknSrcSrc) + clnt, err := oauthClient.NewWithClient(baseClient, httpClient, tknSrcSrc) if err != nil { return nil, err } @@ -114,17 +136,53 @@ func (c *Client) sendDexcomRequestWithDataRange(ctx context.Context, startTime t } func (c *Client) sendDexcomRequest(ctx context.Context, method string, url string, responseBody interface{}, tokenSource oauth.TokenSource) error { - startTime := time.Now() + return log.WarnIfDurationExceedsMaximum(ctx, requestDurationMaximum, url, func(ctx context.Context) error { + return c.client.SendOAuthRequest(ctx, method, url, nil, nil, responseBody, nil, tokenSource) + }) +} + +// Some Dexcom API responses include a "request-time" header with, supposedly, the internal duration of the request. +// This could be useful for debugging connection issues if compared against the calculated request duration. +// See client/prometheus.go for details on how the request duration is calculated and recorded. The format for +// this header is non-standard duration (e.g. "1234 ms") and the space needs to be removed for Golang to parse. - err := c.client.SendOAuthRequest(ctx, method, url, nil, nil, responseBody, []request.ResponseInspector{prometheusCodePathResponseInspector}, tokenSource) +const RequestTimeHeaderName = "request-time" + +type PrometheusRequestMetricsRoundTripper struct { + *client.PrometheusRequestMetricsRoundTripper + requestTimeHistogramVec *prometheus.HistogramVec +} + +func NewPrometheusRequestMetricsRoundTripper(name string, help string) *PrometheusRequestMetricsRoundTripper { + return &PrometheusRequestMetricsRoundTripper{ + PrometheusRequestMetricsRoundTripper: client.NewPrometheusRequestMetricsRoundTripper(name, help), + requestTimeHistogramVec: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: fmt.Sprintf("%s_request_time_seconds", name), + Help: fmt.Sprintf("%s request time (seconds)", help), + Buckets: client.DurationBucketsDefault, + }, + client.PrometheusLabelNames(), + ), + } +} - if requestDuration := time.Since(startTime); requestDuration > requestDurationMaximum { - log.LoggerFromContext(ctx).WithField("requestDuration", requestDuration.Truncate(time.Millisecond).Seconds()).Warn("Request duration exceeds maximum") +func (p *PrometheusRequestMetricsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + res, err := p.PrometheusRequestMetricsRoundTripper.RoundTrip(req) + + if res != nil { + if labels := p.Labels(req, res); labels != nil { + if requestTimeHeader := strings.ReplaceAll(res.Header.Get(RequestTimeHeaderName), " ", ""); requestTimeHeader != "" { + if requestTime, parseErr := time.ParseDuration(requestTimeHeader); parseErr == nil { + p.requestTimeHistogramVec.With(*labels).Observe(requestTime.Seconds()) + } + } + } } - return err + return res, err } -const requestDurationMaximum = 30 * time.Second +const requestDurationMaximum = 60 * time.Second -var prometheusCodePathResponseInspector = request.NewPrometheusCodePathResponseInspector("tidepool_dexcom_api_client_requests", "Dexcom API client requests") +var prometheusRequestMetricsRoundTripper = NewPrometheusRequestMetricsRoundTripper("tidepool_dexcom_api", "Tidepool Dexcom API") diff --git a/dexcom/client/client_test.go b/dexcom/client/client_test.go index bfffb34488..17c832ae1d 100644 --- a/dexcom/client/client_test.go +++ b/dexcom/client/client_test.go @@ -4,12 +4,15 @@ import ( "context" "fmt" "net/http" + "strconv" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/ghttp" + "go.uber.org/mock/gomock" + "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/dexcom" dexcomClient "github.com/tidepool-org/platform/dexcom/client" @@ -20,6 +23,7 @@ import ( logTest "github.com/tidepool-org/platform/log/test" oauthTest "github.com/tidepool-org/platform/oauth/test" "github.com/tidepool-org/platform/pointer" + prometheusTest "github.com/tidepool-org/platform/prometheus/test" "github.com/tidepool-org/platform/test" testHttp "github.com/tidepool-org/platform/test/http" ) @@ -27,17 +31,15 @@ import ( var _ = Describe("Client", func() { var userAgent string var config *client.Config - var tokenSourceSource *oauthTest.TokenSourceSource + var mockController *gomock.Controller + var mockTokenSourceSource *oauthTest.MockTokenSourceSource BeforeEach(func() { userAgent = testHttp.NewUserAgent() config = client.NewConfig() config.UserAgent = userAgent - tokenSourceSource = oauthTest.NewTokenSourceSource() - }) - - AfterEach(func() { - tokenSourceSource.AssertOutputsEmpty() + mockController = gomock.NewController(GinkgoT()) + mockTokenSourceSource = oauthTest.NewMockTokenSourceSource(mockController) }) Context("New", func() { @@ -46,14 +48,14 @@ var _ = Describe("Client", func() { }) It("returns an error when config is missing", func() { - clnt, err := dexcomClient.New(nil, tokenSourceSource) + clnt, err := dexcomClient.New(nil, mockTokenSourceSource) Expect(err).To(MatchError("config is missing")) Expect(clnt).To(BeNil()) }) It("returns an error when config is invalid", func() { config.Address = "" - clnt, err := dexcomClient.New(config, tokenSourceSource) + clnt, err := dexcomClient.New(config, mockTokenSourceSource) Expect(err).To(MatchError("config is invalid; address is missing")) Expect(clnt).To(BeNil()) }) @@ -65,7 +67,7 @@ var _ = Describe("Client", func() { }) It("returns successfully", func() { - Expect(dexcomClient.New(config, tokenSourceSource)).ToNot(BeNil()) + Expect(dexcomClient.New(config, mockTokenSourceSource)).ToNot(BeNil()) }) }) @@ -73,20 +75,20 @@ var _ = Describe("Client", func() { var server *Server var responseHeaders http.Header var ctx context.Context - var tokenSource *oauthTest.TokenSource + var mockTokenSource *oauthTest.MockTokenSource var clnt *dexcomClient.Client BeforeEach(func() { server = NewServer() responseHeaders = http.Header{"Content-Type": []string{"application/json; charset=utf-8"}} ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) - tokenSource = oauthTest.NewTokenSource() + mockTokenSource = oauthTest.NewMockTokenSource(mockController) }) JustBeforeEach(func() { config.Address = server.URL() var err error - clnt, err = dexcomClient.New(config, tokenSourceSource) + clnt, err = dexcomClient.New(config, mockTokenSourceSource) Expect(err).ToNot(HaveOccurred()) Expect(clnt).ToNot(BeNil()) }) @@ -95,7 +97,6 @@ var _ = Describe("Client", func() { if server != nil { server.Close() } - tokenSource.AssertOutputsEmpty() }) Context("GetDataRange", func() { @@ -104,6 +105,8 @@ var _ = Describe("Client", func() { var responseDataRangesResponse *dexcom.DataRangesResponse BeforeEach(func() { + lastSyncTime = nil + requestQuery = "" responseDataRangesResponse = dexcomTest.RandomDataRangesResponse() }) @@ -114,23 +117,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + dataRangeResponse, err := clnt.GetDataRange(context.Context(nil), lastSyncTime, mockTokenSource) + Expect(err).To(MatchError("unable to get data range; context is missing")) + Expect(dataRangeResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get data range; %s", responseErr))) Expect(dataRangeResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError(`unable to get data range; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(dataRangeResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -139,26 +147,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) - Expect(err).To(MatchError("unable to get data range; context is missing")) - Expect(dataRangeResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get data range; unable to perform request to .*: connect: connection refused")) Expect(dataRangeResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -175,7 +173,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; bad request")) Expect(dataRangeResponse).To(BeNil()) }) @@ -194,7 +192,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; authentication token is not authorized for requested action")) Expect(dataRangeResponse).To(BeNil()) }) @@ -213,7 +211,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; resource not found")) Expect(dataRangeResponse).To(BeNil()) }) @@ -232,7 +230,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get data range; unexpected response status code 500 from")) Expect(dataRangeResponse).To(BeNil()) @@ -252,7 +250,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; json is malformed")) Expect(dataRangeResponse).To(BeNil()) }) @@ -271,7 +269,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(dataRangeResponse).To(Equal(responseDataRangesResponse)) }) @@ -285,8 +283,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -295,8 +291,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request without last sync time", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -305,9 +299,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/dataRange", requestQuery), @@ -319,8 +313,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -339,7 +331,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; authentication token is invalid")) Expect(dataRangeResponse).To(BeNil()) }) @@ -373,23 +365,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + alertsResponse, err := clnt.GetAlerts(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get alerts; context is missing")) + Expect(alertsResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get alerts; %s", responseErr))) Expect(alertsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get alerts; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(alertsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -398,26 +395,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get alerts; context is missing")) - Expect(alertsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get alerts; unable to perform request to .*: connect: connection refused")) Expect(alertsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -434,7 +421,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; bad request")) Expect(alertsResponse).To(BeNil()) }) @@ -453,7 +440,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; authentication token is not authorized for requested action")) Expect(alertsResponse).To(BeNil()) }) @@ -472,7 +459,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; resource not found")) Expect(alertsResponse).To(BeNil()) }) @@ -491,7 +478,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get alerts; unexpected response status code 500 from")) Expect(alertsResponse).To(BeNil()) @@ -511,7 +498,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; json is malformed")) Expect(alertsResponse).To(BeNil()) }) @@ -530,7 +517,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(alertsResponse).To(Equal(responseAlertsResponse)) }) @@ -539,8 +526,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) requestAssertions() @@ -548,9 +533,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/alerts", requestQuery), @@ -562,8 +547,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -582,7 +565,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; authentication token is invalid")) Expect(alertsResponse).To(BeNil()) }) @@ -605,23 +588,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + calibrationsResponse, err := clnt.GetCalibrations(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get calibrations; context is missing")) + Expect(calibrationsResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get calibrations; %s", responseErr))) Expect(calibrationsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get calibrations; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(calibrationsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -630,26 +618,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get calibrations; context is missing")) - Expect(calibrationsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get calibrations; unable to perform request to .*: connect: connection refused")) Expect(calibrationsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -666,7 +644,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; bad request")) Expect(calibrationsResponse).To(BeNil()) }) @@ -685,7 +663,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; authentication token is not authorized for requested action")) Expect(calibrationsResponse).To(BeNil()) }) @@ -704,7 +682,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; resource not found")) Expect(calibrationsResponse).To(BeNil()) }) @@ -723,7 +701,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get calibrations; unexpected response status code 500 from")) Expect(calibrationsResponse).To(BeNil()) @@ -743,7 +721,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; json is malformed")) Expect(calibrationsResponse).To(BeNil()) }) @@ -762,7 +740,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(calibrationsResponse).To(Equal(responseCalibrationsResponse)) }) @@ -771,8 +749,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -781,9 +757,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/calibrations", requestQuery), @@ -795,8 +771,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -815,7 +789,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; authentication token is invalid")) Expect(calibrationsResponse).To(BeNil()) }) @@ -838,23 +812,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + devicesResponse, err := clnt.GetDevices(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get devices; context is missing")) + Expect(devicesResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get devices; %s", responseErr))) Expect(devicesResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get devices; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(devicesResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -863,26 +842,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get devices; context is missing")) - Expect(devicesResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get devices; unable to perform request to .*: connect: connection refused")) Expect(devicesResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -899,7 +868,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; bad request")) Expect(devicesResponse).To(BeNil()) }) @@ -918,7 +887,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; authentication token is not authorized for requested action")) Expect(devicesResponse).To(BeNil()) }) @@ -937,7 +906,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; resource not found")) Expect(devicesResponse).To(BeNil()) }) @@ -956,7 +925,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get devices; unexpected response status code 500 from")) Expect(devicesResponse).To(BeNil()) @@ -976,7 +945,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; json is malformed")) Expect(devicesResponse).To(BeNil()) }) @@ -995,7 +964,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(devicesResponse).To(Equal(responseDevicesResponse)) }) @@ -1004,8 +973,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -1014,9 +981,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/devices", requestQuery), @@ -1028,8 +995,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -1048,7 +1013,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; authentication token is invalid")) Expect(devicesResponse).To(BeNil()) }) @@ -1071,23 +1036,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + egvsResponse, err := clnt.GetEGVs(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get egvs; context is missing")) + Expect(egvsResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get egvs; %s", responseErr))) Expect(egvsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get egvs; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(egvsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -1096,26 +1066,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get egvs; context is missing")) - Expect(egvsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get egvs; unable to perform request to .*: connect: connection refused")) Expect(egvsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -1132,7 +1092,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; bad request")) Expect(egvsResponse).To(BeNil()) }) @@ -1151,7 +1111,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; authentication token is not authorized for requested action")) Expect(egvsResponse).To(BeNil()) }) @@ -1170,7 +1130,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; resource not found")) Expect(egvsResponse).To(BeNil()) }) @@ -1189,7 +1149,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get egvs; unexpected response status code 500 from")) Expect(egvsResponse).To(BeNil()) @@ -1209,7 +1169,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; json is malformed")) Expect(egvsResponse).To(BeNil()) }) @@ -1228,7 +1188,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(egvsResponse).To(Equal(responseEGVsResponse)) }) @@ -1237,8 +1197,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -1247,9 +1205,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/egvs", requestQuery), @@ -1261,8 +1219,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -1281,7 +1237,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; authentication token is invalid")) Expect(egvsResponse).To(BeNil()) }) @@ -1304,23 +1260,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + eventsResponse, err := clnt.GetEvents(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get events; context is missing")) + Expect(eventsResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get events; %s", responseErr))) Expect(eventsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get events; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(eventsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -1329,26 +1290,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get events; context is missing")) - Expect(eventsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get events; unable to perform request to .*: connect: connection refused")) Expect(eventsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -1365,7 +1316,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; bad request")) Expect(eventsResponse).To(BeNil()) }) @@ -1384,7 +1335,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; authentication token is not authorized for requested action")) Expect(eventsResponse).To(BeNil()) }) @@ -1403,7 +1354,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; resource not found")) Expect(eventsResponse).To(BeNil()) }) @@ -1422,7 +1373,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get events; unexpected response status code 500 from")) Expect(eventsResponse).To(BeNil()) @@ -1442,7 +1393,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; json is malformed")) Expect(eventsResponse).To(BeNil()) }) @@ -1461,7 +1412,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(eventsResponse).To(Equal(responseEventsResponse)) }) @@ -1470,8 +1421,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -1480,9 +1429,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/events", requestQuery), @@ -1494,8 +1443,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -1514,7 +1461,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; authentication token is invalid")) Expect(eventsResponse).To(BeNil()) }) @@ -1524,4 +1471,98 @@ var _ = Describe("Client", func() { }) }) }) + + It("RequestTimeHeaderName is expected", func() { + Expect(dexcomClient.RequestTimeHeaderName).To(Equal("request-time")) + }) + + Context("PrometheusRequestMetricsRoundTripper", func() { + Context("NewPrometheusRequestMetricsRoundTripper", func() { + It("returns successfully", func() { + roundTripper := dexcomClient.NewPrometheusRequestMetricsRoundTripper(prometheusTest.RandomMetricName(), prometheusTest.RandomMetricHelp()) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestMetricsRoundTripper).ToNot(BeNil()) + }) + }) + + Context("RoundTrip", func() { + var testRoundTripper *testHttp.RoundTripper + var name string + var roundTripper *dexcomClient.PrometheusRequestMetricsRoundTripper + var request *http.Request + + BeforeEach(func() { + testRoundTripper = testHttp.NewRoundTripper() + name = prometheusTest.RandomMetricName() + roundTripper = dexcomClient.NewPrometheusRequestMetricsRoundTripper(name, prometheusTest.RandomMetricHelp()) + roundTripper.WithRoundTripper(testRoundTripper) + request = testHttp.NewRequest() + }) + + It("returns the response from the resolved round tripper", func() { + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + + result := test.Must(roundTripper.RoundTrip(request)) + Expect(result).To(BeIdenticalTo(testRoundTripper.Response)) + Expect(testRoundTripper.Request).To(BeIdenticalTo(request)) + }) + + It("returns the error from the resolved round tripper", func() { + testErr := errorsTest.RandomError() + testRoundTripper.Error = testErr + + result, err := roundTripper.RoundTrip(request) + Expect(err).To(Equal(testErr)) + Expect(result).To(BeNil()) + }) + + It("does not record a request time metric when the resolved round tripper returns an error", func() { + testRoundTripper.Error = errorsTest.RandomError() + + _, _ = roundTripper.RoundTrip(request) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) + }) + + It("records a request time metric when the response has a valid request-time header", func() { + statusCode := testHttp.NewStatusCode() + requestTime := time.Duration(test.RandomIntFromRange(1, 60*1000)) * time.Millisecond + header := http.Header{} + header.Set(dexcomClient.RequestTimeHeaderName, requestTime.String()) + testRoundTripper.Response = &http.Response{StatusCode: statusCode, Header: header} + + _ = test.Must(roundTripper.RoundTrip(request)) + + family := prometheusTest.MetricFamilyFromName(name + "_request_time_seconds") + Expect(family).ToNot(BeNil()) + Expect(family.GetMetric()).To(HaveLen(1)) + metric := family.GetMetric()[0] + Expect(metric.GetHistogram().GetSampleCount()).To(Equal(uint64(1))) + Expect(metric.GetHistogram().GetSampleSum()).To(Equal(requestTime.Seconds())) + Expect(prometheusTest.LabelPairsToMap(metric.GetLabel())).To(Equal(map[string]string{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: strconv.Itoa(statusCode), + })) + }) + + It("does not record a request time metric when the response does not have a request-time header", func() { + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode(), Header: http.Header{}} + + _ = test.Must(roundTripper.RoundTrip(request)) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) + }) + + It("does not record a request time metric when the request-time header is not a valid duration", func() { + header := http.Header{} + header.Set(dexcomClient.RequestTimeHeaderName, test.RandomStringFromCharset(test.CharsetAlpha)) + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode(), Header: header} + + _ = test.Must(roundTripper.RoundTrip(request)) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) + }) + }) + }) }) diff --git a/dexcom/data_range.go b/dexcom/data_range.go index 3064e53144..87c119c6bf 100644 --- a/dexcom/data_range.go +++ b/dexcom/data_range.go @@ -79,8 +79,8 @@ func (d *DataRangesResponse) DataRange() *DataRange { endMoments = append(endMoments, d.Events.End) } - startMoments = startMoments.Compact() - endMoments = endMoments.Compact() + startMoments = startMoments.CompactBySystemTimeRaw() + endMoments = endMoments.CompactBySystemTimeRaw() if len(startMoments) == 0 || len(endMoments) == 0 { return nil } diff --git a/dexcom/data_range_test.go b/dexcom/data_range_test.go index d033ecc4a8..5d2e013c24 100644 --- a/dexcom/data_range_test.go +++ b/dexcom/data_range_test.go @@ -242,6 +242,39 @@ var _ = Describe("DataRange", func() { } Expect(datum.DataRange()).To(Equal(expectedDataRange)) }) + + It("returns data range excluding moments without a system time", func() { + datum := dexcomTest.RandomDataRangesResponse() + datum.Calibrations.Start = dexcomTest.RandomMomentFromTime(time.Unix(1730000000, 0)) + datum.Calibrations.End = dexcomTest.RandomMomentFromTime(time.Unix(1790000000, 0)) + datum.EGVs.Start = dexcomTest.RandomMomentFromTime(time.Unix(1710000000, 0)) + datum.EGVs.End = dexcomTest.RandomMomentFromTime(time.Unix(1780000000, 0)) + datum.Events.Start = dexcomTest.RandomMomentFromTime(time.Unix(1750000000, 0)) + datum.Events.End = dexcomTest.RandomMomentFromTime(time.Unix(1770000000, 0)) + datum.EGVs.Start.SystemTime = nil + datum.Calibrations.End.SystemTime = nil + expectedDataRange := &dexcom.DataRange{ + Start: datum.Calibrations.Start, + End: datum.EGVs.End, + } + Expect(datum.DataRange()).To(Equal(expectedDataRange)) + }) + + It("returns nil if no start moments have a system time", func() { + datum := dexcomTest.RandomDataRangesResponse() + datum.Calibrations.Start.SystemTime = nil + datum.EGVs.Start.SystemTime = nil + datum.Events.Start.SystemTime = nil + Expect(datum.DataRange()).To(BeNil()) + }) + + It("returns nil if no end moments have a system time", func() { + datum := dexcomTest.RandomDataRangesResponse() + datum.Calibrations.End.SystemTime = nil + datum.EGVs.End.SystemTime = nil + datum.Events.End.SystemTime = nil + Expect(datum.DataRange()).To(BeNil()) + }) }) }) diff --git a/dexcom/device.go b/dexcom/device.go index 0934e5dbaf..456d46e6c5 100644 --- a/dexcom/device.go +++ b/dexcom/device.go @@ -33,7 +33,7 @@ const ( DeviceTransmitterGenerationG4 = "g4" DeviceTransmitterGenerationG5 = "g5" DeviceTransmitterGenerationG6 = "g6" - DeviceTransmitterGenerationG6Pro = "g6 pro" // NOTE: Not specfied in API specs but found during actual usage + DeviceTransmitterGenerationG6Pro = "g6 pro" // NOTE: Not specified in API specs but found during actual usage DeviceTransmitterGenerationG6Plus = "g6+" DeviceTransmitterGenerationPro = "dexcomPro" DeviceTransmitterGenerationG7 = "g7" diff --git a/dexcom/egv.go b/dexcom/egv.go index a9a32560eb..ae14a7d423 100644 --- a/dexcom/egv.go +++ b/dexcom/egv.go @@ -241,7 +241,7 @@ func (e *EGV) Validate(validator structure.Validator) { logger.Warnf("Trend is '%s'", *e.Trend) } if e.TransmitterID != nil && *e.TransmitterID == "" { - logger.Warnf("TransmitterID is empty", *e.TransmitterID) + logger.Warn("TransmitterID is empty") } if e.TransmitterTicks != nil && *e.TransmitterTicks == EGVTransmitterTickMinimum { logger.Warnf("TransmitterTicks is %d", *e.TransmitterTicks) diff --git a/dexcom/event.go b/dexcom/event.go index 6747bb0529..5aca4e92b4 100644 --- a/dexcom/event.go +++ b/dexcom/event.go @@ -287,7 +287,7 @@ func (e *Event) Validate(validator structure.Validator) { logger.Warnf("EventType is '%s'", *e.EventType) } if e.TransmitterID != nil && *e.TransmitterID == "" { - logger.Warnf("TransmitterID is empty", *e.TransmitterID) + logger.Warn("TransmitterID is empty") } if e.DisplayDevice != nil && *e.DisplayDevice == DeviceDisplayDeviceUnknown { logger.Warnf("DisplayDevice is '%s'", *e.DisplayDevice) diff --git a/dexcom/fetch/runner.go b/dexcom/fetch/runner.go index 982676e28b..63175915b5 100644 --- a/dexcom/fetch/runner.go +++ b/dexcom/fetch/runner.go @@ -2,13 +2,13 @@ package fetch import ( "context" - "math/rand" "net/http" "sort" "strconv" "time" "github.com/tidepool-org/platform/auth" + "github.com/tidepool-org/platform/crypto" "github.com/tidepool-org/platform/data" dataDeduplicatorDeduplicator "github.com/tidepool-org/platform/data/deduplicator/deduplicator" dataSource "github.com/tidepool-org/platform/data/source" @@ -110,8 +110,8 @@ func (r *Runner) GetRunnerType() string { return Type } -func (r *Runner) GetRunnerDeadline() time.Time { - return time.Now().Add(TaskDurationMaximum * 3) +func (r *Runner) GetRunnerDeadline() time.Duration { + return TaskDurationMaximum * 3 } func (r *Runner) GetRunnerTimeout() time.Duration { @@ -150,7 +150,9 @@ type TaskRunner struct { deviceHashes map[string]string dataSet *data.DataSet dataSetPreloaded bool - deadline time.Time + runTime time.Time + availableAfter *time.Duration + importCompleted bool } func NewTaskRunner(provider Provider, tsk *task.Task) (*TaskRunner, error) { @@ -170,21 +172,32 @@ func NewTaskRunner(provider Provider, tsk *task.Task) (*TaskRunner, error) { func (t *TaskRunner) Run(ctx context.Context) { t.context = ctx t.logger = log.LoggerFromContext(t.context) - t.deadline = time.Now().Add(t.GetRunnerDurationMaximum()) + t.runTime = time.Now() t.task.ClearError() - if err := t.run(); err == nil { - t.rescheduleTask() - } else if !t.task.HasError() { - t.rescheduleTaskWithResourceError(err) + + if err := t.run(); err != nil { + t.task.AppendError(err) + } + + // If we didn't lose the claim, then update data source and repeat if not failed + if !errors.Is(context.Cause(t.context), task.ErrClaimLost) { + err := t.updateDataSourceWithTaskState() + if err != nil { + t.task.AppendError(err) + } + if err != nil || !t.task.IsFailed() { + t.task.RepeatAvailableAfter(pointer.Default(t.availableAfter, availableAfterDuration())) + } + } else { + t.logger.Warn("Skipped updating data source and task because the task claim was lost") } } func (t *TaskRunner) run() error { - defer t.updateDataSourceWithTaskState() - if len(t.task.Data) == 0 { - return t.failTaskWithInvalidStateError(errors.New("data is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data is missing")) } if err := t.getDataSource(); err != nil { @@ -202,9 +215,6 @@ func (t *TaskRunner) run() error { if err := t.fetchSinceLatestDataTime(); err != nil { return err } - if err := t.updateDataSourceWithLastImportTime(); err != nil { - return err - } return nil } @@ -212,14 +222,16 @@ func (t *TaskRunner) run() error { func (t *TaskRunner) getProviderSession() error { providerSessionID, ok := t.task.Data[dexcom.DataKeyProviderSessionID].(string) if !ok || providerSessionID == "" { - return t.failTaskWithInvalidStateError(errors.New("provider session id is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("provider session id is missing")) } providerSession, err := t.AuthClient().GetProviderSession(t.context, providerSessionID) if err != nil { - return t.rescheduleTaskWithResourceError(errors.Wrap(err, "unable to get provider session")) + return ErrorResourceFailureError(errors.Wrap(err, "unable to get provider session")) } else if providerSession == nil { - return t.failTaskWithInvalidStateError(errors.New("provider session is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("provider session is missing")) } t.providerSession = providerSession @@ -250,14 +262,16 @@ func (t *TaskRunner) updateProviderSession() error { func (t *TaskRunner) getDataSource() error { dataSourceID, ok := t.task.Data[dexcom.DataKeyDataSourceID].(string) if !ok || dataSourceID == "" { - return t.failTaskWithInvalidStateError(errors.New("data source id is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data source id is missing")) } source, err := t.DataSourceClient().Get(t.context, dataSourceID) if err != nil { - return t.rescheduleTaskWithResourceError(errors.Wrap(err, "unable to get data source")) + return ErrorResourceFailureError(errors.Wrap(err, "unable to get data source")) } else if source == nil { - return t.failTaskWithInvalidStateError(errors.New("data source is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data source is missing")) } t.dataSource = source @@ -282,14 +296,11 @@ func (t *TaskRunner) updateDataSourceWithDataTime(earliestDataTime *time.Time, l return t.updateDataSource(update) } -func (t *TaskRunner) updateDataSourceWithLastImportTime() error { - update := dataSource.NewUpdate() - update.LastImportTime = pointer.FromTime(time.Now()) - return t.updateDataSource(update) -} - func (t *TaskRunner) updateDataSourceWithTaskState() error { update := dataSource.NewUpdate() + if t.importCompleted { + update.LastImportTime = pointer.FromTime(time.Now()) + } if t.task.IsFailed() { update.State = pointer.FromString(dataSource.StateError) } @@ -305,9 +316,10 @@ func (t *TaskRunner) updateDataSource(update *dataSource.Update) error { // Without cancel to ensure data source is updated in the database dataSource, err := t.DataSourceClient().Update(context.WithoutCancel(t.context), t.dataSource.ID, nil, update) if err != nil { - return t.rescheduleTaskWithResourceError(errors.WithMeta(errors.Wrap(err, "unable to update data source"), update)) + return ErrorResourceFailureError(errors.WithMeta(errors.Wrap(err, "unable to update data source"), update)) } else if dataSource == nil { - return t.failTaskWithInvalidStateError(errors.New("data source is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data source is missing")) } t.dataSource = dataSource @@ -317,7 +329,8 @@ func (t *TaskRunner) updateDataSource(update *dataSource.Update) error { func (t *TaskRunner) createTokenSource() error { tokenSource, err := oauthToken.NewSourceWithToken(t.providerSession.OAuthToken) if err != nil { - return t.failTaskWithInvalidStateError(errors.Wrap(err, "unable to create token source")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.Wrap(err, "unable to create token source")) } t.tokenSource = tokenSource @@ -329,16 +342,18 @@ func (t *TaskRunner) getDeviceHashes() error { if !rawOK || raw == nil { return nil } - rawMap, rawMapOK := raw.(map[string]interface{}) + rawMap, rawMapOK := raw.(map[string]any) if !rawMapOK || rawMap == nil { - return t.failTaskWithInvalidStateError(errors.New("device hashes is invalid")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("device hashes is invalid")) } deviceHashes := map[string]string{} for key, value := range rawMap { if valueString, valueStringOK := value.(string); valueStringOK { deviceHashes[key] = valueString } else { - return t.failTaskWithInvalidStateError(errors.New("device hash is invalid")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("device hash is invalid")) } } @@ -381,9 +396,10 @@ func (t *TaskRunner) updateDataSet(update *data.DataSetUpdate) error { // Without cancel to ensure data set is updated in the database dataSet, err := t.DataClient().UpdateDataSet(context.WithoutCancel(t.context), *t.dataSet.UploadID, update) if err != nil { - return t.rescheduleTaskWithResourceError(errors.WithMeta(errors.Wrap(err, "unable to update data set"), update)) + return ErrorResourceFailureError(errors.WithMeta(errors.Wrap(err, "unable to update data set"), update)) } else if dataSet == nil { - return t.failTaskWithInvalidStateError(errors.New("data set is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data set is missing")) } t.dataSet = dataSet @@ -395,7 +411,8 @@ func (t *TaskRunner) fetchSinceLatestDataTime() error { if err != nil { return err } else if dataRange == nil { - return nil // Nothing to fetch + t.importCompleted = true // No data, but still successful import + return nil } startTime := dataRange.StartTime @@ -409,19 +426,20 @@ func (t *TaskRunner) fetchSinceLatestDataTime() error { return err } - // If past deadline (based upon runner maximum duration), then bail - if time.Now().After(t.deadline) { - return t.rescheduleTaskWithResourceError(context.DeadlineExceeded) + // If the task has been running for longer than the maximum duration, then stop fetching and repeat after a minute + if time.Since(t.runTime) > t.GetRunnerDurationMaximum() { + t.availableAfter = pointer.From(time.Minute) + return nil } startTime = startTime.AddDate(0, 0, DataRangeDaysMaximum) } - return t.updateDataSourceWithLastImportTime() + t.importCompleted = true + return nil } func (t *TaskRunner) fetchDataRange() (*DataRange, error) { - // HACK: Dexcom V3 (2024-05-30) - Can only use latest data time as last sync time if not // older than 100 days, otherwise will return erroneous results. Use 30 days to be on // the safe side. @@ -701,7 +719,7 @@ func (t *TaskRunner) findDataSet() (*data.DataSet, error) { } dataSet, err := t.DataClient().GetDataSet(t.context, *t.dataSource.DataSetID) if err != nil { - return nil, t.rescheduleTaskWithResourceError(errors.Wrap(err, "unable to get data set")) + return nil, ErrorResourceFailureError(errors.Wrap(err, "unable to get data set")) } return dataSet, nil } @@ -723,7 +741,7 @@ func (t *TaskRunner) createDataSet() (*data.DataSet, error) { dataSet, err := t.DataClient().CreateUserDataSet(t.context, t.providerSession.UserID, dataSetCreate) if err != nil { - return nil, t.rescheduleTaskWithResourceError(errors.WithMeta(errors.Wrap(err, "unable to create data set"), dataSetCreate)) + return nil, ErrorResourceFailureError(errors.WithMeta(errors.Wrap(err, "unable to create data set"), dataSetCreate)) } if err = t.updateDataSourceWithDataSet(dataSet); err != nil { return nil, err @@ -743,7 +761,7 @@ func (t *TaskRunner) storeDatumArray(datumArray data.Data) error { partialDatumArray := datumArray[startIndex:endIndex] if err := t.DataClient().CreateDataSetsData(t.context, *t.dataSet.UploadID, partialDatumArray); err != nil { - return t.rescheduleTaskWithResourceError(errors.Wrap(err, "unable to create data set data")) + return ErrorResourceFailureError(errors.Wrap(err, "unable to create data set data")) } earliestDataTime := partialDatumArray[0].GetTime() @@ -777,62 +795,26 @@ func (t *TaskRunner) afterLatestDataTime(latestDataTime *time.Time) bool { return latestDataTime != nil && (t.dataSource.LatestDataTime == nil || latestDataTime.After(*t.dataSource.LatestDataTime)) } -// Handle potential dexcom client error. Update provider session with latest token. -// If error, then retry or reschedule. Otherwise, reset retry count. func (t *TaskRunner) handleDexcomClientError(err error) error { - if err != nil { - return t.retryOrRescheduleTaskWithDexcomClientError(err) - } else { - return t.resetTaskRetryCount() + // If success, then reset retry count and return no error + if err == nil { + t.resetTaskRetryCount() + return nil } -} -// Retry task if Dexcom authentication failure. Otherwise, reschedule task. -func (t *TaskRunner) retryOrRescheduleTaskWithDexcomClientError(err error) error { - if request.IsErrorUnauthenticated(errors.Cause(err)) { - return t.retryTaskWithError(ErrorAuthenticationFailureError(err)) - } else { - return t.rescheduleTaskWithResourceError(err) + // If not an authentication error, then just treat as a resource failure + if !request.IsErrorUnauthenticated(errors.Cause(err)) { + return ErrorResourceFailureError(err) } -} -// Increment task retry count. If task retry count exceeds maximum, then fail task. -// Otherwise, reschedule task. Typically used for Dexcom authentication failures that -// may or may not be transient. -func (t *TaskRunner) retryTaskWithError(err error) error { - retryCount := t.incrementTaskRetryCount() - if retryCount > TaskRetryCountMaximum { - return t.failTaskWithError(err) + // It is an authentication error, attempt retry, if possible + err = ErrorAuthenticationFailureError(err) + if retryCount := t.incrementTaskRetryCount(); retryCount <= TaskRetryCountMaximum { + t.availableAfter = pointer.From(availableAfterDurationWithRetryCount(retryCount)) + return err } - t.task.AppendError(err) - t.task.RepeatAvailableAfter(availableAfterDurationWithFallbackFactor(fallbackFactorWithRetryCount(retryCount))) - return err -} - -func (t *TaskRunner) rescheduleTaskWithResourceError(err error) error { - return t.rescheduleTaskWithError(ErrorResourceFailureError(err)) -} - -// Reschedule task for next run. Append error to task. -func (t *TaskRunner) rescheduleTaskWithError(err error) error { - t.task.AppendError(err) - t.rescheduleTask() - return err -} - -func (t *TaskRunner) rescheduleTask() { - t.task.RepeatAvailableAfter(availableAfterDuration()) -} - -func (t *TaskRunner) failTaskWithInvalidStateError(err error) error { - return t.failTaskWithError(ErrorInvalidStateError(err)) -} - -// Fail task immediately and permanently. Do not reschedule. For situations where any future attempt is -// also guaranteed to fail. For example, when the task data is missing information. Should not normally happen. -func (t *TaskRunner) failTaskWithError(err error) error { - t.task.AppendError(err) + // Otherwise, we are failed t.task.SetFailed() return err } @@ -844,13 +826,12 @@ func (t *TaskRunner) incrementTaskRetryCount() int { retryCount = int(value) + 1 } } - t.task.Data[dexcom.DataKeyRetryCount] = retryCount + t.task.Data[dexcom.DataKeyRetryCount] = int32(retryCount) return retryCount } -func (t *TaskRunner) resetTaskRetryCount() error { +func (t *TaskRunner) resetTaskRetryCount() { delete(t.task.Data, dexcom.DataKeyRetryCount) - return nil } func (t *TaskRunner) HTTPClient(ctx context.Context, tokenSourceSource oauth.TokenSourceSource) (*http.Client, error) { @@ -902,8 +883,12 @@ func availableAfterDuration() time.Duration { return availableAfterDurationWithFallbackFactor(1) } +func availableAfterDurationWithRetryCount(retryCount int) time.Duration { + return availableAfterDurationWithFallbackFactor(fallbackFactorWithRetryCount(retryCount)) +} + func availableAfterDurationWithFallbackFactor(fallbackFactor float64) time.Duration { - return time.Duration(float64(AvailableAfterDuration)*fallbackFactor) + time.Duration(rand.Int63n(int64(2*AvailableAfterDurationJitter))) - AvailableAfterDurationJitter + return time.Duration(float64(AvailableAfterDuration)*fallbackFactor) + time.Duration(crypto.RandomInt64N(int64(2*AvailableAfterDurationJitter))) - AvailableAfterDurationJitter } func fallbackFactorWithRetryCount(retryCount int) float64 { diff --git a/dexcom/fetch/runner_test.go b/dexcom/fetch/runner_test.go index ccd1d627a2..f9a170347b 100644 --- a/dexcom/fetch/runner_test.go +++ b/dexcom/fetch/runner_test.go @@ -109,7 +109,7 @@ var _ = Describe("Runner", func() { }) It("returns the runner deadline", func() { - Expect(runner.GetRunnerDeadline()).Should(BeTemporally("~", time.Now().Add(45*time.Minute), time.Second)) + Expect(runner.GetRunnerDeadline()).To(Equal(45 * time.Minute)) }) It("returns the runner timeout", func() { @@ -141,15 +141,17 @@ var _ = Describe("Runner", func() { Context("with provider and task", func() { var provider *dexcomFetchTest.MockProvider + var runnerDurationMaximum time.Duration var tsk *task.Task BeforeEach(func() { provider = dexcomFetchTest.NewMockProvider(mockController) + runnerDurationMaximum = time.Second provider.EXPECT().AuthClient().Return(authClient).AnyTimes() provider.EXPECT().DataClient().Return(dataClient).AnyTimes() provider.EXPECT().DataSourceClient().Return(dataSourceClient).AnyTimes() provider.EXPECT().DexcomClient().Return(dexcomClient).AnyTimes() - provider.EXPECT().GetRunnerDurationMaximum().Return(time.Second).AnyTimes() + provider.EXPECT().GetRunnerDurationMaximum().DoAndReturn(func() time.Duration { return runnerDurationMaximum }).AnyTimes() tsk = &task.Task{ State: task.TaskStateRunning, Data: map[string]any{ @@ -185,6 +187,7 @@ var _ = Describe("Runner", func() { Context("with task runner and context", func() { var taskRunner *dexcomFetch.TaskRunner + var logger *logTest.Logger var ctx context.Context BeforeEach(func() { @@ -192,7 +195,8 @@ var _ = Describe("Runner", func() { taskRunner, err = dexcomFetch.NewTaskRunner(provider, tsk) Expect(err).ToNot(HaveOccurred()) Expect(taskRunner).ToNot(BeNil()) - ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) + logger = logTest.NewLogger() + ctx = log.NewContextWithLogger(context.Background(), logger) }) assertTaskState := func(state string) { @@ -206,8 +210,19 @@ var _ = Describe("Runner", func() { } } + assertTaskAvailableSoon := func() { + Expect(tsk.AvailableTime).ToNot(BeNil()) + Expect(*tsk.AvailableTime).To(BeTemporally("~", time.Now().Add(time.Minute), 5*time.Second)) + } + + assertTaskAvailableAfterStandardDuration := func() { + Expect(tsk.AvailableTime).ToNot(BeNil()) + Expect(*tsk.AvailableTime).To(BeTemporally(">", time.Now().Add(dexcomFetch.AvailableAfterDuration-dexcomFetch.AvailableAfterDurationJitter-time.Second))) + Expect(*tsk.AvailableTime).To(BeTemporally("<", time.Now().Add(dexcomFetch.AvailableAfterDuration+dexcomFetch.AvailableAfterDurationJitter))) + } + assertTaskRetryCount := func(retryCount int) { - Expect(tsk.Data[dexcom.DataKeyRetryCount]).To(Equal(retryCount)) + Expect(tsk.Data[dexcom.DataKeyRetryCount]).To(Equal(int32(retryCount))) } assertTaskRetryCountNotPresent := func() { @@ -258,7 +273,7 @@ var _ = Describe("Runner", func() { It("fails if getting the data source fails", func() { testErr := errorsTest.RandomError() - dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(nil, testErr).Times(1) + dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(nil, testErr) taskRunner.Run(ctx) assertTaskState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -266,7 +281,7 @@ var _ = Describe("Runner", func() { }) It("fails if the data source is missing", func() { - dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(nil, nil).Times(1) + dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(nil, nil) taskRunner.Run(ctx) assertTaskState(task.TaskStateFailed) assertTaskRetryCountNotPresent() @@ -282,7 +297,7 @@ var _ = Describe("Runner", func() { ProviderSessionID: pointer.FromString("test-provider-session-id"), State: dataSource.StateConnected, } - dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(dataSrc, nil).Times(1) + dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(dataSrc, nil) }) assertTaskAndDataSourceState := func(state string) { @@ -309,10 +324,14 @@ var _ = Describe("Runner", func() { Expect(tsk.HasError()).To(BeFalse()) } + assertDataSourceLastImportTimePresent := func() { + Expect(dataSrc.LastImportTime).ToNot(BeNil()) + } + It("fails if provider session id is missing and update data source returns an error", func() { testErr := errorsTest.RandomError() delete(tsk.Data, dexcom.DataKeyProviderSessionID) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).Return(nil, testErr).Times(1) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).Return(nil, testErr) taskRunner.Run(ctx) assertTaskState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -321,7 +340,7 @@ var _ = Describe("Runner", func() { It("fails if provider session id is missing", func() { delete(tsk.Data, dexcom.DataKeyProviderSessionID) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStateFailed) assertTaskRetryCountNotPresent() @@ -330,7 +349,7 @@ var _ = Describe("Runner", func() { It("fails if provider session id is empty", func() { tsk.Data[dexcom.DataKeyProviderSessionID] = "" - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStateFailed) assertTaskRetryCountNotPresent() @@ -339,17 +358,32 @@ var _ = Describe("Runner", func() { It("fails if getting the provider session fails", func() { testErr := errorsTest.RandomError() - authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(nil, testErr).Times(1) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(nil, testErr) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() assertTaskAndDataSourceError(dexcomFetch.ErrorCodeResourceFailure, "unable to get provider session") }) + It("discards the run outcome if the task claim is lost", func() { + claimContext, claimCancel := context.WithCancelCause(ctx) + defer claimCancel(nil) + testErr := errorsTest.RandomError() + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").DoAndReturn(func(ctx context.Context, id string) (*auth.ProviderSession, error) { + claimCancel(task.ErrClaimLost) + return nil, testErr + }) + taskRunner.Run(claimContext) + assertTaskState(task.TaskStateRunning) + Expect(dataSrc.State).To(Equal(dataSource.StateConnected)) + Expect(dataSrc.HasError()).To(BeFalse()) + logger.AssertWarn("Skipped updating data source and task because the task claim was lost") + }) + It("fails if the provider session is missing", func() { - authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(nil, nil).Times(1) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(nil, nil) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStateFailed) assertTaskRetryCountNotPresent() @@ -372,8 +406,8 @@ var _ = Describe("Runner", func() { UserID: "test-user-id", OAuthToken: oauthToken, } - authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(providerSession, nil).Times(1) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(providerSession, nil) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) }) assertProviderSessionRefreshedTimes := func(times int) { @@ -412,7 +446,7 @@ var _ = Describe("Runner", func() { It("fails if get data ranges returns a general error", func() { testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, nil, testErr)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, nil, testErr)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -424,7 +458,7 @@ var _ = Describe("Runner", func() { latestDataTime := pointer.FromTime(time.Now().Add(-Day)) dataSrc.LatestDataTime = latestDataTime testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetDataRange(matchContext(), latestDataTime, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, nil, testErr)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), latestDataTime, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, nil, testErr)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -434,8 +468,8 @@ var _ = Describe("Runner", func() { It("fails if get data ranges refreshes the token and returns a general error", func() { testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -445,8 +479,8 @@ var _ = Describe("Runner", func() { It("fails if get data ranges refreshes the token and returns an authentication error", func() { testErr := request.ErrorUnauthenticated() - dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCount(1) @@ -468,8 +502,8 @@ var _ = Describe("Runner", func() { End: &dexcom.Moment{SystemTime: &dexcom.Time{Time: endTime}}, }, } - dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, dataRangeResponse, nil)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, dataRangeResponse, nil)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) }) It("is successful if the Dexcom data ranges is not valid", func() { @@ -492,8 +526,8 @@ var _ = Describe("Runner", func() { It("fails if get alerts returns a general error", func() { testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](nil, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](nil, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -503,8 +537,8 @@ var _ = Describe("Runner", func() { It("fails if get alerts refreshes the token and returns a general error", func() { testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](&MockTokenSource{Refresh: true}, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](&MockTokenSource{Refresh: true}, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -514,8 +548,8 @@ var _ = Describe("Runner", func() { It("fails if get alerts refreshes the token and returns an authentication error", func() { testErr := request.ErrorUnauthenticated() - dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](&MockTokenSource{Refresh: true}, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](&MockTokenSource{Refresh: true}, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCount(1) @@ -532,9 +566,9 @@ var _ = Describe("Runner", func() { BeforeEach(func() { alertsResponse = &dexcom.AlertsResponse{Records: &dexcom.Alerts{}} - dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, alertsResponse, nil)).Times(1) + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, alertsResponse, nil)) calibrationsResponse = &dexcom.CalibrationsResponse{Records: &dexcom.Calibrations{}} - dexcomClient.EXPECT().GetCalibrations(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, calibrationsResponse, nil)).Times(1) + dexcomClient.EXPECT().GetCalibrations(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, calibrationsResponse, nil)) devicesResponse = &dexcom.DevicesResponse{ Records: &dexcom.Devices{ { @@ -547,11 +581,11 @@ var _ = Describe("Runner", func() { }, }, } - dexcomClient.EXPECT().GetDevices(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, devicesResponse, nil)).Times(1) + dexcomClient.EXPECT().GetDevices(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, devicesResponse, nil)) egvsResponse = &dexcom.EGVsResponse{Records: &dexcom.EGVs{}} - dexcomClient.EXPECT().GetEGVs(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, egvsResponse, nil)).Times(1) + dexcomClient.EXPECT().GetEGVs(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, egvsResponse, nil)) eventsResponse = &dexcom.EventsResponse{Records: &dexcom.Events{}} - dexcomClient.EXPECT().GetEvents(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, eventsResponse, nil)).Times(1) + dexcomClient.EXPECT().GetEvents(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, eventsResponse, nil)) authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(5) }) @@ -569,14 +603,54 @@ var _ = Describe("Runner", func() { ID: pointer.FromString("test-data-set-id"), UploadID: pointer.FromString("test-data-set-upload-id"), } - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(3) - dataClient.EXPECT().CreateUserDataSet(matchContext(), "test-user-id", matchNotNil()).DoAndReturn(mockDataClientCreateUserDataSet(dataSet, nil)).Times(1) - dataClient.EXPECT().CreateDataSetsData(matchContext(), "test-data-set-upload-id", matchNotNil()).DoAndReturn(mockDataClientCreateDataSetsData(nil)).Times(1) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(2) + dataClient.EXPECT().CreateUserDataSet(matchContext(), "test-user-id", matchNotNil()).DoAndReturn(mockDataClientCreateUserDataSet(dataSet, nil)) + dataClient.EXPECT().CreateDataSetsData(matchContext(), "test-data-set-upload-id", matchNotNil()).DoAndReturn(mockDataClientCreateDataSetsData(nil)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) + assertTaskAvailableAfterStandardDuration() assertTaskDeviceHashesCount(3) assertTaskRetryCountNotPresent() assertTaskAndDataSourceErrorNotPresent() + assertDataSourceLastImportTimePresent() + assertProviderSessionRefreshedTimes(6) + }) + + It("is available soon if the deadline is exceeded", func() { + runnerDurationMaximum = -time.Second + dataSet := &data.DataSet{ + ID: pointer.FromString("test-data-set-id"), + UploadID: pointer.FromString("test-data-set-upload-id"), + } + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(2) + dataClient.EXPECT().CreateUserDataSet(matchContext(), "test-user-id", matchNotNil()).DoAndReturn(mockDataClientCreateUserDataSet(dataSet, nil)) + dataClient.EXPECT().CreateDataSetsData(matchContext(), "test-data-set-upload-id", matchNotNil()).DoAndReturn(mockDataClientCreateDataSetsData(nil)) + taskRunner.Run(ctx) + assertTaskAndDataSourceState(task.TaskStatePending) + assertTaskAvailableSoon() + assertTaskRetryCountNotPresent() + assertTaskAndDataSourceErrorNotPresent() + assertDataSourceLastImportTimePresent() + assertProviderSessionRefreshedTimes(6) + }) + + It("is available soon if the deadline is exceeded and a later update fails", func() { + runnerDurationMaximum = -time.Second + testErr := errorsTest.RandomError() + dataSet := &data.DataSet{ + ID: pointer.FromString("test-data-set-id"), + UploadID: pointer.FromString("test-data-set-upload-id"), + } + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).Return(nil, testErr).Times(1) + dataClient.EXPECT().CreateUserDataSet(matchContext(), "test-user-id", matchNotNil()).DoAndReturn(mockDataClientCreateUserDataSet(dataSet, nil)) + dataClient.EXPECT().CreateDataSetsData(matchContext(), "test-data-set-upload-id", matchNotNil()).DoAndReturn(mockDataClientCreateDataSetsData(nil)) + taskRunner.Run(ctx) + assertTaskState(task.TaskStatePending) + assertTaskAvailableSoon() + assertTaskRetryCountNotPresent() + assertTaskError(dexcomFetch.ErrorCodeResourceFailure, "unable to update data source") + assertDataSourceLastImportTimePresent() assertProviderSessionRefreshedTimes(6) }) }) @@ -586,7 +660,78 @@ var _ = Describe("Runner", func() { // deviceHashes - not in data // dataSource.LatestDataTime - not nil (recent) // refresh token - // data ranges multiple 30 day segments + }) + + Context("with provider session and a data range spanning multiple chunks", func() { + var providerSession *auth.ProviderSession + var firstChunkStartTime time.Time + var firstChunkEndTime time.Time + var secondChunkEndTime time.Time + + BeforeEach(func() { + providerSession = &auth.ProviderSession{ + ID: "test-provider-session-id", + UserID: "test-user-id", + OAuthToken: &auth.OAuthToken{ + AccessToken: "test-access-token-1", + TokenType: "Bearer", + RefreshToken: "test-refresh-token-1", + ExpirationTime: time.Now().Add(time.Minute), + }, + } + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(providerSession, nil) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).AnyTimes() + firstChunkStartTime = time.Now().Add(-45 * Day) + firstChunkEndTime = firstChunkStartTime.AddDate(0, 0, dexcomFetch.DataRangeDaysMaximum) + secondChunkEndTime = time.Now().Add(-3 * Day) + dataRangeResponse := &dexcom.DataRangesResponse{ + Calibrations: &dexcom.DataRange{ + Start: &dexcom.Moment{SystemTime: &dexcom.Time{Time: firstChunkStartTime}}, + End: &dexcom.Moment{SystemTime: &dexcom.Time{Time: secondChunkEndTime}}, + }, + } + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, dataRangeResponse, nil)) + }) + + // Expects the fetch of a single chunk, all responses empty, invoking onEvents, if any, during the + // final fetch of the chunk + expectFetchChunk := func(startTime time.Time, endTime time.Time, onEvents func()) { + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, &dexcom.AlertsResponse{Records: &dexcom.Alerts{}}, nil)) + dexcomClient.EXPECT().GetCalibrations(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, &dexcom.CalibrationsResponse{Records: &dexcom.Calibrations{}}, nil)) + dexcomClient.EXPECT().GetDevices(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, &dexcom.DevicesResponse{Records: &dexcom.Devices{}}, nil)) + dexcomClient.EXPECT().GetEGVs(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, &dexcom.EGVsResponse{Records: &dexcom.EGVs{}}, nil)) + dexcomClient.EXPECT().GetEvents(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(func(ctx context.Context, startTime time.Time, endTime time.Time, tokenSource oauth.TokenSource) (*dexcom.EventsResponse, error) { + if onEvents != nil { + onEvents() + } + return &dexcom.EventsResponse{Records: &dexcom.Events{}}, nil + }) + } + + It("fetches every chunk of the data range", func() { + expectFetchChunk(firstChunkStartTime, firstChunkEndTime, nil) + expectFetchChunk(firstChunkEndTime, secondChunkEndTime, nil) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) + taskRunner.Run(ctx) + assertTaskAndDataSourceState(task.TaskStatePending) + assertTaskAvailableAfterStandardDuration() + assertTaskRetryCountNotPresent() + assertTaskAndDataSourceErrorNotPresent() + assertDataSourceLastImportTimePresent() + }) + + It("discards the run outcome if the task claim is lost mid-fetch", func() { + claimContext, claimCancel := context.WithCancelCause(ctx) + defer claimCancel(nil) + expectFetchChunk(firstChunkStartTime, firstChunkEndTime, func() { claimCancel(task.ErrClaimLost) }) + // The canceled context fails the next chunk, ending the run + dexcomClient.EXPECT().GetAlerts(matchContext(), firstChunkEndTime, secondChunkEndTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](nil, nil, context.Canceled)) + taskRunner.Run(claimContext) + assertTaskState(task.TaskStateRunning) + Expect(dataSrc.State).To(Equal(dataSource.StateConnected)) + Expect(dataSrc.HasError()).To(BeFalse()) + logger.AssertWarn("Skipped updating data source and task because the task claim was lost") + }) }) }) }) diff --git a/dexcom/moment.go b/dexcom/moment.go index f45b53ee85..e1cb1dc0b3 100644 --- a/dexcom/moment.go +++ b/dexcom/moment.go @@ -50,10 +50,10 @@ func (m *Moment) SystemTimeRaw() *time.Time { type Moments []*Moment -func (m Moments) Compact() Moments { +func (m Moments) CompactBySystemTimeRaw() Moments { var moments Moments for _, moment := range m { - if moment != nil { + if systemTimeRaw(moment) != nil { moments = append(moments, moment) } } diff --git a/dexcom/moment_test.go b/dexcom/moment_test.go index 2b7a222a96..1c3f0fbee5 100644 --- a/dexcom/moment_test.go +++ b/dexcom/moment_test.go @@ -187,13 +187,17 @@ var _ = Describe("Moment", func() { }) }) - Context("Moment", func() { - Context("Compact", func() { - It("removes nil moments from array", func() { + Context("Moments", func() { + Context("CompactBySystemTimeRaw", func() { + It("removes nil moments and moments without a system time from array", func() { moment1 := dexcomTest.RandomMomentFromRange(test.PastFarTime(), test.FutureFarTime()) moment2 := dexcomTest.RandomMomentFromRange(test.PastFarTime(), test.FutureFarTime()) - moments := dexcom.Moments{nil, moment1, nil, moment2, nil} - Expect(moments.Compact()).To(Equal(dexcom.Moments{moment1, moment2})) + moment3 := dexcomTest.RandomMomentFromRange(test.PastFarTime(), test.FutureFarTime()) + moment3.SystemTime = nil + moment4 := dexcomTest.RandomMomentFromRange(test.PastFarTime(), test.FutureFarTime()) + moment4.SystemTime.Time = time.Time{} + moments := dexcom.Moments{nil, moment1, moment3, moment2, moment4, nil} + Expect(moments.CompactBySystemTimeRaw()).To(Equal(dexcom.Moments{moment1, moment2})) }) }) }) diff --git a/dexcom/provider/provider.go b/dexcom/provider/provider.go index 33a3cb5cb7..cafaff401d 100644 --- a/dexcom/provider/provider.go +++ b/dexcom/provider/provider.go @@ -117,7 +117,7 @@ func (p *Provider) OnCreate(ctx context.Context, providerSession *auth.ProviderS if _, err = p.dataSourceClient.Update(ctx, source.ID, nil, update); err != nil { // Attempt to delete task if data source not marked as connected - if taskErr := p.taskClient.DeleteTask(context.WithoutCancel(ctx), task.ID); taskErr != nil { + if taskErr := p.taskClient.DeleteTask(context.WithoutCancel(ctx), task.ID, nil); taskErr != nil { logger.WithError(taskErr).Error("Failure deleting task after failed data source update") } @@ -151,7 +151,7 @@ func (p *Provider) OnDelete(ctx context.Context, providerSession *auth.ProviderS logger.WithError(err).WithField(dexcom.DataKeyDataSourceID, dataSourceID).Error("Unable to update data source while deleting provider session") } } - if err = p.taskClient.DeleteTask(ctx, task.ID); err != nil { + if err = p.taskClient.DeleteTask(ctx, task.ID, nil); err != nil { logger.WithError(err).WithField("taskId", task.ID).Error("unable to delete task while deleting provider session") } } diff --git a/dexcom/provider/provider_test.go b/dexcom/provider/provider_test.go deleted file mode 100644 index 887329603b..0000000000 --- a/dexcom/provider/provider_test.go +++ /dev/null @@ -1,8 +0,0 @@ -package provider_test - -import ( - . "github.com/onsi/ginkgo/v2" -) - -var _ = Describe("Provider", func() { -}) diff --git a/duration/duration.go b/duration/duration.go new file mode 100644 index 0000000000..c5ee713bea --- /dev/null +++ b/duration/duration.go @@ -0,0 +1,42 @@ +package duration + +import ( + "math" + "strconv" + "time" + + "github.com/tidepool-org/platform/errors" +) + +// Parse returns the duration represented by specified value. Value may be a standard Go duration string +// or a number string without units, which is then assumed to be in the specified units. +func Parse(value string, units time.Duration) (time.Duration, error) { + if units <= 0 { + return 0, errors.New("units is invalid") + } + + // Attempt to parse as standard Go duration string first. + if valueParsed, err := time.ParseDuration(value); err == nil { + return valueParsed, nil + } + + // Determine minimum and maximum possible values in the specified units. + valueMinimum := math.MinInt64 / int64(units) + valueMaximum := math.MaxInt64 / int64(units) + + // Attempt to parse as an integer then float without units, which is then assumed to be in the specified + // units. Integer values are inclusive of the minimum and maximum, since the multiply below is exact. Float + // values are exclusive, because float64 rounds the minimum and maximum themselves past the true bounds + // whenever either exceeds 2^53, which happens for units finer than 1024ns. + if valueInt, err := strconv.ParseInt(value, 10, 64); err == nil { + if valueInt >= valueMinimum && valueInt <= valueMaximum { + return time.Duration(valueInt * int64(units)), nil + } + } else if valueFloat, err := strconv.ParseFloat(value, 64); err == nil { + if valueFloat > float64(valueMinimum) && valueFloat < float64(valueMaximum) { + return time.Duration(valueFloat * float64(units)), nil + } + } + + return 0, errors.New("unable to parse duration") +} diff --git a/duration/duration_suite_test.go b/duration/duration_suite_test.go new file mode 100644 index 0000000000..02dc2aa0e6 --- /dev/null +++ b/duration/duration_suite_test.go @@ -0,0 +1,11 @@ +package duration_test + +import ( + "testing" + + "github.com/tidepool-org/platform/test" +) + +func TestSuite(t *testing.T) { + test.Test(t) +} diff --git a/duration/duration_test.go b/duration/duration_test.go new file mode 100644 index 0000000000..1a0aaaa735 --- /dev/null +++ b/duration/duration_test.go @@ -0,0 +1,85 @@ +package duration_test + +import ( + "math" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tidepool-org/platform/duration" + "github.com/tidepool-org/platform/test" +) + +var _ = Describe("Duration", func() { + Context("Parse", func() { + DescribeTable("returns the expected duration when the value", + func(value string, units time.Duration, expectedDuration time.Duration) { + Expect(test.MustDuration(duration.Parse(value, units))).To(Equal(expectedDuration)) + }, + Entry("has a single unit", "45s", time.Second, 45*time.Second), + Entry("has multiple units", "1m30s", time.Second, 90*time.Second), + Entry("has a fractional unit", "1.5h", time.Second, 90*time.Minute), + Entry("has units, which override the units argument", "45s", time.Hour, 45*time.Second), + Entry("has units and is negative", "-1m", time.Second, -time.Minute), + Entry("has no units, using the units argument", "45", time.Second, 45*time.Second), + Entry("has no units, using non-second units", "45", time.Millisecond, 45*time.Millisecond), + Entry("has no units and is fractional", "1.5", time.Second, 1500*time.Millisecond), + Entry("has no units and is fractional, using non-second units", "1.5", time.Hour, 90*time.Minute), + Entry("has no units and is in exponent notation", "1e3", time.Second, 1000*time.Second), + Entry("has no units and is negative", "-3", time.Second, -3*time.Second), + Entry("has no units and is zero", "0", time.Second, time.Duration(0)), + Entry("has no units and is finer than a nanosecond", "0.0000000001", time.Second, time.Duration(0)), + // The integer bounds are inclusive and the multiply is exact, so the extremes convert without loss. + Entry("has no units and is the largest duration", "9223372036854775807", time.Nanosecond, time.Duration(math.MaxInt64)), + Entry("has no units and is the smallest duration", "-9223372036854775808", time.Nanosecond, time.Duration(math.MinInt64)), + Entry("has no units and is the largest whole number of units", "9223372036", time.Second, 9223372036*time.Second), + Entry("has no units and is the smallest whole number of units", "-9223372036", time.Second, -9223372036*time.Second), + Entry("has no units and is an integer beyond float64 precision", "9007199254740993", time.Nanosecond, time.Duration(9007199254740993)), + // The float bounds are exclusive, and float64 carries only 53 bits, so these two do lose the low bits. + Entry("has no units, is fractional, and is the largest float64 below the largest duration", "9223372036854774784.0", time.Nanosecond, time.Duration(9223372036854774784)), + Entry("has no units, is fractional, and is beyond float64 precision", "9223372035.5", time.Second, time.Duration(9223372035500000256)), + ) + + DescribeTable("returns an error when the value", + func(value string, units time.Duration) { + parsedDuration, err := duration.Parse(value, units) + Expect(err).To(MatchError("unable to parse duration")) + Expect(parsedDuration).To(BeZero()) + }, + Entry("is empty", "", time.Second), + Entry("is not a number or a duration", "invalid", time.Second), + Entry("has a leading space", " 45", time.Second), + Entry("has a trailing space", "45 ", time.Second), + Entry("has an unrecognized unit", "45y", time.Second), + Entry("has an upper case unit", "45S", time.Second), + Entry("has trailing characters after the units", "45sx", time.Second), + Entry("is hexadecimal", "0x10", time.Second), + Entry("is not a number", "NaN", time.Second), + Entry("is not a number in lower case", "nan", time.Second), + Entry("is positive infinity", "Inf", time.Second), + Entry("is negative infinity", "-Inf", time.Second), + Entry("is infinity spelled out", "infinity", time.Second), + Entry("overflows the units in exponent notation", "1e30", time.Second), + Entry("overflows the units", "9223372036854775807", time.Second), + Entry("is one past the largest whole number of units", "9223372037", time.Second), + Entry("is one past the smallest whole number of units", "-9223372037", time.Second), + Entry("is one past the largest duration", "9223372036854775808", time.Nanosecond), + // The float bounds are exclusive, so a fraction above the largest whole number of units is given up. + Entry("is fractional and above the largest whole number of units", "9223372036.5", time.Second), + ) + + DescribeTable("returns an error when the units", + func(value string, units time.Duration) { + parsedDuration, err := duration.Parse(value, units) + Expect(err).To(MatchError("units is invalid")) + Expect(parsedDuration).To(BeZero()) + }, + Entry("is zero", "45", time.Duration(0)), + Entry("is negative", "45", -time.Second), + Entry("is the smallest duration", "45", time.Duration(math.MinInt64)), + Entry("is zero and the value has units", "45s", time.Duration(0)), + Entry("is zero and the value is not parsable", "invalid", time.Duration(0)), + ) + }) +}) diff --git a/ehr/reconcile/runner.go b/ehr/reconcile/runner.go index 5c9e61b7cb..08f79322e8 100644 --- a/ehr/reconcile/runner.go +++ b/ehr/reconcile/runner.go @@ -41,8 +41,8 @@ func (r *Runner) GetRunnerType() string { return Type } -func (r *Runner) GetRunnerDeadline() time.Time { - return time.Now().Add(TaskDurationMaximum * 3) +func (r *Runner) GetRunnerDeadline() time.Duration { + return TaskDurationMaximum * 3 } func (r *Runner) GetRunnerTimeout() time.Duration { @@ -117,7 +117,7 @@ func (r *Runner) getSyncTasks(ctx context.Context) (map[string]task.Task, error) func (r *Runner) reconcileTasks(ctx context.Context, tsk *task.Task, plan ReconciliationPlan) { for _, t := range plan.ToDelete { - if err := r.taskClient.DeleteTask(ctx, t.ID); err != nil { + if err := r.taskClient.DeleteTask(ctx, t.ID, nil); err != nil { tsk.AppendError(errors.Wrap(err, "unable to delete task")) } } @@ -127,7 +127,7 @@ func (r *Runner) reconcileTasks(ctx context.Context, tsk *task.Task, plan Reconc } } for id, update := range plan.ToUpdate { - if _, err := r.taskClient.UpdateTask(ctx, id, update); err != nil { + if _, err := r.taskClient.UpdateTask(ctx, id, nil, update); err != nil { tsk.AppendError(errors.Wrap(err, "unable to update task")) } } diff --git a/ehr/reconcile/runner_test.go b/ehr/reconcile/runner_test.go index ad499aa3ae..4ec44b2d17 100644 --- a/ehr/reconcile/runner_test.go +++ b/ehr/reconcile/runner_test.go @@ -82,7 +82,7 @@ var _ = Describe("Runner", func() { clinicsClient.EXPECT().ListEHREnabledClinics(gomock.Any()).Return(clinics, nil) taskClient.EXPECT().ListTasks(gomock.Any(), gomock.Any(), gomock.Any()).Return(tasksList, nil) - taskClient.EXPECT().DeleteTask(gomock.Any(), gomock.Eq(tasks[*toBeDeleted.Id].ID)).Return(nil) + taskClient.EXPECT().DeleteTask(gomock.Any(), gomock.Eq(tasks[*toBeDeleted.Id].ID), gomock.Nil()).Return(nil) taskClient.EXPECT().CreateTask(gomock.Any(), gomock.Any()).Return(nil, nil) runner.Run(context.Background(), t) }) diff --git a/ehr/reconcile/task.go b/ehr/reconcile/task.go index 6d5df324c3..1b4e55245b 100644 --- a/ehr/reconcile/task.go +++ b/ehr/reconcile/task.go @@ -1,8 +1,6 @@ package reconcile import ( - "time" - "github.com/tidepool-org/platform/pointer" "github.com/tidepool-org/platform/task" ) @@ -13,8 +11,7 @@ const ( func NewTaskCreate() *task.TaskCreate { return &task.TaskCreate{ - Name: pointer.FromString(Type), - Type: Type, - AvailableTime: pointer.FromAny(time.Now().UTC()), + Name: pointer.FromString(Type), + Type: Type, } } diff --git a/ehr/reconcile/task_test.go b/ehr/reconcile/task_test.go index 5b0dd6bc0e..72daa99119 100644 --- a/ehr/reconcile/task_test.go +++ b/ehr/reconcile/task_test.go @@ -1,8 +1,6 @@ package reconcile_test import ( - "time" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" @@ -17,7 +15,6 @@ var _ = Describe("Task", func() { Expect(create).ToNot(BeNil()) Expect(create.Name).To(PointTo(Equal(reconcile.Type))) Expect(create.Type).To(Equal(reconcile.Type)) - Expect(create.AvailableTime).To(PointTo(BeTemporally("~", time.Now(), 3*time.Second))) }) }) }) diff --git a/ehr/sync/runner.go b/ehr/sync/runner.go index 428f7a27c7..107fb665b1 100644 --- a/ehr/sync/runner.go +++ b/ehr/sync/runner.go @@ -31,8 +31,8 @@ func (r *Runner) GetRunnerType() string { return Type } -func (r *Runner) GetRunnerDeadline() time.Time { - return time.Now().Add(TaskDurationMaximum * 3) +func (r *Runner) GetRunnerDeadline() time.Duration { + return TaskDurationMaximum * 3 } func (r *Runner) GetRunnerTimeout() time.Duration { @@ -58,9 +58,8 @@ func (r *Runner) Run(ctx context.Context, tsk *task.Task) { func (r *Runner) doRun(ctx context.Context, tsk *task.Task) { clinicId, err := GetClinicId(tsk.Data) if err != nil { - tsk.AppendError(errors.Wrap(err, "unable to get clinicId from task data")) // Unrecoverable condition, move the task to failed state so it won't be retried - tsk.SetFailed() + tsk.SetFailedWithError(errors.Wrap(err, "unable to get clinicId from task data")) return } diff --git a/ehr/sync/task.go b/ehr/sync/task.go index 7d32118966..88db979785 100644 --- a/ehr/sync/task.go +++ b/ehr/sync/task.go @@ -21,9 +21,8 @@ func TaskName(clinicId string) string { func NewTaskCreate(clinicId string, cadence time.Duration) *task.TaskCreate { tsk := &task.TaskCreate{ - Name: pointer.FromString(TaskName(clinicId)), - Type: Type, - AvailableTime: pointer.FromAny(time.Now().UTC()), + Name: pointer.FromString(TaskName(clinicId)), + Type: Type, Data: map[string]interface{}{ "clinicId": clinicId, }, diff --git a/ehr/sync/task_test.go b/ehr/sync/task_test.go index e561aca722..e288c83f98 100644 --- a/ehr/sync/task_test.go +++ b/ehr/sync/task_test.go @@ -19,7 +19,6 @@ var _ = Describe("Task", func() { Expect(create).ToNot(BeNil()) Expect(create.Name).To(PointTo(Equal(sync.TaskName(*clinic.Id)))) Expect(create.Type).To(Equal(sync.Type)) - Expect(create.AvailableTime).ToNot(BeNil()) }) It("stores the clinic id in the data", func() { diff --git a/env.sh b/env.sh index 13f8a66e31..5dc4e67cb6 100644 --- a/env.sh +++ b/env.sh @@ -1,6 +1,8 @@ +# shellcheck shell=sh + export TIDEPOOL_ENV="local" export TIDEPOOL_LOGGER_LEVEL="debug" -export TIDEPOOL_STORE_ADDRESSES="localhost" +export TIDEPOOL_STORE_ADDRESSES="127.0.0.1" export TIDEPOOL_STORE_DATABASE="tidepool" export TIDEPOOL_STORE_TLS="false" export TIDEPOOL_SERVER_TLS="false" diff --git a/env.test.sh b/env.test.sh index cd71c2c36b..5d893811bd 100644 --- a/env.test.sh +++ b/env.test.sh @@ -1,11 +1,14 @@ +# shellcheck shell=sh + # Clear all TIDEPOOL_* environment variables -unset `env | cut -d'=' -f1 | grep '^TIDEPOOL_' | xargs` +# shellcheck disable=SC2046 +unset $(env | cut -d'=' -f1 | grep '^TIDEPOOL_' | xargs) 2> /dev/null || true export TIDEPOOL_ENV="test" export TIDEPOOL_LOGGER_LEVEL="error" -export TIDEPOOL_STORE_ADDRESSES="localhost" +export TIDEPOOL_STORE_ADDRESSES="127.0.0.1" export TIDEPOOL_STORE_TLS="false" export TIDEPOOL_STORE_DATABASE="tidepool_test" diff --git a/errors/errors.go b/errors/errors.go index a31ba89677..faadebf934 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -156,15 +156,7 @@ func WithSource(err error, src Source) error { if _, arrayOK := err.(*array); arrayOK { return err } else if objectErr, objectOK := err.(*object); objectOK { - return &object{ - Code: objectErr.Code, - Title: objectErr.Title, - Detail: objectErr.Detail, - Source: s, - Meta: objectErr.Meta, - Caller: objectErr.Caller, - Cause: objectErr.Cause, - } + return objectErr.withSource(s) } else if err != nil { return &object{ Detail: err.Error(), @@ -200,15 +192,7 @@ func WithMeta(err error, meta interface{}) error { if _, arrayOK := err.(*array); arrayOK { return err } else if objectErr, objectOK := err.(*object); objectOK { - return &object{ - Code: objectErr.Code, - Title: objectErr.Title, - Detail: objectErr.Detail, - Source: objectErr.Source, - Meta: meta, - Caller: objectErr.Caller, - Cause: objectErr.Cause, - } + return objectErr.withMeta(meta) } else if err != nil { return &object{ Detail: err.Error(), @@ -315,9 +299,6 @@ type Serializable struct { } func NewSerializable(err error) *Serializable { - if err == nil { - return nil - } return &Serializable{ Error: err, } @@ -363,11 +344,15 @@ func (s Serializable) MarshalJSON() ([]byte, error) { if arrayErr, arrayOK := s.Error.(*array); arrayOK { return json.Marshal(arrayErr.Errors) } else if objectErr, objectOK := s.Error.(*object); objectOK { - return json.Marshal(objectErr) + bites, err := json.Marshal(objectErr) + if err != nil && objectErr.Meta != nil { // If failure to marshal with meta, then remove meta and try again; likely unmarshallable value within meta + bites, err = json.Marshal(objectErr.withMeta(nil)) + } + return bites, err } else if s.Error != nil { return []byte(strconv.Quote(s.Error.Error())), nil } - return nil, nil + return []byte("null"), nil } func (s *Serializable) UnmarshalJSON(bites []byte) error { @@ -399,7 +384,11 @@ func (s Serializable) MarshalBSONValue() (bsontype.Type, []byte, error) { if arrayErr, arrayOK := s.Error.(*array); arrayOK { return bson.MarshalValue(arrayErr.Errors) } else if objectErr, objectOK := s.Error.(*object); objectOK { - return bson.MarshalValue(objectErr) + bsonType, bites, err := bson.MarshalValue(objectErr) + if err != nil && objectErr.Meta != nil { // If failure to marshal with meta, then remove meta and try again; likely unmarshallable value within meta + bsonType, bites, err = bson.MarshalValue(objectErr.withMeta(nil)) + } + return bsonType, bites, err } else if s.Error != nil { return bsontype.String, bsoncore.AppendString(nil, s.Error.Error()), nil } @@ -668,6 +657,30 @@ func (o *object) Is(target error) bool { return o.Cause != nil && o.Cause.Error == target } +func (o *object) withSource(src *source) *object { + return &object{ + Code: o.Code, + Title: o.Title, + Detail: o.Detail, + Source: src, + Meta: o.Meta, + Caller: o.Caller, + Cause: o.Cause, + } +} + +func (o *object) withMeta(meta any) *object { + return &object{ + Code: o.Code, + Title: o.Title, + Detail: o.Detail, + Source: o.Source, + Meta: meta, + Caller: o.Caller, + Cause: o.Cause, + } +} + type contextKey string const errorContextKey contextKey = "error" diff --git a/errors/errors_test.go b/errors/errors_test.go index 7a5596be8b..13876ee7eb 100644 --- a/errors/errors_test.go +++ b/errors/errors_test.go @@ -182,9 +182,10 @@ var _ = Describe("Errors", func() { }) Context("NewSerializable", func() { - It("returns nil if the error is nil", func() { + It("returns a serializable if the error is nil", func() { serializable := errors.NewSerializable(nil) - Expect(serializable).To(BeNil()) + Expect(serializable).ToNot(BeNil()) + Expect(serializable.Error).To(BeNil()) }) It("returns a serializable if the error is not nil", func() { diff --git a/go.mod b/go.mod index f43b7d4bd2..1da9663fad 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/onsi/ginkgo/v2 v2.27.5 github.com/onsi/gomega v1.39.0 github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/client_model v0.6.1 github.com/rinchsan/device-check-go v1.3.0 github.com/solworktech/md2pdf/v2 v2.2.18 github.com/tidepool-org/clinic/client v0.0.0-20250122123230-f89e2b1540dc @@ -105,7 +106,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect - github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/radovskyb/watcher v1.0.7 // indirect diff --git a/log/test/serializer.go b/log/test/serializer.go index ae8851777e..960d2907e2 100644 --- a/log/test/serializer.go +++ b/log/test/serializer.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "regexp" + "sync" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" @@ -12,10 +13,13 @@ import ( type Serializer struct { SerializedFields []log.Fields + mutex *sync.Mutex } func NewSerializer() *Serializer { - return &Serializer{} + return &Serializer{ + mutex: &sync.Mutex{}, + } } func (s *Serializer) Serialize(fields log.Fields) error { @@ -23,6 +27,9 @@ func (s *Serializer) Serialize(fields log.Fields) error { return errors.New("fields are missing") } + s.mutex.Lock() + defer s.mutex.Unlock() + s.SerializedFields = append(s.SerializedFields, fields) return nil @@ -76,6 +83,9 @@ func (s *Serializer) AssertErrorExpression(messageExpression *regexp.Regexp, con } func (s *Serializer) assertContainsFields(containsFields []log.Fields, matcher func(serializedFields log.Fields) bool) { + s.mutex.Lock() + defer s.mutex.Unlock() + joinedContainsFields := s.joinContainsFields(containsFields) for _, serializedFields := range s.SerializedFields { if s.serializedFieldsContainsFields(serializedFields, joinedContainsFields) && (matcher == nil || matcher(serializedFields)) { diff --git a/oauth/client/client.go b/oauth/client/client.go index 3b0dde4187..9b2198ff56 100644 --- a/oauth/client/client.go +++ b/oauth/client/client.go @@ -2,6 +2,9 @@ package client import ( "context" + "net/http" + + "golang.org/x/oauth2" "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/errors" @@ -12,23 +15,24 @@ import ( type Client struct { baseClient *client.Client + httpClient *http.Client tokenSourceSource oauth.TokenSourceSource } -func New(config *client.Config, tokenSourceSource oauth.TokenSourceSource) (*Client, error) { - return NewWithErrorParser(config, tokenSourceSource, nil) +func New(config *client.Config, httpClient *http.Client, tokenSourceSource oauth.TokenSourceSource) (*Client, error) { + return NewWithErrorParser(config, httpClient, tokenSourceSource, nil) } -func NewWithErrorParser(config *client.Config, tokenSourceSource oauth.TokenSourceSource, errorResponseParser client.ErrorResponseParser) (*Client, error) { +func NewWithErrorParser(config *client.Config, httpClient *http.Client, tokenSourceSource oauth.TokenSourceSource, errorResponseParser client.ErrorResponseParser) (*Client, error) { baseClient, err := client.NewWithErrorParser(config, errorResponseParser) if err != nil { return nil, err } - return NewWithClient(baseClient, tokenSourceSource) + return NewWithClient(baseClient, httpClient, tokenSourceSource) } -func NewWithClient(baseClient *client.Client, tokenSourceSource oauth.TokenSourceSource) (*Client, error) { +func NewWithClient(baseClient *client.Client, httpClient *http.Client, tokenSourceSource oauth.TokenSourceSource) (*Client, error) { if baseClient == nil { return nil, errors.New("base client is missing") } @@ -39,6 +43,7 @@ func NewWithClient(baseClient *client.Client, tokenSourceSource oauth.TokenSourc return &Client{ baseClient: baseClient, tokenSourceSource: tokenSourceSource, + httpClient: httpClient, }, nil } @@ -81,6 +86,14 @@ func (c *Client) SendOAuthRequest(ctx context.Context, method string, url string } func (c *Client) sendOAuthRequest(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody interface{}, responseBody interface{}, inspectors []request.ResponseInspector, tokenSource oauth.TokenSource) error { + if ctx == nil { + return errors.New("context is missing") + } + + if c.httpClient != nil { + ctx = context.WithValue(ctx, oauth2.HTTPClient, c.httpClient) + } + httpClient, err := tokenSource.HTTPClient(ctx, c.tokenSourceSource) if err != nil { return err diff --git a/oauth/client/client_test.go b/oauth/client/client_test.go index 3ea6e58958..fdf2dd832e 100644 --- a/oauth/client/client_test.go +++ b/oauth/client/client_test.go @@ -11,6 +11,8 @@ import ( . "github.com/onsi/gomega" . "github.com/onsi/gomega/ghttp" + "go.uber.org/mock/gomock" + "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/errors" errorsTest "github.com/tidepool-org/platform/errors/test" @@ -36,7 +38,8 @@ var _ = Describe("Client", func() { var address string var userAgent string var baseConfig *client.Config - var tokenSourceSource *oauthTest.TokenSourceSource + var mockController *gomock.Controller + var mockTokenSourceSource *oauthTest.MockTokenSourceSource BeforeEach(func() { address = testHttp.NewAddress() @@ -44,22 +47,19 @@ var _ = Describe("Client", func() { baseConfig = client.NewConfig() baseConfig.Address = address baseConfig.UserAgent = userAgent - tokenSourceSource = oauthTest.NewTokenSourceSource() - }) - - AfterEach(func() { - tokenSourceSource.AssertOutputsEmpty() + mockController = gomock.NewController(GinkgoT()) + mockTokenSourceSource = oauthTest.NewMockTokenSourceSource(mockController) }) Context("New", func() { It("returns an error when token source source is missing", func() { - clnt, err := oauthClient.New(baseConfig, nil) + clnt, err := oauthClient.New(baseConfig, nil, nil) Expect(err).To(MatchError("token source source is missing")) Expect(clnt).To(BeNil()) }) It("returns successfully", func() { - Expect(oauthClient.New(baseConfig, tokenSourceSource)).ToNot(BeNil()) + Expect(oauthClient.New(baseConfig, nil, mockTokenSourceSource)).ToNot(BeNil()) }) }) @@ -68,7 +68,7 @@ var _ = Describe("Client", func() { JustBeforeEach(func() { var err error - clnt, err = oauthClient.New(baseConfig, tokenSourceSource) + clnt, err = oauthClient.New(baseConfig, nil, mockTokenSourceSource) Expect(err).ToNot(HaveOccurred()) Expect(clnt).ToNot(BeNil()) }) @@ -172,7 +172,7 @@ var _ = Describe("Client", func() { var requestString string var requestBody *RequestBody var responseString string - var tokenSource *oauthTest.TokenSource + var mockTokenSource *oauthTest.MockTokenSource var clnt *oauthClient.Client BeforeEach(func() { @@ -188,13 +188,13 @@ var _ = Describe("Client", func() { requestString = test.RandomStringFromRangeAndCharset(0, 32, test.CharsetText) requestBody = &RequestBody{Request: requestString} responseString = test.RandomStringFromRangeAndCharset(0, 32, test.CharsetText) - tokenSource = oauthTest.NewTokenSource() + mockTokenSource = oauthTest.NewMockTokenSource(mockController) baseConfig.Address = server.URL() }) JustBeforeEach(func() { var err error - clnt, err = oauthClient.New(baseConfig, tokenSourceSource) + clnt, err = oauthClient.New(baseConfig, nil, mockTokenSourceSource) Expect(err).ToNot(HaveOccurred()) Expect(clnt).ToNot(BeNil()) }) @@ -203,7 +203,6 @@ var _ = Describe("Client", func() { if server != nil { server.Close() } - tokenSource.AssertOutputsEmpty() }) Context("SendOAuthRequest", func() { @@ -213,26 +212,21 @@ var _ = Describe("Client", func() { responseBody = &ResponseBody{} }) + It("returns error when context is missing", func() { + Expect(clnt.SendOAuthRequest(context.Context(nil), method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(MatchError("context is missing")) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source is missing", func() { Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, nil)).To(MatchError("token source is missing")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) When("token source is not missing", func() { - var expectedHTTPClientInputs []oauthTest.HTTPClientInput - - BeforeEach(func() { - expectedHTTPClientInputs = []oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}} - }) - - AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal(expectedHTTPClientInputs)) - }) - It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(Equal(responseErr)) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(Equal(responseErr)) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -241,44 +235,37 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - expectedHTTPClientInputs = []oauthTest.HTTPClientInput{{Context: nil, TokenSourceSource: tokenSourceSource}} - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(MatchError("context is missing")) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when method is missing", func() { - Expect(clnt.SendOAuthRequest(ctx, "", url, mutators, requestBody, responseBody, nil, tokenSource)).To(MatchError("method is missing")) + Expect(clnt.SendOAuthRequest(ctx, "", url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(MatchError("method is missing")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when url is missing", func() { - Expect(clnt.SendOAuthRequest(ctx, method, "", mutators, requestBody, responseBody, nil, tokenSource)).To(MatchError("url is missing")) + Expect(clnt.SendOAuthRequest(ctx, method, "", mutators, requestBody, responseBody, nil, mockTokenSource)).To(MatchError("url is missing")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when the request object cannot be encoded", func() { invalidRequestBody := struct{ Func interface{} }{func() {}} - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, invalidRequestBody, responseBody, nil, tokenSource).Error()).To(MatchRegexp("unable to serialize request to .*; json: unsupported type: func()")) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, invalidRequestBody, responseBody, nil, mockTokenSource).Error()).To(MatchRegexp("unable to serialize request to .*; json: unsupported type: func()")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when mutator returns an error", func() { errorMutator := request.NewHeaderMutator("", "") invalidMutators := []request.RequestMutator{headerMutator, errorMutator, parameterMutator} - Expect(clnt.SendOAuthRequest(ctx, method, url, invalidMutators, requestBody, responseBody, nil, tokenSource).Error()).To(MatchRegexp("unable to mutate request to .*; key is missing")) + Expect(clnt.SendOAuthRequest(ctx, method, url, invalidMutators, requestBody, responseBody, nil, mockTokenSource).Error()).To(MatchRegexp("unable to mutate request to .*; key is missing")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource).Error()).To(MatchRegexp("unable to perform request to .*: connect: connection refused")) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource).Error()).To(MatchRegexp("unable to perform request to .*: connect: connection refused")) }) Context("with a successful response and no request body", func() { @@ -295,7 +282,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, nil, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, nil, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(Equal(responseString)) @@ -317,7 +304,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorBadRequest()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -341,7 +328,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorBadRequest()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -349,9 +336,9 @@ var _ = Describe("Client", func() { Context("with an unauthorized response 401", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest(method, path, fmt.Sprintf("%s=%s", parameterMutator.Key, parameterMutator.Value)), @@ -370,11 +357,10 @@ var _ = Describe("Client", func() { RespondWith(http.StatusUnauthorized, "NOT JSON", responseHeaders), ), ) - expectedHTTPClientInputs = []oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}} }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorUnauthenticated()) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -395,7 +381,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorUnauthorized()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -416,7 +402,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorResourceNotFound()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -440,7 +426,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorResourceNotFound()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -461,7 +447,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorTooManyRequests()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -482,7 +468,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf(`unexpected response status code 500 from %s "%s?%s=%s"`, method, url, parameterMutator.Key, parameterMutator.Value))) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -510,7 +496,7 @@ var _ = Describe("Client", func() { Expect(err).ToNot(HaveOccurred()) Expect(req).ToNot(BeNil()) res := &http.Response{StatusCode: http.StatusInternalServerError} - err = clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err = clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorUnexpectedResponse(res, req)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -531,7 +517,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorJSONMalformed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -552,7 +538,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(BeEmpty()) @@ -574,7 +560,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(BeEmpty()) @@ -595,7 +581,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, nil, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, nil, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(Equal(responseString)) @@ -616,7 +602,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, strings.NewReader(requestString), responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, strings.NewReader(requestString), responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(Equal(responseString)) @@ -638,7 +624,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(Equal(responseString)) @@ -660,7 +646,7 @@ var _ = Describe("Client", func() { }) It("returns success without parsing response body", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, nil, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, nil, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) }) diff --git a/oauth/provider/client/client.go b/oauth/provider/client/client.go index 02d39998c7..8aa13ece97 100644 --- a/oauth/provider/client/client.go +++ b/oauth/provider/client/client.go @@ -1,6 +1,8 @@ package client import ( + "net/http" + "github.com/lestrrat-go/jwx/v2/jwk" "github.com/tidepool-org/platform/client" @@ -14,11 +16,11 @@ type Provider struct { *oauthClient.Client } -func New(name string, config *Config, jwks jwk.Set) (*Provider, error) { - return NewWithErrorParser(name, config, jwks, nil) +func New(name string, config *Config, httpClient *http.Client, jwks jwk.Set) (*Provider, error) { + return NewWithErrorParser(name, config, httpClient, jwks, nil) } -func NewWithErrorParser(name string, config *Config, jwks jwk.Set, errorResponseParser client.ErrorResponseParser) (*Provider, error) { +func NewWithErrorParser(name string, config *Config, httpClient *http.Client, jwks jwk.Set, errorResponseParser client.ErrorResponseParser) (*Provider, error) { if name == "" { return nil, errors.New("name is missing") } @@ -30,7 +32,7 @@ func NewWithErrorParser(name string, config *Config, jwks jwk.Set, errorResponse if err != nil { return nil, err } - clnt, err := oauthClient.NewWithErrorParser(config.ClientConfig, prvdr, errorResponseParser) + clnt, err := oauthClient.NewWithErrorParser(config.ClientConfig, httpClient, prvdr, errorResponseParser) if err != nil { return nil, err } diff --git a/oauth/test/token_source.go b/oauth/test/token_source.go deleted file mode 100644 index 9223340bf7..0000000000 --- a/oauth/test/token_source.go +++ /dev/null @@ -1,113 +0,0 @@ -package test - -import ( - "context" - "net/http" - - "github.com/tidepool-org/platform/oauth" -) - -type HTTPClientInput struct { - Context context.Context - TokenSourceSource oauth.TokenSourceSource -} - -type HTTPClientOutput struct { - HTTPClient *http.Client - Error error -} - -type UpdateTokenOutput struct { - Updated bool - Error error -} - -type ExpireTokenOutput struct { - Expired bool - Error error -} - -type TokenSource struct { - HTTPClientInvocations int - HTTPClientInputs []HTTPClientInput - HTTPClientStub func(ctx context.Context, tokenSourceSource oauth.TokenSourceSource) (*http.Client, error) - HTTPClientOutputs []HTTPClientOutput - HTTPClientOutput *HTTPClientOutput - UpdateTokenInvocations int - UpdateTokenInputs []context.Context - UpdateTokenStub func(ctx context.Context) (bool, error) - UpdateTokenOutputs []UpdateTokenOutput - UpdateTokenOutput *UpdateTokenOutput - ExpireTokenInvocations int - ExpireTokenInputs []context.Context - ExpireTokenStub func(ctx context.Context) (bool, error) - ExpireTokenOutputs []ExpireTokenOutput - ExpireTokenOutput *ExpireTokenOutput -} - -func NewTokenSource() *TokenSource { - return &TokenSource{} -} - -func (t *TokenSource) HTTPClient(ctx context.Context, tokenSourceSource oauth.TokenSourceSource) (*http.Client, error) { - t.HTTPClientInvocations++ - t.HTTPClientInputs = append(t.HTTPClientInputs, HTTPClientInput{Context: ctx, TokenSourceSource: tokenSourceSource}) - if t.HTTPClientStub != nil { - return t.HTTPClientStub(ctx, tokenSourceSource) - } - if len(t.HTTPClientOutputs) > 0 { - output := t.HTTPClientOutputs[0] - t.HTTPClientOutputs = t.HTTPClientOutputs[1:] - return output.HTTPClient, output.Error - } - if t.HTTPClientOutput != nil { - return t.HTTPClientOutput.HTTPClient, t.HTTPClientOutput.Error - } - panic("HTTPClient has no output") -} - -func (t *TokenSource) UpdateToken(ctx context.Context) (bool, error) { - t.UpdateTokenInvocations++ - t.UpdateTokenInputs = append(t.UpdateTokenInputs, ctx) - if t.UpdateTokenStub != nil { - return t.UpdateTokenStub(ctx) - } - if len(t.UpdateTokenOutputs) > 0 { - output := t.UpdateTokenOutputs[0] - t.UpdateTokenOutputs = t.UpdateTokenOutputs[1:] - return output.Updated, output.Error - } - if t.UpdateTokenOutput != nil { - return t.UpdateTokenOutput.Updated, t.UpdateTokenOutput.Error - } - panic("UpdateToken has no output") -} - -func (t *TokenSource) ExpireToken(ctx context.Context) (bool, error) { - t.ExpireTokenInvocations++ - t.ExpireTokenInputs = append(t.ExpireTokenInputs, ctx) - if t.ExpireTokenStub != nil { - return t.ExpireTokenStub(ctx) - } - if len(t.ExpireTokenOutputs) > 0 { - output := t.ExpireTokenOutputs[0] - t.ExpireTokenOutputs = t.ExpireTokenOutputs[1:] - return output.Expired, output.Error - } - if t.ExpireTokenOutput != nil { - return t.ExpireTokenOutput.Expired, t.ExpireTokenOutput.Error - } - panic("ExpireToken has no output") -} - -func (t *TokenSource) AssertOutputsEmpty() { - if len(t.HTTPClientOutputs) > 0 { - panic("HTTPClientOutputs is not empty") - } - if len(t.UpdateTokenOutputs) > 0 { - panic("UpdateTokenOutputs is not empty") - } - if len(t.ExpireTokenOutputs) > 0 { - panic("ExpireTokenOutputs is not empty") - } -} diff --git a/oauth/test/token_source_source.go b/oauth/test/token_source_source.go deleted file mode 100644 index 2ae4f4d6b6..0000000000 --- a/oauth/test/token_source_source.go +++ /dev/null @@ -1,54 +0,0 @@ -package test - -import ( - "context" - - "golang.org/x/oauth2" - - "github.com/tidepool-org/platform/auth" -) - -type TokenSourceInput struct { - Context context.Context - Token *auth.OAuthToken -} - -type TokenSourceOutput struct { - TokenSource oauth2.TokenSource - Error error -} - -type TokenSourceSource struct { - TokenSourceInvocations int - TokenSourceInputs []TokenSourceInput - TokenSourceStub func(ctx context.Context, token *auth.OAuthToken) (oauth2.TokenSource, error) - TokenSourceOutputs []TokenSourceOutput - TokenSourceOutput *TokenSourceOutput -} - -func NewTokenSourceSource() *TokenSourceSource { - return &TokenSourceSource{} -} - -func (t *TokenSourceSource) TokenSource(ctx context.Context, token *auth.OAuthToken) (oauth2.TokenSource, error) { - t.TokenSourceInvocations++ - t.TokenSourceInputs = append(t.TokenSourceInputs, TokenSourceInput{Context: ctx, Token: token}) - if t.TokenSourceStub != nil { - return t.TokenSourceStub(ctx, token) - } - if len(t.TokenSourceOutputs) > 0 { - output := t.TokenSourceOutputs[0] - t.TokenSourceOutputs = t.TokenSourceOutputs[1:] - return output.TokenSource, output.Error - } - if t.TokenSourceOutput != nil { - return t.TokenSourceOutput.TokenSource, t.TokenSourceOutput.Error - } - panic("TokenSource has no output") -} - -func (t *TokenSourceSource) AssertOutputsEmpty() { - if len(t.TokenSourceOutputs) > 0 { - panic("TokenSourceOutputs is not empty") - } -} diff --git a/oura/client/client.go b/oura/client/client.go index 4475c6eeb4..9588ffbe51 100644 --- a/oura/client/client.go +++ b/oura/client/client.go @@ -182,7 +182,7 @@ func (c *Client) GetData(ctx context.Context, dataType string, timeRange *times. mutators := []request.RequestMutator{request.NewParametersMutator(parameters)} // Possible response status codes (see below for details): 200 (DataResponse), 400, 401, 403, 422, 429 - url := c.client.ConstructURL("v2", "usercollection", DataTypeToPath(dataType)) + url := c.client.ConstructURL("v2", "usercollection", oura.DataTypeToPath(dataType)) dataResponse := &oura.DataResponse{} if err := c.sendOAuthRequest(ctx, http.MethodGet, url, mutators, nil, dataResponse, tokenSource); err != nil { return nil, errors.Wrap(err, "unable to get data") @@ -203,7 +203,7 @@ func (c *Client) GetDatum(ctx context.Context, dataType string, dataID string, t } // Possible response status codes (see below for details): 200 (DataResponse), 400, 401, 403, 422, 429 - url := c.client.ConstructURL("v2", "usercollection", DataTypeToPath(dataType), dataID) + url := c.client.ConstructURL("v2", "usercollection", oura.DataTypeToPath(dataType), dataID) dataResponse := oura.Datum{} if err := c.sendOAuthRequest(ctx, http.MethodGet, url, nil, nil, &dataResponse, tokenSource); err != nil { return nil, errors.Wrap(err, "unable to get datum") @@ -229,7 +229,7 @@ func (c *Client) RevokeOAuthToken(ctx context.Context, oauthToken *auth.OAuthTok func (c *Client) sendOAuthRequest(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody any, responseBody any, tokenSource oauth.TokenSource) error { return log.WarnIfDurationExceedsMaximum(ctx, requestDurationMaximum, url, func(ctx context.Context) error { - return c.client.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, []request.ResponseInspector{prometheusCodePathResponseInspector}, tokenSource) + return c.client.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) }) } @@ -243,34 +243,11 @@ func (c *Client) sendClientRequest(ctx context.Context, method string, url strin func (c *Client) sendBaseRequest(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody any, responseBody any, inspectors []request.ResponseInspector) error { return log.WarnIfDurationExceedsMaximum(ctx, requestDurationMaximum, url, func(ctx context.Context) error { - return c.client.Client().RequestDataWithHTTPClient(ctx, method, url, mutators, requestBody, responseBody, append(inspectors, prometheusCodePathResponseInspector), http.DefaultClient) + return c.client.Client().RequestDataWithHTTPClient(ctx, method, url, mutators, requestBody, responseBody, inspectors, http.DefaultClient) }) } -func DataTypeToPath(dataType string) string { - switch dataType { - case oura.DataTypeVO2Max: - return "vO2_max" // Capitalization inconsistency - default: - return dataType - } -} - -func PrometheusCodePathPatterns() []string { - var patterns []string - for _, eventDataType := range oura.EventDataTypes() { - patterns = append(patterns, fmt.Sprintf("/v2/usercollection/%s/{document_id}", DataTypeToPath(eventDataType))) - } - return append(patterns, - "/v2/webhook/subscription/{id}", - "/v2/webhook/subscription/renew/{id}", - request.PatternAny, - ) -} - -const requestDurationMaximum = 30 * time.Second - -var prometheusCodePathResponseInspector = request.NewPrometheusCodePathResponseInspectorWithPatterns("tidepool_oura_api_client_requests", "Oura API client requests", PrometheusCodePathPatterns()...) +const requestDurationMaximum = 60 * time.Second // Possible response status codes from Oura API: // 200: successful get/list; response body contains the requested resource(s) diff --git a/oura/client/client_test.go b/oura/client/client_test.go index b38181bb50..ea2520a995 100644 --- a/oura/client/client_test.go +++ b/oura/client/client_test.go @@ -61,7 +61,7 @@ var _ = Describe("client", func() { mockTokenSourceSource = oauthTest.NewMockTokenSourceSource(mockController) mockProvider = ouraClientTest.NewMockProvider(mockController) server = NewServer() - baseClient, err = oauthClient.NewWithErrorParser(&client.Config{Address: server.URL()}, mockTokenSourceSource, &ouraClient.ErrorResponseParser{}) + baseClient, err = oauthClient.NewWithErrorParser(&client.Config{Address: server.URL()}, nil, mockTokenSourceSource, &ouraClient.ErrorResponseParser{}) Expect(err).ToNot(HaveOccurred()) Expect(baseClient).ToNot(BeNil()) }) @@ -526,7 +526,7 @@ var _ = Describe("client", func() { mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(false, nil) server.AppendHandlers( CombineHandlers( - VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s", ouraClient.DataTypeToPath(dataType)), expectedQuery.Encode()), + VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s", oura.DataTypeToPath(dataType)), expectedQuery.Encode()), VerifyHeader(http.Header{}), VerifyBody(nil), RespondWith(http.StatusInternalServerError, nil), @@ -544,7 +544,7 @@ var _ = Describe("client", func() { mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(false, nil) server.AppendHandlers( CombineHandlers( - VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s", ouraClient.DataTypeToPath(dataType)), expectedQuery.Encode()), + VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s", oura.DataTypeToPath(dataType)), expectedQuery.Encode()), VerifyBody(nil), RespondWithJSONEncoded(http.StatusOK, expectedData), ), @@ -614,7 +614,7 @@ var _ = Describe("client", func() { mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(false, nil) server.AppendHandlers( CombineHandlers( - VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s/%s", ouraClient.DataTypeToPath(dataType), dataID)), + VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s/%s", oura.DataTypeToPath(dataType), dataID)), VerifyHeader(http.Header{}), VerifyBody(nil), RespondWith(http.StatusInternalServerError, nil), @@ -632,7 +632,7 @@ var _ = Describe("client", func() { mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(false, nil) server.AppendHandlers( CombineHandlers( - VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s/%s", ouraClient.DataTypeToPath(dataType), dataID)), + VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s/%s", oura.DataTypeToPath(dataType), dataID)), VerifyBody(nil), RespondWithJSONEncoded(http.StatusOK, expectedDatum), ), @@ -699,58 +699,4 @@ var _ = Describe("client", func() { }) }) }) - - Context("DataTypeToPath", func() { - It("returns expected path for valid data type", func() { - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyActivity)).To(Equal(oura.DataTypeDailyActivity)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyCardiovascularAge)).To(Equal(oura.DataTypeDailyCardiovascularAge)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyCyclePhases)).To(Equal(oura.DataTypeDailyCyclePhases)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyReadiness)).To(Equal(oura.DataTypeDailyReadiness)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyResilience)).To(Equal(oura.DataTypeDailyResilience)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailySleep)).To(Equal(oura.DataTypeDailySleep)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailySpO2)).To(Equal(oura.DataTypeDailySpO2)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyStress)).To(Equal(oura.DataTypeDailyStress)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeEnhancedTag)).To(Equal(oura.DataTypeEnhancedTag)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeHeartRate)).To(Equal(oura.DataTypeHeartRate)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeRestModePeriod)).To(Equal(oura.DataTypeRestModePeriod)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeRingBatteryLevel)).To(Equal(oura.DataTypeRingBatteryLevel)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeRingConfiguration)).To(Equal(oura.DataTypeRingConfiguration)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeSession)).To(Equal(oura.DataTypeSession)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeSleep)).To(Equal(oura.DataTypeSleep)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeSleepTime)).To(Equal(oura.DataTypeSleepTime)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeVO2Max)).To(Equal("vO2_max")) - Expect(ouraClient.DataTypeToPath(oura.DataTypeWorkout)).To(Equal(oura.DataTypeWorkout)) - }) - - It("returns data type for unknown data type", func() { - dataType := test.RandomString() - Expect(ouraClient.DataTypeToPath(dataType)).To(Equal(dataType)) - }) - }) - - Context("PrometheusCodePathPatterns", func() { - It("returns expected patterns", func() { - Expect(ouraClient.PrometheusCodePathPatterns()).To(Equal([]string{ - "/v2/usercollection/daily_activity/{document_id}", - "/v2/usercollection/daily_cardiovascular_age/{document_id}", - "/v2/usercollection/daily_cycle_phases/{document_id}", - "/v2/usercollection/daily_readiness/{document_id}", - "/v2/usercollection/daily_resilience/{document_id}", - "/v2/usercollection/daily_sleep/{document_id}", - "/v2/usercollection/daily_spo2/{document_id}", - "/v2/usercollection/daily_stress/{document_id}", - "/v2/usercollection/enhanced_tag/{document_id}", - "/v2/usercollection/rest_mode_period/{document_id}", - "/v2/usercollection/ring_configuration/{document_id}", - "/v2/usercollection/session/{document_id}", - "/v2/usercollection/sleep/{document_id}", - "/v2/usercollection/sleep_time/{document_id}", - "/v2/usercollection/vO2_max/{document_id}", - "/v2/usercollection/workout/{document_id}", - "/v2/webhook/subscription/{id}", - "/v2/webhook/subscription/renew/{id}", - "/", - })) - }) - }) }) diff --git a/oura/oura.go b/oura/oura.go index 2840ef7422..37314e5942 100644 --- a/oura/oura.go +++ b/oura/oura.go @@ -218,6 +218,15 @@ func DataTypeInScope(dataType string, scope string) bool { return slices.Contains(ScopesForDataType(dataType), scope) } +func DataTypeToPath(dataType string) string { + switch dataType { + case DataTypeVO2Max: + return "vO2_max" // Capitalization inconsistency + default: + return dataType + } +} + type BaseClient interface { ClientID() string ClientSecret() string diff --git a/oura/oura_test.go b/oura/oura_test.go index d029a4ce4f..505bc1d5e1 100644 --- a/oura/oura_test.go +++ b/oura/oura_test.go @@ -375,6 +375,34 @@ var _ = Describe("oura", func() { }) }) + Context("DataTypeToPath", func() { + It("returns expected path for valid data type", func() { + Expect(oura.DataTypeToPath(oura.DataTypeDailyActivity)).To(Equal(oura.DataTypeDailyActivity)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyCardiovascularAge)).To(Equal(oura.DataTypeDailyCardiovascularAge)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyCyclePhases)).To(Equal(oura.DataTypeDailyCyclePhases)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyReadiness)).To(Equal(oura.DataTypeDailyReadiness)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyResilience)).To(Equal(oura.DataTypeDailyResilience)) + Expect(oura.DataTypeToPath(oura.DataTypeDailySleep)).To(Equal(oura.DataTypeDailySleep)) + Expect(oura.DataTypeToPath(oura.DataTypeDailySpO2)).To(Equal(oura.DataTypeDailySpO2)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyStress)).To(Equal(oura.DataTypeDailyStress)) + Expect(oura.DataTypeToPath(oura.DataTypeEnhancedTag)).To(Equal(oura.DataTypeEnhancedTag)) + Expect(oura.DataTypeToPath(oura.DataTypeHeartRate)).To(Equal(oura.DataTypeHeartRate)) + Expect(oura.DataTypeToPath(oura.DataTypeRestModePeriod)).To(Equal(oura.DataTypeRestModePeriod)) + Expect(oura.DataTypeToPath(oura.DataTypeRingBatteryLevel)).To(Equal(oura.DataTypeRingBatteryLevel)) + Expect(oura.DataTypeToPath(oura.DataTypeRingConfiguration)).To(Equal(oura.DataTypeRingConfiguration)) + Expect(oura.DataTypeToPath(oura.DataTypeSession)).To(Equal(oura.DataTypeSession)) + Expect(oura.DataTypeToPath(oura.DataTypeSleep)).To(Equal(oura.DataTypeSleep)) + Expect(oura.DataTypeToPath(oura.DataTypeSleepTime)).To(Equal(oura.DataTypeSleepTime)) + Expect(oura.DataTypeToPath(oura.DataTypeVO2Max)).To(Equal("vO2_max")) + Expect(oura.DataTypeToPath(oura.DataTypeWorkout)).To(Equal(oura.DataTypeWorkout)) + }) + + It("returns data type for unknown data type", func() { + dataType := test.RandomString() + Expect(oura.DataTypeToPath(dataType)).To(Equal(dataType)) + }) + }) + Context("CreateSubscription", func() { DescribeTable("serializes the datum as expected", func(mutator func(datum *oura.CreateSubscription)) { diff --git a/oura/provider/provider.go b/oura/provider/provider.go index 561c60a3f8..b04f99ff48 100644 --- a/oura/provider/provider.go +++ b/oura/provider/provider.go @@ -2,10 +2,14 @@ package provider import ( "context" + "fmt" + "net/http" "slices" + "time" "github.com/tidepool-org/platform/auth" authProviderSession "github.com/tidepool-org/platform/auth/providersession" + "github.com/tidepool-org/platform/client" dataSource "github.com/tidepool-org/platform/data/source" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" @@ -63,7 +67,18 @@ func New(dependencies Dependencies) (*Provider, error) { return nil, errors.Wrap(err, "dependencies is invalid") } - oauthProviderClient, err := oauthProviderClient.NewWithErrorParser(oura.ProviderName, dependencies.Config.Config, nil, &ouraClient.ErrorResponseParser{}) + if dependencies.Config.ClientConfig.Timeout == 0 { + dependencies.Config.ClientConfig.Timeout = 1 * time.Minute + } + + httpClient := &http.Client{ + Transport: prometheusRequestMetricsRoundTripper, + CheckRedirect: http.DefaultClient.CheckRedirect, + Jar: http.DefaultClient.Jar, + Timeout: http.DefaultClient.Timeout, + } + + oauthProviderClient, err := oauthProviderClient.NewWithErrorParser(oura.ProviderName, dependencies.Config.Config, httpClient, nil, &ouraClient.ErrorResponseParser{}) if err != nil { return nil, err } @@ -298,3 +313,17 @@ func (p *Provider) createUserRevokeWork(ctx context.Context, providerSession *au log.LoggerFromContext(ctx).Debug("created user revoke work") return nil } + +func PrometheusPathPatterns() []string { + var pathPatterns []string + for _, eventDataType := range oura.EventDataTypes() { + pathPatterns = append(pathPatterns, fmt.Sprintf("/v2/usercollection/%s/{document_id}", oura.DataTypeToPath(eventDataType))) + } + return append(pathPatterns, + "/v2/webhook/subscription/{id}", + "/v2/webhook/subscription/renew/{id}", + client.PathPatternAny, + ) +} + +var prometheusRequestMetricsRoundTripper = client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets("tidepool_oura_api", "Tidepool Oura API", PrometheusPathPatterns(), nil) diff --git a/oura/provider/provider_test.go b/oura/provider/provider_test.go index 677c4499a3..6c3f86ddfc 100644 --- a/oura/provider/provider_test.go +++ b/oura/provider/provider_test.go @@ -257,4 +257,30 @@ var _ = Describe("provider", func() { }) }) }) + + Context("PrometheusCodePathPatterns", func() { + It("returns expected patterns", func() { + Expect(ouraProvider.PrometheusPathPatterns()).To(Equal([]string{ + "/v2/usercollection/daily_activity/{document_id}", + "/v2/usercollection/daily_cardiovascular_age/{document_id}", + "/v2/usercollection/daily_cycle_phases/{document_id}", + "/v2/usercollection/daily_readiness/{document_id}", + "/v2/usercollection/daily_resilience/{document_id}", + "/v2/usercollection/daily_sleep/{document_id}", + "/v2/usercollection/daily_spo2/{document_id}", + "/v2/usercollection/daily_stress/{document_id}", + "/v2/usercollection/enhanced_tag/{document_id}", + "/v2/usercollection/rest_mode_period/{document_id}", + "/v2/usercollection/ring_configuration/{document_id}", + "/v2/usercollection/session/{document_id}", + "/v2/usercollection/sleep/{document_id}", + "/v2/usercollection/sleep_time/{document_id}", + "/v2/usercollection/vO2_max/{document_id}", + "/v2/usercollection/workout/{document_id}", + "/v2/webhook/subscription/{id}", + "/v2/webhook/subscription/renew/{id}", + "/", + })) + }) + }) }) diff --git a/plugin/abbott/go.mod b/plugin/abbott/go.mod index 988e47d6c3..a90d315439 100644 --- a/plugin/abbott/go.mod +++ b/plugin/abbott/go.mod @@ -20,6 +20,7 @@ require ( github.com/golang/mock v1.6.0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gowebpki/jcs v1.0.1 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/klauspost/compress v1.18.5 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect diff --git a/plugin/abbott/go.sum b/plugin/abbott/go.sum index 688b299209..a15e308884 100644 --- a/plugin/abbott/go.sum +++ b/plugin/abbott/go.sum @@ -37,6 +37,8 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU= +github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= @@ -80,6 +82,7 @@ github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKk github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= diff --git a/pointer/default.go b/pointer/default.go index edbbec0037..92acddbdc6 100644 --- a/pointer/default.go +++ b/pointer/default.go @@ -9,6 +9,13 @@ func Default[T any, S *T](value S, defaultValue T) T { return *value } +func DefaultArray[T any](value []T, defaultValue []T) []T { + if value == nil { + return defaultValue + } + return value +} + func DefaultPointer[T any](value *T, defaultValue *T) *T { if value == nil { return defaultValue diff --git a/pointer/default_test.go b/pointer/default_test.go index 376d4603ee..4fdf874f8c 100644 --- a/pointer/default_test.go +++ b/pointer/default_test.go @@ -25,8 +25,23 @@ var _ = Describe("Default", func() { }) }) + Context("DefaultArray", func() { + It("returns the default value if the value is nil", func() { + defaultValue := test.RandomStringArray() + result := pointer.DefaultArray(nil, defaultValue) + Expect(result).To(Equal(defaultValue)) + }) + + It("returns the value if it is not nil", func() { + value := test.RandomStringArray() + defaultValue := test.RandomStringArray() + result := pointer.DefaultArray(value, defaultValue) + Expect(result).To(Equal(value)) + }) + }) + Context("DefaultPointer", func() { - It("returns a pointer to the the default value if the pointer to the value is nil", func() { + It("returns a pointer to the default value if the pointer to the value is nil", func() { defaultValue := test.RandomString() result := pointer.DefaultPointer(nil, &defaultValue) Expect(result).To(PointTo(Equal(defaultValue))) diff --git a/private/plugin/abbott b/private/plugin/abbott index 806e28a7da..d25d5a2e9b 160000 --- a/private/plugin/abbott +++ b/private/plugin/abbott @@ -1 +1 @@ -Subproject commit 806e28a7dae5dd2878f99c332d59094741690b8b +Subproject commit d25d5a2e9bc5f18eb84001fde1e98737fe4068c8 diff --git a/prometheus/test/prometheus.go b/prometheus/test/prometheus.go new file mode 100644 index 0000000000..15355d651c --- /dev/null +++ b/prometheus/test/prometheus.go @@ -0,0 +1,34 @@ +package test + +import ( + "github.com/prometheus/client_golang/prometheus" + prometheusModel "github.com/prometheus/client_model/go" + + "github.com/tidepool-org/platform/test" +) + +func RandomMetricName() string { + return test.RandomStringFromRangeAndCharset(8, 16, test.CharsetLowercase) +} + +func RandomMetricHelp() string { + return test.RandomString() +} + +func MetricFamilyFromName(name string) *prometheusModel.MetricFamily { + metricFamilies := test.Must(prometheus.DefaultGatherer.Gather()) + for _, metricFamily := range metricFamilies { + if metricFamily.GetName() == name { + return metricFamily + } + } + return nil +} + +func LabelPairsToMap(labelPairs []*prometheusModel.LabelPair) map[string]string { + labels := map[string]string{} + for _, labelPair := range labelPairs { + labels[labelPair.GetName()] = labelPair.GetValue() + } + return labels +} diff --git a/request/condition.go b/request/condition.go index fbdcd08e10..ea4454b52b 100644 --- a/request/condition.go +++ b/request/condition.go @@ -15,6 +15,12 @@ func NewCondition() *Condition { return &Condition{} } +func NewConditionWithRevision(revision *int) *Condition { + return &Condition{ + Revision: revision, + } +} + func (c *Condition) Parse(parser structure.ObjectParser) { c.Revision = parser.Int("revision") } diff --git a/request/inspector.go b/request/inspector.go index c0332ddbf0..d3bd40ebe2 100644 --- a/request/inspector.go +++ b/request/inspector.go @@ -2,10 +2,6 @@ package request import ( "net/http" - "strconv" - - "github.com/prometheus/client_golang/prometheus" - prometheusPromauto "github.com/prometheus/client_golang/prometheus/promauto" ) type ResponseInspector interface { @@ -31,53 +27,3 @@ func NewHeadersInspector() *HeadersInspector { func (h *HeadersInspector) InspectResponse(res *http.Response) { h.Headers = res.Header } - -type PrometheusCodePathResponseInspector struct { - *prometheus.CounterVec - patternMux *http.ServeMux -} - -// When there are only a few discrete paths possible, then no need to simplify via patterns. -// -// For example: /one, /two -func NewPrometheusCodePathResponseInspector(name string, help string) *PrometheusCodePathResponseInspector { - return NewPrometheusCodePathResponseInspectorWithPatterns(name, help) -} - -// Where there are numerous discrete paths possible, then must simplify via patterns to prevent overwhelming Prometheus. -// If no patterns are specified, then all paths are recorded as-is. If one or more patterns are specified, but a path -// does not match one of the patterns, then the path is NOT captured by Prometheus. If you wish to match "all other" paths -// and records those paths as-is, then add the pattern PatternAny at the end of your patterns. -// -// Uses standard Go HTTP pattern matching. See https://go.dev/src/net/http/pattern.go. -// -// For example: /one/{id}, /two/{id} -func NewPrometheusCodePathResponseInspectorWithPatterns(name string, help string, patterns ...string) *PrometheusCodePathResponseInspector { - var patternMux *http.ServeMux - - if len(patterns) > 0 { - patternMux = http.NewServeMux() - for _, pattern := range patterns { - patternMux.HandleFunc(pattern, func(http.ResponseWriter, *http.Request) {}) - } - } - - return &PrometheusCodePathResponseInspector{ - CounterVec: prometheusPromauto.NewCounterVec(prometheus.CounterOpts{Name: name, Help: help}, []string{"code", "path"}), - patternMux: patternMux, - } -} - -func (p *PrometheusCodePathResponseInspector) InspectResponse(res *http.Response) { - path := res.Request.URL.Path - if p.patternMux != nil { - if _, pattern := p.patternMux.Handler(res.Request); pattern == "" { - return - } else if pattern != PatternAny { - path = pattern - } - } - p.With(prometheus.Labels{"code": strconv.Itoa(res.StatusCode), "path": path}).Inc() -} - -const PatternAny = "/" diff --git a/service/server/config.go b/service/server/config.go index f050e76a97..2adb673e80 100644 --- a/service/server/config.go +++ b/service/server/config.go @@ -6,6 +6,7 @@ import ( "time" "github.com/tidepool-org/platform/config" + "github.com/tidepool-org/platform/duration" "github.com/tidepool-org/platform/errors" ) @@ -40,13 +41,10 @@ func (c *Config) Load(configReporter config.Reporter) error { } c.TLSCertificateFile = configReporter.GetWithDefault("tls_certificate_file", "") c.TLSKeyFile = configReporter.GetWithDefault("tls_key_file", "") - if timeoutString, err := configReporter.Get("timeout"); err == nil { - var timeout int64 - timeout, err = strconv.ParseInt(timeoutString, 10, 0) - if err != nil { - return errors.New("timeout is invalid") - } - c.Timeout = time.Duration(timeout) * time.Second + if timeout, err := duration.Parse(configReporter.GetWithDefault("timeout", c.Timeout.String()), time.Second); err != nil { + return errors.New("timeout is invalid") + } else { + c.Timeout = timeout } return nil diff --git a/services/tools/dexcom_analyze/dexcom_analyze.go b/services/tools/dexcom_analyze/dexcom_analyze.go index 27b2f4c153..f8c06d002a 100644 --- a/services/tools/dexcom_analyze/dexcom_analyze.go +++ b/services/tools/dexcom_analyze/dexcom_analyze.go @@ -10,7 +10,6 @@ import ( "sort" "strconv" "strings" - "time" "github.com/urfave/cli" "golang.org/x/exp/maps" @@ -93,15 +92,9 @@ const ( Issue_Task_With_DeviceHashes_And_DataSource_LastImportTime_Missing = "task with device hashes and data source last import time missing" Issue_Task_With_DeviceHashes_And_DataSource_LatestDataTime_Missing = "task with device hashes and data source latest data time missing" Issue_Task_With_State_Failed_AvailableTime_Present = "task with state failed available time present" - Issue_Task_With_State_Failed_DeadlineTime_Present = "task with state failed deadline time present" Issue_Task_With_State_Failed_Error_Missing = "task with state failed error missing" - Issue_Task_With_State_Failed_ExpirationTime_Present = "task with state failed expiration time present" - Issue_Task_With_State_Pending_DeadlineTime_Present = "task with state pending deadline time present" - Issue_Task_With_State_Pending_ExpirationTime_Present = "task with state pending expiration time present" Issue_Task_With_State_Running_AvailableTime_Present = "task with state running available time present" - Issue_Task_With_State_Running_DeadlineTime_Missing = "task with state running deadline time missing" Issue_Task_With_State_Running_Error_Present = "task with state running error present" - Issue_Task_With_State_Running_ExpirationTime_Present = "task with state running expiration time present" IssueFormat_DataSource_Invalid = "data source invalid ('%s', '%s')" IssueFormat_DataSource_ProviderSession_Mismatch = "data source provider session mismatch ('%s', '%s')" @@ -160,15 +153,9 @@ func Issues() []string { Issue_Task_With_DeviceHashes_And_DataSource_LastImportTime_Missing, Issue_Task_With_DeviceHashes_And_DataSource_LatestDataTime_Missing, Issue_Task_With_State_Failed_AvailableTime_Present, - Issue_Task_With_State_Failed_DeadlineTime_Present, Issue_Task_With_State_Failed_Error_Missing, - Issue_Task_With_State_Failed_ExpirationTime_Present, - Issue_Task_With_State_Pending_DeadlineTime_Present, - Issue_Task_With_State_Pending_ExpirationTime_Present, Issue_Task_With_State_Running_AvailableTime_Present, - Issue_Task_With_State_Running_DeadlineTime_Missing, Issue_Task_With_State_Running_Error_Present, - Issue_Task_With_State_Running_ExpirationTime_Present, } } @@ -1105,38 +1092,20 @@ func (t *Tool) analyzeTasks() { switch record.State { case task.TaskStatePending: - if record.DeadlineTime != nil { - record.AppendIssue(Issue_Task_With_State_Pending_DeadlineTime_Present) - } - if record.ExpirationTime != nil { - record.AppendIssue(Issue_Task_With_State_Pending_ExpirationTime_Present) - } case task.TaskStateRunning: if record.AvailableTime != nil { record.AppendIssue(Issue_Task_With_State_Running_AvailableTime_Present) } - if record.DeadlineTime == nil { - record.AppendIssue(Issue_Task_With_State_Running_DeadlineTime_Missing) - } if record.Error != nil { record.AppendIssue(Issue_Task_With_State_Running_Error_Present) } - if record.ExpirationTime != nil { - record.AppendIssue(Issue_Task_With_State_Running_ExpirationTime_Present) - } case task.TaskStateFailed: if record.AvailableTime != nil { record.AppendIssue(Issue_Task_With_State_Failed_AvailableTime_Present) } - if record.DeadlineTime != nil { - record.AppendIssue(Issue_Task_With_State_Failed_DeadlineTime_Present) - } if record.Error == nil { record.AppendIssue(Issue_Task_With_State_Failed_Error_Missing) } - if record.ExpirationTime != nil { - record.AppendIssue(Issue_Task_With_State_Failed_ExpirationTime_Present) - } case task.TaskStateCompleted: record.AppendIssuef(IssueFormat_Task_State_Invalid, record.State) } @@ -1345,37 +1314,18 @@ func (t *Tool) outputIssue(issue string, marshalables Marshalables, issueMarshal t.outputMongoWriteOperationsHeader() t.outputMongoOperationf("db.tasks.updateMany({state: 'failed', availableTime: {$exists: true}}, {$unset: {availableTime: true}})") return - case Issue_Task_With_State_Failed_DeadlineTime_Present: - t.outputResolutionHeader("FIXED with BACK-3116. Will keep occurring until deployed. Will need to manually update failed tasks post-deploy.") - t.outputMongoReadOperationsHeader() - t.outputMongoOperationf("db.tasks.countDocuments({state: 'failed', deadlineTime: {$exists: true}})") - t.outputMongoWriteOperationsHeader() - t.outputMongoOperationf("db.tasks.updateMany({state: 'failed', deadlineTime: {$exists: true}}, {$unset: {deadlineTime: true}})") - return case Issue_Task_With_State_Failed_Error_Missing: - case Issue_Task_With_State_Failed_ExpirationTime_Present: - case Issue_Task_With_State_Pending_DeadlineTime_Present: - case Issue_Task_With_State_Pending_ExpirationTime_Present: case Issue_Task_With_State_Running_AvailableTime_Present: t.outputResolutionHeader("FIXED with BACK-3116. Will keep occurring until deployed. Will need to manually update failed tasks post-deploy.") // NOTE: Uncomment this block to see further details. // t.outputMongoReadOperationsHeader() // t.outputMongoTasksAggregation(marshalables.Tasks().IDs()) return - case Issue_Task_With_State_Running_DeadlineTime_Missing: - t.outputResolutionHeader("Examine each to determine why it is missing.") - t.outputMongoReadOperationsHeader() - t.outputMongoTasksAggregation(marshalables.Tasks().IDs()) - t.outputMongoOperationf("db.tasks.find({id: {$in: [%s]}})", mongoIDs(marshalables.Tasks().IDs())) - t.outputMongoWriteOperationsHeader() - t.outputMongoOperationf("db.tasks.updateMany({id: {$in: [%s]}}, {$set: {deadlineTime: ISODate('%s')}})", mongoIDs(marshalables.Tasks().IDs()), time.Now().Format(time.RFC3339)) - return case Issue_Task_With_State_Running_Error_Present: t.outputResolutionHeader("Examine each to determine why it is present.") t.outputMongoReadOperationsHeader() t.outputMongoTasksAggregation(marshalables.Tasks().IDs()) return - case Issue_Task_With_State_Running_ExpirationTime_Present: default: t.analyzeIssueFormat(issue, marshalables) return diff --git a/store/structured/condition.go b/store/structured/condition.go index cfc7e6cb95..4cb763bd0c 100644 --- a/store/structured/condition.go +++ b/store/structured/condition.go @@ -13,6 +13,12 @@ func NewCondition() *Condition { return &Condition{} } +func NewConditionWithRevision(revision *int) *Condition { + return &Condition{ + Revision: revision, + } +} + func MapCondition(condition *request.Condition) *Condition { if condition == nil { return nil diff --git a/store/structured/mongo/config.go b/store/structured/mongo/config.go index a8ab8ae012..5bb45a1e8b 100644 --- a/store/structured/mongo/config.go +++ b/store/structured/mongo/config.go @@ -85,7 +85,7 @@ func (c *Config) SetDatabaseFromReporter(configReporter platformConfig.Reporter) } // Validate that all parameters are syntactically valid, that all required parameters are present, -// and the the URL constructed from those parameters is parsable by the Mongo driver +// and the URL constructed from those parameters is parsable by the Mongo driver func (c *Config) Validate() error { if len(c.Addresses) == 0 { return errors.New("addresses is missing") diff --git a/store/structured/mongo/result.go b/store/structured/mongo/result.go index a41ea64408..adab4efc36 100644 --- a/store/structured/mongo/result.go +++ b/store/structured/mongo/result.go @@ -1,8 +1,13 @@ package mongo import ( + "context" + "time" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/page" ) @@ -91,3 +96,24 @@ func BSONToAny(input any) any { return output } } + +//nolint:contextcheck // Use a background context if the provided context is nil +func CloseCursor(ctx context.Context, cursor *mongo.Cursor) { + if cursor == nil { + return + } + + if ctx == nil { + ctx = context.Background() + } + + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + + if err := cursor.Close(ctx); err != nil { + if lgr := log.LoggerFromContext(ctx); lgr != nil { + lgr.WithError(err).Warn("Unable to close cursor") + } + } +} diff --git a/store/structured/mongo/result_test.go b/store/structured/mongo/result_test.go index ce9183e57d..a7e88c4a74 100644 --- a/store/structured/mongo/result_test.go +++ b/store/structured/mongo/result_test.go @@ -1,32 +1,39 @@ package mongo_test import ( + "context" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" - "github.com/tidepool-org/platform/store/structured/mongo" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" + storeStructuredMongoTest "github.com/tidepool-org/platform/store/structured/mongo/test" ) var _ = Describe("Result", func() { Describe("BSONToMap", func() { It("returns nil for nil input", func() { - Expect(mongo.BSONToMap(nil)).To(BeNil()) + Expect(storeStructuredMongo.BSONToMap(nil)).To(BeNil()) }) It("handles empty bson.M", func() { - Expect(mongo.BSONToMap(bson.M{})).To(Equal(map[string]any{})) + Expect(storeStructuredMongo.BSONToMap(bson.M{})).To(Equal(map[string]any{})) }) It("converts bson.M to map[string]any", func() { - result := mongo.BSONToMap(bson.M{"string": "value", "number": 42}) + result := storeStructuredMongo.BSONToMap(bson.M{"string": "value", "number": 42}) Expect(result).To(Equal(map[string]any{"string": "value", "number": 42})) }) It("handles deeply nested structures", func() { - result := mongo.BSONToMap(bson.M{"zero": bson.A{"nested", "array"}, "one": bson.M{"nested": "object"}}) + result := storeStructuredMongo.BSONToMap(bson.M{"zero": bson.A{"nested", "array"}, "one": bson.M{"nested": "object"}}) Expect(result).To(MatchAllKeys(Keys{ "zero": Equal([]any{"nested", "array"}), "one": Equal(map[string]any{"nested": "object"}), @@ -36,20 +43,20 @@ var _ = Describe("Result", func() { Describe("BSONToArray", func() { It("returns nil for nil input", func() { - Expect(mongo.BSONToArray(nil)).To(BeNil()) + Expect(storeStructuredMongo.BSONToArray(nil)).To(BeNil()) }) It("handles empty bson.A", func() { - Expect(mongo.BSONToArray(bson.A{})).To(Equal([]any{})) + Expect(storeStructuredMongo.BSONToArray(bson.A{})).To(Equal([]any{})) }) It("converts bson.A to []any", func() { - result := mongo.BSONToArray(bson.A{"value", 42}) + result := storeStructuredMongo.BSONToArray(bson.A{"value", 42}) Expect(result).To(Equal([]any{"value", 42})) }) It("handles deeply nested structures", func() { - result := mongo.BSONToArray(bson.A{bson.A{"nested", "array"}, bson.M{"nested": "object"}}) + result := storeStructuredMongo.BSONToArray(bson.A{bson.A{"nested", "array"}, bson.M{"nested": "object"}}) Expect(result).To(MatchAllElementsWithIndex(IndexIdentity, Elements{ "0": Equal([]any{"nested", "array"}), "1": Equal(map[string]any{"nested": "object"}), @@ -59,20 +66,20 @@ var _ = Describe("Result", func() { Describe("BSONToAny", func() { It("returns nil for nil input", func() { - Expect(mongo.BSONToAny(nil)).To(BeNil()) + Expect(storeStructuredMongo.BSONToAny(nil)).To(BeNil()) }) It("returns primitive types unchanged", func() { - Expect(mongo.BSONToAny(true)).To(Equal(true)) - Expect(mongo.BSONToAny(42)).To(Equal(42)) - Expect(mongo.BSONToAny(42.345)).To(Equal(42.345)) - Expect(mongo.BSONToAny("string")).To(Equal("string")) - Expect(mongo.BSONToAny(map[string]string{"string": "value", "number": "42"})).To(Equal(map[string]string{"string": "value", "number": "42"})) - Expect(mongo.BSONToAny([]string{"value", "42"})).To(Equal([]string{"value", "42"})) + Expect(storeStructuredMongo.BSONToAny(true)).To(Equal(true)) + Expect(storeStructuredMongo.BSONToAny(42)).To(Equal(42)) + Expect(storeStructuredMongo.BSONToAny(42.345)).To(Equal(42.345)) + Expect(storeStructuredMongo.BSONToAny("string")).To(Equal("string")) + Expect(storeStructuredMongo.BSONToAny(map[string]string{"string": "value", "number": "42"})).To(Equal(map[string]string{"string": "value", "number": "42"})) + Expect(storeStructuredMongo.BSONToAny([]string{"value", "42"})).To(Equal([]string{"value", "42"})) }) It("converts bson.M", func() { - result := mongo.BSONToAny(bson.M{"zero": bson.A{"nested", "array"}, "one": bson.M{"nested": "object"}}) + result := storeStructuredMongo.BSONToAny(bson.M{"zero": bson.A{"nested", "array"}, "one": bson.M{"nested": "object"}}) Expect(result).To(MatchAllKeys(Keys{ "zero": Equal([]any{"nested", "array"}), "one": Equal(map[string]any{"nested": "object"}), @@ -80,11 +87,52 @@ var _ = Describe("Result", func() { }) It("converts bson.A", func() { - result := mongo.BSONToAny(bson.A{bson.A{"nested", "array"}, bson.M{"nested": "object"}}) + result := storeStructuredMongo.BSONToAny(bson.A{bson.A{"nested", "array"}, bson.M{"nested": "object"}}) Expect(result).To(MatchAllElementsWithIndex(IndexIdentity, Elements{ "0": Equal([]any{"nested", "array"}), "1": Equal(map[string]any{"nested": "object"}), })) }) }) + + Describe("CloseCursor", func() { + var lgr *logTest.Logger + var ctx context.Context + var repository *storeStructuredMongo.Repository + + BeforeEach(func() { + lgr = logTest.NewLogger() + ctx = log.NewContextWithLogger(context.Background(), lgr) + repository = storeStructuredMongoTest.GetSuiteStore().GetRepository(storeStructuredMongoTest.NewCollectionPrefix()) + _, err := repository.InsertMany(ctx, []any{bson.M{"value": 1}, bson.M{"value": 2}}) + Expect(err).ToNot(HaveOccurred()) + }) + + openCursor := func() *mongo.Cursor { + cursor, err := repository.Find(ctx, bson.M{}, options.Find().SetBatchSize(1)) + Expect(err).ToNot(HaveOccurred()) + Expect(cursor.ID()).ToNot(BeZero()) + return cursor + } + + It("does not panic when the cursor is nil", func() { + Expect(func() { storeStructuredMongo.CloseCursor(ctx, nil) }).ToNot(Panic()) + }) + + It("does not panic when both the context and the cursor are nil", func() { + Expect(func() { storeStructuredMongo.CloseCursor(context.Context(nil), nil) }).ToNot(Panic()) + }) + + It("closes the cursor when the context is provided", func() { + cursor := openCursor() + storeStructuredMongo.CloseCursor(ctx, cursor) + Expect(cursor.ID()).To(BeZero()) + }) + + It("closes the cursor when the context is nil", func() { + cursor := openCursor() + storeStructuredMongo.CloseCursor(context.Context(nil), cursor) + Expect(cursor.ID()).To(BeZero()) + }) + }) }) diff --git a/store/structured/mongo/test/config.go b/store/structured/mongo/test/config.go index 4981ba9798..810294f34a 100644 --- a/store/structured/mongo/test/config.go +++ b/store/structured/mongo/test/config.go @@ -12,6 +12,6 @@ func NewConfig() *storeStructuredMongo.Config { Addresses: []string{Address()}, Database: Database(), CollectionPrefix: NewCollectionPrefix(), - Timeout: 5 * time.Second, + Timeout: 15 * time.Second, } } diff --git a/summary/task/migrationrunner.go b/summary/task/migrationrunner.go index ea2a137394..8ff2ca122c 100644 --- a/summary/task/migrationrunner.go +++ b/summary/task/migrationrunner.go @@ -33,10 +33,8 @@ type MigrationRunner struct { func NewDefaultMigrationTaskCreate(summaryType string) *task.TaskCreate { typ := MigrationType + "." + summaryType return &task.TaskCreate{ - Name: pointer.FromAny(typ), - Type: typ, - Priority: 5, - AvailableTime: pointer.FromAny(time.Now().UTC()), + Name: pointer.FromAny(typ), + Type: typ, Data: map[string]any{ ConfigMinInterval: int32(DefaultMigrationAvailableAfterDurationMinimum.Seconds()), ConfigMaxInterval: int32(DefaultMigrationAvailableAfterDurationMaximum.Seconds()), @@ -71,8 +69,8 @@ func (r *MigrationRunner) GetRunnerType() string { return MigrationType + "." + r.summaryType } -func (r *MigrationRunner) GetRunnerDeadline() time.Time { - return time.Now().Add(MigrationTaskDurationMaximum * 3) +func (r *MigrationRunner) GetRunnerDeadline() time.Duration { + return MigrationTaskDurationMaximum * 3 } func (r *MigrationRunner) GetRunnerTimeout() time.Duration { @@ -171,17 +169,17 @@ func (t *MigrationTaskRunner) run() error { pagination.Size = t.GetBatch() typ := t.summaryType - t.logger.Infof("Searching for User %s Summaries requiring Migration", typ) + t.logger.Debugf("Searching for User %s Summaries requiring Migration", typ) outdatedUserIds, err := t.dataClient.GetMigratableUserIDs(t.context, typ, pagination) if err != nil { return err } if len(outdatedUserIds) == 0 { - t.logger.Infof("No %s Summaries requiring migrations found", typ) + t.logger.Debugf("No %s Summaries requiring migrations found", typ) return nil } - t.logger.Infof("Found batch of %d %s Summaries to Migrate", len(outdatedUserIds), typ) + t.logger.Debugf("Found batch of %d %s Summaries to Migrate", len(outdatedUserIds), typ) t.logger.Debugf("Starting User %s Summary Migration", typ) err = updateSummaries(t.context, t.logger, t.dataClient, typ, outdatedUserIds, MigrationWorkerCount, t.deadline, "Migrating") diff --git a/summary/task/migrationrunner_test.go b/summary/task/migrationrunner_test.go index 2904ecc18b..14e88850a9 100644 --- a/summary/task/migrationrunner_test.go +++ b/summary/task/migrationrunner_test.go @@ -57,8 +57,6 @@ var _ = Describe("migrate runner tasks", func() { Expect(t.Name).ToNot(BeNil()) Expect(*t.Name).To(Equal("org.tidepool.summary.migrate.cgm")) Expect(t.Type).To(Equal("org.tidepool.summary.migrate.cgm")) - Expect(t.AvailableTime).ToNot(BeNil()) - Expect(t.AvailableTime.IsZero()).ToNot(BeTrue()) batch, ok := t.Data[ConfigBatch].(int32) Expect(ok).To(BeTrue()) diff --git a/summary/task/updaterunner.go b/summary/task/updaterunner.go index 899208d153..3ceaecdc76 100644 --- a/summary/task/updaterunner.go +++ b/summary/task/updaterunner.go @@ -34,10 +34,8 @@ type UpdateRunner struct { func NewDefaultUpdateTaskCreate(summaryType string) *task.TaskCreate { typ := UpdateType + "." + summaryType return &task.TaskCreate{ - Name: pointer.FromAny(typ), - Type: typ, - Priority: 5, - AvailableTime: pointer.FromAny(time.Now().UTC()), + Name: pointer.FromAny(typ), + Type: typ, Data: map[string]any{ ConfigMinInterval: int32(DefaultUpdateAvailableAfterDurationMinimum.Seconds()), ConfigMaxInterval: int32(DefaultUpdateAvailableAfterDurationMaximum.Seconds()), @@ -69,8 +67,8 @@ func (r *UpdateRunner) GetRunnerType() string { return UpdateType + "." + r.summaryType } -func (r *UpdateRunner) GetRunnerDeadline() time.Time { - return time.Now().Add(UpdateTaskDurationMaximum * 3) +func (r *UpdateRunner) GetRunnerDeadline() time.Duration { + return UpdateTaskDurationMaximum * 3 } func (r *UpdateRunner) GetRunnerTimeout() time.Duration { @@ -173,17 +171,17 @@ func (t *UpdateTaskRunner) run() error { t.logger.Debugf("Starting User %s Summary Update", typ) for i := 1; i <= IterLimit; i++ { - t.logger.Infof("Searching for User %s Summaries requiring Update", typ) + t.logger.Debugf("Searching for User %s Summaries requiring Update", typ) outdated, err := t.dataClient.GetOutdatedUserIDs(t.context, typ, pagination) if err != nil { return err } if len(outdated.UserIds) == 0 { - t.logger.Infof("No %s Summaries requiring updates found", typ) + t.logger.Debugf("No %s Summaries requiring updates found", typ) return nil } - t.logger.Infof("Found batch of %d %s Summaries to Update", len(outdated.UserIds), typ) + t.logger.Debugf("Found batch of %d %s Summaries to Update", len(outdated.UserIds), typ) err = updateSummaries(t.context, t.logger, t.dataClient, typ, outdated.UserIds, UpdateWorkerCount, t.deadline, "Updating") if err != nil { diff --git a/summary/task/updaterunner_test.go b/summary/task/updaterunner_test.go index 5ea1833d88..64dddb1535 100644 --- a/summary/task/updaterunner_test.go +++ b/summary/task/updaterunner_test.go @@ -57,8 +57,6 @@ var _ = Describe("update runner tasks", func() { Expect(t.Name).ToNot(BeNil()) Expect(*t.Name).To(Equal("org.tidepool.summary.update.cgm")) Expect(t.Type).To(Equal("org.tidepool.summary.update.cgm")) - Expect(t.AvailableTime).ToNot(BeNil()) - Expect(t.AvailableTime.IsZero()).ToNot(BeTrue()) batch, ok := t.Data[ConfigBatch].(int32) Expect(ok).To(BeTrue()) diff --git a/task/client/client.go b/task/client/client.go index 7cc6dd4e2c..5ffc67c78f 100644 --- a/task/client/client.go +++ b/task/client/client.go @@ -59,7 +59,7 @@ func (c *Client) CreateTask(ctx context.Context, create *task.TaskCreate) (*task if create == nil { return nil, errors.New("create is missing") } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(create); err != nil { - return nil, errors.New("create is invalid") + return nil, errors.Wrap(err, "create is invalid") } url := c.client.ConstructURL("v1", "tasks") @@ -71,17 +71,22 @@ func (c *Client) CreateTask(ctx context.Context, create *task.TaskCreate) (*task return tsk, nil } -func (c *Client) GetTask(ctx context.Context, id string) (*task.Task, error) { +func (c *Client) GetTask(ctx context.Context, id string, condition *request.Condition) (*task.Task, error) { if ctx == nil { return nil, errors.New("context is missing") } if id == "" { return nil, errors.New("id is missing") } + if condition == nil { + condition = request.NewCondition() + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return nil, errors.Wrap(err, "condition is invalid") + } url := c.client.ConstructURL("v1", "tasks", id) tsk := &task.Task{} - if err := c.client.RequestData(ctx, http.MethodGet, url, nil, nil, tsk); err != nil { + if err := c.client.RequestData(ctx, http.MethodGet, url, []request.RequestMutator{condition}, nil, tsk); err != nil { if request.IsErrorResourceNotFound(err) { return nil, nil } @@ -91,13 +96,18 @@ func (c *Client) GetTask(ctx context.Context, id string) (*task.Task, error) { return tsk, nil } -func (c *Client) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { +func (c *Client) UpdateTask(ctx context.Context, id string, condition *request.Condition, update *task.TaskUpdate) (*task.Task, error) { if ctx == nil { return nil, errors.New("context is missing") } if id == "" { return nil, errors.New("id is missing") } + if condition == nil { + condition = request.NewCondition() + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return nil, errors.Wrap(err, "condition is invalid") + } if update == nil { return nil, errors.New("update is missing") } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(update); err != nil { @@ -106,7 +116,7 @@ func (c *Client) UpdateTask(ctx context.Context, id string, update *task.TaskUpd url := c.client.ConstructURL("v1", "tasks", id) tsk := &task.Task{} - if err := c.client.RequestData(ctx, http.MethodPut, url, nil, update, tsk); err != nil { + if err := c.client.RequestData(ctx, http.MethodPut, url, []request.RequestMutator{condition}, update, tsk); err != nil { if request.IsErrorResourceNotFound(err) { return nil, nil } @@ -116,14 +126,19 @@ func (c *Client) UpdateTask(ctx context.Context, id string, update *task.TaskUpd return tsk, nil } -func (c *Client) DeleteTask(ctx context.Context, id string) error { +func (c *Client) DeleteTask(ctx context.Context, id string, condition *request.Condition) error { if ctx == nil { return errors.New("context is missing") } if id == "" { return errors.New("id is missing") } + if condition == nil { + condition = request.NewCondition() + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return errors.Wrap(err, "condition is invalid") + } url := c.client.ConstructURL("v1", "tasks", id) - return c.client.RequestData(ctx, http.MethodDelete, url, nil, nil, nil) + return c.client.RequestData(ctx, http.MethodDelete, url, []request.RequestMutator{condition}, nil, nil) } diff --git a/task/queue/multi.go b/task/queue/multi.go index 47d2cd294d..273f719b3e 100644 --- a/task/queue/multi.go +++ b/task/queue/multi.go @@ -1,63 +1,73 @@ package queue import ( + "maps" + "sync" + "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/task/store" ) -// MultiQueue creates a queue per registered runner +// MultiQueue runs a queue per runner provided at construction, each processing only tasks of that runner's type. Like +// Queue, it is single-use. The queues map is immutable after NewMultiQueue, so no synchronization is required. type MultiQueue struct { - queues map[string]Queue - cfg *Config - lgr log.Logger - str store.Store + queues map[string]*Queue } -func NewMultiQueue(cfg *Config, lgr log.Logger, str store.Store) (Queue, error) { - return &MultiQueue{ - queues: make(map[string]Queue), - cfg: cfg, - lgr: lgr, - str: str, - }, nil -} - -func (m *MultiQueue) RegisterRunner(runner Runner) error { - typ := runner.GetRunnerType() - if _, ok := m.queues[typ]; ok { - return errors.New("runner of the same type is already registered") +func NewMultiQueue(cfg *Config, lgr log.Logger, str store.Store, runners ...Runner) (*MultiQueue, error) { + if cfg == nil { + return nil, errors.New("config is missing") } - - str := m.str.WithTypeFilter(typ) - q, err := New(m.cfg, m.lgr, str) - if err != nil { - return err + if lgr == nil { + return nil, errors.New("logger is missing") + } + if str == nil { + return nil, errors.New("store is missing") } - if err := q.RegisterRunner(runner); err != nil { - return err + + queues := make(map[string]*Queue, len(runners)) + for _, runner := range runners { + if runner == nil { + return nil, errors.New("runner is missing") + } + + typ := runner.GetRunnerType() + if _, ok := queues[typ]; ok { + return nil, errors.New("runner type already registered") + } + + q, err := New(typ, cfg, lgr, str.WithTypeFilter(typ), runner) + if err != nil { + return nil, err + } + + queues[typ] = q } - m.queues[typ] = q - return nil + return &MultiQueue{ + queues: queues, + }, nil } func (m *MultiQueue) Start() { for _, q := range m.queues { - q := q q.Start() } } func (m *MultiQueue) Stop() { + // Stop the queues concurrently so total shutdown latency is bounded by a single queue's stop timeout rather than + // the sum across every queue. + var waitGroup sync.WaitGroup for _, q := range m.queues { - q := q - q.Stop() + waitGroup.Go(q.Stop) } + waitGroup.Wait() } -func (m *MultiQueue) GetQueues() map[string]Queue { - return m.queues +// GetQueues returns a copy of the type-to-queue map so callers cannot mutate the internal map. The queue values are +// shared references, not copies. +func (m *MultiQueue) GetQueues() map[string]*Queue { + return maps.Clone(m.queues) } - -var _ Queue = &MultiQueue{} diff --git a/task/queue/multi_test.go b/task/queue/multi_test.go index 62f9c6cc77..c3831e068b 100644 --- a/task/queue/multi_test.go +++ b/task/queue/multi_test.go @@ -8,17 +8,18 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.mongodb.org/mongo-driver/bson" "github.com/tidepool-org/platform/log" - "github.com/tidepool-org/platform/log/null" + logNull "github.com/tidepool-org/platform/log/null" "github.com/tidepool-org/platform/page" storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" storeStructuredMongoTest "github.com/tidepool-org/platform/store/structured/mongo/test" "github.com/tidepool-org/platform/task" - "github.com/tidepool-org/platform/task/queue" - "github.com/tidepool-org/platform/task/queue/test" - "github.com/tidepool-org/platform/task/store/mongo" + taskQueue "github.com/tidepool-org/platform/task/queue" + taskQueueTest "github.com/tidepool-org/platform/task/queue/test" + taskStoreMongo "github.com/tidepool-org/platform/task/store/mongo" ) var ( @@ -28,48 +29,86 @@ var ( var _ = Describe("multi queue", func() { var config *storeStructuredMongo.Config + var queueConfig *taskQueue.Config var lgr log.Logger - var str *mongo.Store - var multi *queue.MultiQueue + var str *taskStoreMongo.Store + var multi *taskQueue.MultiQueue BeforeEach(func() { config = storeStructuredMongoTest.NewConfig() var err error - str, err = mongo.NewStore(config) + str, err = taskStoreMongo.NewStore(config) Expect(err).ToNot(HaveOccurred()) Expect(str).ToNot(BeNil()) - lgr = null.NewLogger() - - var q queue.Queue - q, err = queue.NewMultiQueue( - &queue.Config{ - Workers: 10, - Delay: 1, - }, - lgr, - str, - ) - Expect(err).ToNot(HaveOccurred()) - Expect(q).ToNot(BeNil()) - - var ok bool - multi, ok = q.(*queue.MultiQueue) - Expect(ok).To(BeTrue()) + lgr = logNull.NewLogger() + queueConfig = taskQueue.NewConfig() + queueConfig.Workers = 10 + queueConfig.StartManagerDelay = time.Millisecond + queueConfig.DispatchTasksDelay = time.Millisecond + multi = nil }) AfterEach(func() { + if multi != nil { + multi.Stop() + } Expect(str.Terminate(context.Background())).To(Succeed()) }) - Describe("Register Runner", func() { - It("Creates a new queue for each runner type", func() { - for _, t := range types { - runner := test.NewCountingRunner(t) - Expect(multi.RegisterRunner(runner)).To(Succeed()) - Expect(multi.GetQueues()).To(HaveKey(t)) + Describe("NewMultiQueue", func() { + It("creates a new queue for each runner type", func() { + runners := make([]taskQueue.Runner, 0, len(types)) + for _, typ := range types { + runners = append(runners, taskQueueTest.NewCountingRunner(typ)) } + + var err error + multi, err = taskQueue.NewMultiQueue(queueConfig, lgr, str, runners...) + Expect(err).ToNot(HaveOccurred()) + Expect(multi).ToNot(BeNil()) + queues := multi.GetQueues() Expect(queues).To(HaveLen(len(types))) + for _, typ := range types { + Expect(queues).To(HaveKey(typ)) + } + }) + + It("returns an error when the config is missing", func() { + invalidMulti, err := taskQueue.NewMultiQueue(nil, lgr, str) + Expect(err).To(MatchError("config is missing")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when the logger is missing", func() { + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, nil, str) + Expect(err).To(MatchError("logger is missing")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when the store is missing", func() { + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, lgr, nil) + Expect(err).To(MatchError("store is missing")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when a runner is missing", func() { + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, lgr, str, nil) + Expect(err).To(MatchError("runner is missing")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when two runners have the same type", func() { + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, lgr, str, taskQueueTest.NewCountingRunner(types[0]), taskQueueTest.NewCountingRunner(types[0])) + Expect(err).To(MatchError("runner type already registered")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when a runner has invalid durations", func() { + runner := taskQueueTest.NewSleepRunner(types[0], 2*time.Minute, 2*time.Minute, time.Minute, 0) + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, lgr, str, runner) + Expect(err).To(MatchError("runner deadline is invalid")) + Expect(invalidMulti).To(BeNil()) }) }) @@ -82,20 +121,21 @@ var _ = Describe("multi queue", func() { It("Are partitioned correctly", func() { ctx := log.NewContextWithLogger(context.Background(), lgr) creates := make([]*task.TaskCreate, 0, len(types)*tasksPerType) - runners := make([]*test.CountingRunner, 0, len(types)) + countingRunners := make([]*taskQueueTest.CountingRunner, 0, len(types)) + runners := make([]taskQueue.Runner, 0, len(types)) now := time.Now() // Create tasks and runners for each task type - for _, t := range types { - runner := test.NewCountingRunner(t) + for _, typ := range types { + runner := taskQueueTest.NewCountingRunner(typ) + countingRunners = append(countingRunners, runner) runners = append(runners, runner) - Expect(multi.RegisterRunner(runner)).To(Succeed()) - for i := 0; i < tasksPerType; i++ { - name := fmt.Sprintf("%v:%v", t, i) + for index := 0; index < tasksPerType; index++ { + name := fmt.Sprintf("%v:%v", typ, index) creates = append(creates, &task.TaskCreate{ Name: &name, - Type: t, + Type: typ, AvailableTime: &now, }) } @@ -104,34 +144,23 @@ var _ = Describe("multi queue", func() { // Insert tasks in the database rand.Shuffle(len(creates), func(i, j int) { creates[i], creates[j] = creates[j], creates[i] }) for _, create := range creates { - create := create tsk, err := str.NewTaskRepository().CreateTask(ctx, create) Expect(err).ToNot(HaveOccurred()) Expect(tsk).ToNot(BeNil()) } - // Register runners from all types in the underlying queue - // To make sure they are empty when all work is processed - expectedNoopRunners := make([]*test.CountingRunner, 0) - for typ, q := range multi.GetQueues() { - for _, t := range types { - if typ != t { - runner := test.NewCountingRunner(t) - expectedNoopRunners = append(expectedNoopRunners, runner) - Expect(q.RegisterRunner(runner)).To(Succeed()) - } - } - - } + var err error + multi, err = taskQueue.NewMultiQueue(queueConfig, lgr, str, runners...) + Expect(err).ToNot(HaveOccurred()) + Expect(multi).ToNot(BeNil()) multi.Start() nonTerminalStates := []string{task.TaskStatePending, task.TaskStateRunning} - // Wait until completion, within limits. On my local laptop, this typically - // takes < 15 seconds when run via Gingko (no parallel), but under Go test - // (parallel via package) it takes around 35 seconds. Who knows how long it - // would take running in parallel on a CI host. So give it plenty of time. + // Wait until completion, within limits. On my local laptop, this typically takes < 15 seconds when run via + // Gingko (no parallel), but under Go test (parallel via package) it takes around 35 seconds. Who knows how + // long it would take running in parallel on a CI host. So give it plenty of time. tCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() @@ -167,16 +196,11 @@ var _ = Describe("multi queue", func() { expected[typ] = tasksPerType } results := map[string]int{} - for _, runner := range runners { + for _, runner := range countingRunners { results[runner.GetRunnerType()] = runner.GetCount() } Expect(results).To(Equal(expected)) - - for _, runner := range expectedNoopRunners { - // Check extra runners didn't do any work - Expect(runner.GetCount()).To(Equal(0)) - } }) }) }) diff --git a/task/queue/queue.go b/task/queue/queue.go index ebc3a9f80b..a78d3a686f 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -2,31 +2,88 @@ package queue import ( "context" - "math/rand" "runtime/debug" "strconv" "sync" "time" - "go.mongodb.org/mongo-driver/mongo" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/tidepool-org/platform/config" + "github.com/tidepool-org/platform/crypto" + "github.com/tidepool-org/platform/duration" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/pointer" + storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" "github.com/tidepool-org/platform/task" - "github.com/tidepool-org/platform/task/store" + taskStore "github.com/tidepool-org/platform/task/store" +) + +const ( + // WorkersDefault is the default number of workers for the queue. + WorkersDefault = 5 + + // StartManagerDelayDefault is the default upper bound on the randomized delay the manager waits before it begins + // dispatching tasks, spreading startup across instances. + StartManagerDelayDefault = 1 * time.Minute + + // DispatchTasksDelayDefault is the default delay, jittered, between polls for pending tasks to dispatch. Completing + // a task also dispatches immediately, without waiting for the poll. + DispatchTasksDelayDefault = 1 * time.Minute + + // MonitorTaskDelayDefault is the default interval between checks that each in-flight task's claim is still held + // (the task exists and its claim token is unchanged); a run whose claim is lost is canceled with task.ErrClaimLost. + MonitorTaskDelayDefault = 1 * time.Minute + + // RunnerWatchdogGracePeriodDefault is the extra time beyond the runner timeout that the watchdog waits before + // reporting a runner as blocked. The runner context is still canceled at the runner timeout; the grace period only + // gives a cooperative runner time to observe that cancellation and return before the watchdog reports it as + // non-cooperative. + RunnerWatchdogGracePeriodDefault = 5 * time.Second + + // UnstickTasksDelayDefault is the default delay, jittered, between attempts to unstick tasks. The first attempt is + // made after a randomized delay of at most this duration. + UnstickTasksDelayDefault = 5 * time.Minute + + // UnstickTasksAvailableGracePeriodDefault is the default grace period added to the claim monitor delay and the + // runner watchdog grace period to form the delay after a task is unstuck before it is made available for + // re-dispatch. This ensures that any still-running tasks have likely been canceled and exited before being + // re-dispatched. + UnstickTasksAvailableGracePeriodDefault = 1 * time.Minute + + // StopWaitTimeoutDefault bounds how long Stop waits for in-flight tasks to observe cancellation and exit before + // abandoning them (they are recovered by the deadline and unstick mechanism). Kept under the typical Kubernetes + // termination grace period since we could use it up to twice (once for workers and once for manager) and we still + // want to leave time for the store to flush any pending writes. + StopWaitTimeoutDefault = 10 * time.Second + + TaskDeadlineDefault = 1 * time.Minute + DurationJitterFactor = 0.2 ) type Config struct { - Workers int - Delay time.Duration + Workers int + StartManagerDelay time.Duration + DispatchTasksDelay time.Duration + MonitorTaskDelay time.Duration + RunnerWatchdogGracePeriod time.Duration + UnstickTasksDelay time.Duration + UnstickTasksAvailableGracePeriod time.Duration + StopWaitTimeout time.Duration } func NewConfig() *Config { return &Config{ - Workers: 1, - Delay: 60 * time.Second, + Workers: WorkersDefault, + StartManagerDelay: StartManagerDelayDefault, + DispatchTasksDelay: DispatchTasksDelayDefault, + MonitorTaskDelay: MonitorTaskDelayDefault, + RunnerWatchdogGracePeriod: RunnerWatchdogGracePeriodDefault, + UnstickTasksDelay: UnstickTasksDelayDefault, + UnstickTasksAvailableGracePeriod: UnstickTasksAvailableGracePeriodDefault, + StopWaitTimeout: StopWaitTimeoutDefault, } } @@ -35,21 +92,47 @@ func (c *Config) Load(configReporter config.Reporter) error { return errors.New("config reporter is missing") } - if workersString, err := configReporter.Get("workers"); err == nil { - var workers int64 - workers, err = strconv.ParseInt(workersString, 10, 0) - if err != nil { + if valueString, err := configReporter.Get("workers"); err == nil { + if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { return errors.New("workers is invalid") + } else { + c.Workers = int(value) } - c.Workers = int(workers) } - if delayString, err := configReporter.Get("delay"); err == nil { - var delay int64 - delay, err = strconv.ParseInt(delayString, 10, 0) - if err != nil { - return errors.New("delay is invalid") - } - c.Delay = time.Duration(delay) * time.Second + if value, err := duration.Parse(configReporter.GetWithDefault("start_manager_delay", c.StartManagerDelay.String()), time.Second); err != nil { + return errors.New("start manager delay is invalid") + } else { + c.StartManagerDelay = value + } + if value, err := duration.Parse(configReporter.GetWithDefault("dispatch_tasks_delay", c.DispatchTasksDelay.String()), time.Second); err != nil { + return errors.New("dispatch tasks delay is invalid") + } else { + c.DispatchTasksDelay = value + } + if value, err := duration.Parse(configReporter.GetWithDefault("monitor_task_delay", c.MonitorTaskDelay.String()), time.Second); err != nil { + return errors.New("monitor task delay is invalid") + } else { + c.MonitorTaskDelay = value + } + if value, err := duration.Parse(configReporter.GetWithDefault("runner_watchdog_grace_period", c.RunnerWatchdogGracePeriod.String()), time.Second); err != nil { + return errors.New("runner watchdog grace period is invalid") + } else { + c.RunnerWatchdogGracePeriod = value + } + if value, err := duration.Parse(configReporter.GetWithDefault("unstick_tasks_delay", c.UnstickTasksDelay.String()), time.Second); err != nil { + return errors.New("unstick tasks delay is invalid") + } else { + c.UnstickTasksDelay = value + } + if value, err := duration.Parse(configReporter.GetWithDefault("unstick_tasks_available_grace_period", c.UnstickTasksAvailableGracePeriod.String()), time.Second); err != nil { + return errors.New("unstick tasks available grace period is invalid") + } else { + c.UnstickTasksAvailableGracePeriod = value + } + if value, err := duration.Parse(configReporter.GetWithDefault("stop_wait_timeout", c.StopWaitTimeout.String()), time.Second); err != nil { + return errors.New("stop wait timeout is invalid") + } else { + c.StopWaitTimeout = value } return nil @@ -59,62 +142,55 @@ func (c *Config) Validate() error { if c.Workers < 1 { return errors.New("workers is invalid") } - if c.Delay < 0 { - return errors.New("delay is invalid") + if c.StartManagerDelay <= 0 { + return errors.New("start manager delay is invalid") + } + if c.DispatchTasksDelay <= 0 { + return errors.New("dispatch tasks delay is invalid") + } + if c.MonitorTaskDelay <= 0 { + return errors.New("monitor task delay is invalid") + } + if c.RunnerWatchdogGracePeriod <= 0 { + return errors.New("runner watchdog grace period is invalid") + } + if c.UnstickTasksDelay <= 0 { + return errors.New("unstick tasks delay is invalid") + } + if c.UnstickTasksAvailableGracePeriod <= 0 { + return errors.New("unstick tasks available grace period is invalid") + } + if c.StopWaitTimeout <= 0 { + return errors.New("stop wait timeout is invalid") } - return nil } -type Runner interface { - - // The type of tasks that the runner supports. - GetRunnerType() string - - // The time after which the task manager will forcefully reset the task back to pending - // and available. This is calculated based upon the current time and a duration significantly - // longer that the task duration maximum. Normally this would only be used on a task that - // is in the running state even though it is not running (likely due to a system crash or interruption). - GetRunnerDeadline() time.Time - - // The duration of a task where the task manager will forcefully cancel the task context to interrupt - // the task and force completion. This is typically a duration somewhat longer than the task - // duration maximum. - GetRunnerTimeout() time.Duration - - // The typical duration maximum of the task after which a warning will be displayed. - GetRunnerDurationMaximum() time.Duration - - // Execute the specified task within the specified context. The context will be forcefully - // canceled after a duration specified by GetRunnerTimeout. - Run(ctx context.Context, tsk *task.Task) -} - -type Queue interface { - RegisterRunner(Runner) error - Start() - Stop() -} - -type queue struct { +// The Queue's fields are all immutable after New, except the lifecycle fields, which are guarded by the lifecycle +// mutex, and workersAvailable, which is initialized by Start before the manager exists and is thereafter owned +// exclusively by the manager goroutine. The workers and manager therefore read the channels and runners map freely, +// without synchronization. +type Queue struct { + name string + config *Config logger log.Logger - store store.Store - workers int - delay time.Duration + repository taskStore.TaskRepository runners map[string]Runner - workersCancelFunc context.CancelFunc + dispatchChannel chan *task.Task + completionChannel chan *task.Task + lifecycleMutex sync.Mutex + started bool + stopped bool + cancelFunc context.CancelFunc workersWaitGroup sync.WaitGroup - managerCancelFunc context.CancelFunc managerWaitGroup sync.WaitGroup workersAvailable int - dispatchChannel chan *task.Task - completionChannel chan *task.Task - timer *time.Timer - taskRepository store.TaskRepository - iterator *mongo.Cursor } -func New(cfg *Config, lgr log.Logger, str store.Store) (Queue, error) { +func New(name string, cfg *Config, lgr log.Logger, str taskStore.Store, runners ...Runner) (*Queue, error) { + if name == "" { + return nil, errors.New("name is missing") + } if cfg == nil { return nil, errors.New("config is missing") } @@ -129,311 +205,612 @@ func New(cfg *Config, lgr log.Logger, str store.Store) (Queue, error) { return nil, errors.Wrap(err, "config is invalid") } - workers := cfg.Workers - delay := cfg.Delay + runnerMap := make(map[string]Runner, len(runners)) + for _, runner := range runners { + if runner == nil { + return nil, errors.New("runner is missing") + } + if _, ok := runnerMap[runner.GetRunnerType()]; ok { + return nil, errors.New("runner type already registered") + } + if err := validateRunner(runner); err != nil { + return nil, err + } + runnerMap[runner.GetRunnerType()] = runner + } + + return &Queue{ + name: name, + config: cfg, + logger: lgr.WithField("queue", name), + repository: str.NewTaskRepository(), + runners: runnerMap, - return &queue{ - logger: lgr, - store: str, - workers: workers, - delay: delay, - runners: make(map[string]Runner), - dispatchChannel: make(chan *task.Task, workers), - completionChannel: make(chan *task.Task, workers), + // NOT buffered so a task is only handed off when a worker is ready to receive it. This ensures a dispatched + // task is never stranded in a buffer during shutdown. + dispatchChannel: make(chan *task.Task), + + // Buffered so that a worker can complete a task and hand it off to the manager even if the manager is busy + // dispatching other tasks. + completionChannel: make(chan *task.Task, cfg.Workers), }, nil } -func (q *queue) RegisterRunner(runner Runner) error { - if runner == nil { - return errors.New("runner is missing") +func (q *Queue) Start() { + q.lifecycleMutex.Lock() + defer q.lifecycleMutex.Unlock() + + if q.started || q.stopped { + return } + q.started = true - q.runners[runner.GetRunnerType()] = runner - return nil + q.logger.Debug("Task queue starting") + + ctx, cancelFunc := context.WithCancel(log.NewContextWithLogger(context.Background(), q.logger)) + q.cancelFunc = cancelFunc + + q.startWorkers(ctx) + q.startManager(ctx) + + q.logger.Debug("Task queue started") } -func (q *queue) Start() { - backgroundCtx := log.NewContextWithLogger(context.Background(), q.logger) - if q.workersCancelFunc == nil { - ctx, workersCancelFunc := context.WithCancel(backgroundCtx) - q.workersCancelFunc = workersCancelFunc +func (q *Queue) Stop() { + // Hold the mutex for the entire stop, including the waits, so a concurrent Start cannot observe a partially stopped + // queue. + q.lifecycleMutex.Lock() + defer q.lifecycleMutex.Unlock() - q.startWorkers(ctx) + if q.stopped { + return } - if q.managerCancelFunc == nil { - ctx, managerCancelFunc := context.WithCancel(backgroundCtx) - q.managerCancelFunc = managerCancelFunc + q.stopped = true - q.startManager(ctx) + // Never started, so nothing to stop; the stopped flag ensures it never will be. + if !q.started { + return } -} -func (q *queue) Stop() { - if q.workersCancelFunc != nil { - q.workersCancelFunc() - q.workersCancelFunc = nil + q.logger.Info("Task queue stopping") + + lgr := q.logger.WithField("stopWaitTimeout", q.config.StopWaitTimeout) + + // Cancel the manager, so it stops dispatching new tasks and begins draining completions from the workers, and the + // workers, to interrupt any in-flight task. + q.cancelFunc() + + // Wait for all workers to exit, but only up to a bounded timeout so a runner that does not honor cancellation + // cannot block shutdown forever. If a worker is still running we must NOT close the channels: a stuck worker that + // later finishes would panic sending on a closed completion channel. Instead we leave the goroutines orphaned + // (reaped at process exit); the abandoned task stays running and is recovered by the deadline/unstick mechanism. + if !waitWithTimeout(&q.workersWaitGroup, q.config.StopWaitTimeout) { + lgr.Error("Task queue workers did not stop within timeout; abandoning in-flight tasks; will be fixed with UnstickTasks later") + return } - q.workersWaitGroup.Wait() + // All workers have exited, so completion channel can be closed. close(q.completionChannel) - if q.managerCancelFunc != nil { - q.managerCancelFunc() - q.managerCancelFunc = nil + // Wait for manager to exit. This should be prompt now that the completion channel is closed, but bound it too in + // case a completion write is slow. + if !waitWithTimeout(&q.managerWaitGroup, q.config.StopWaitTimeout) { + lgr.Error("Task queue manager did not stop within timeout") + return } - q.managerWaitGroup.Wait() + // Manager has exited, so no further tasks will be dispatched. Because the dispatch channel is not buffered, no + // dispatched task can be stranded in it; any task the manager could not hand off was reverted to pending during + // dispatch. close(q.dispatchChannel) + + q.logger.Info("Task queue stopped") } -func (q *queue) startWorkers(ctx context.Context) { - for q.workersAvailable = 0; q.workersAvailable < q.workers; q.workersAvailable++ { - q.startWorker(ctx) +func (q *Queue) startWorkers(ctx context.Context) { + for q.workersAvailable = 0; q.workersAvailable < q.config.Workers; q.workersAvailable++ { + q.startWorker(log.ContextWithField(ctx, "worker", q.workersAvailable)) } + WorkersAvailable.WithLabelValues(q.name).Set(float64(q.workersAvailable)) + WorkersTotal.WithLabelValues(q.name).Set(float64(q.config.Workers)) } -func (q *queue) startWorker(ctx context.Context) { - q.workersWaitGroup.Add(1) - go func() { - defer q.workersWaitGroup.Done() +func (q *Queue) startWorker(ctx context.Context) { + q.workersWaitGroup.Go(func() { + lgr := log.LoggerFromContext(ctx) + + lgr.Debug("Task queue worker started") for { - select { - case <-ctx.Done(): + if err := q.executeWorker(ctx); err != nil { + lgr.WithError(err).Debug("Task queue worker stopped") return - case tsk := <-q.dispatchChannel: - q.runTask(ctx, tsk) - q.completionChannel <- tsk } } - }() + }) +} + +func (q *Queue) executeWorker(ctx context.Context) error { + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } + + select { + case <-ctx.Done(): + return context.Cause(ctx) + case tsk := <-q.dispatchChannel: + if tsk != nil { + q.runTask(ctx, tsk) + q.completionChannel <- tsk + } + } + + return nil } -func (q *queue) runTask(ctx context.Context, tsk *task.Task) { - ctx = log.ContextWithField(ctx, "taskId", tsk.ID) +func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { + ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) + + runner, ok := q.runners[tsk.Type] + if !ok { + // A whole task type is unprocessable by this queue; surface it distinctly (a configuration problem) rather than + // letting it blend in with ordinary per-task failures downstream. + lgr.Error("Runner not found for task type; task cannot be processed") + tsk.SetFailedWithError(errors.New("runner not found for task type")) + RunnerNotFoundTotal.WithLabelValues(tsk.Type).Inc() + return + } + + // The claim context is canceled with task.ErrClaimLost by the task claim monitor when the task is deleted or re-claimed + // mid-run. Claim loss is irreversible for this run (every claim gets a fresh token), so once canceled the outcome + // can never be persisted. + claimContext, claimCancel := context.WithCancelCause(ctx) + defer claimCancel(nil) + + // Clearing the claim token marks the outcome as unpersistable, which completeTask discards. Done in a defer, after + // the recover below, so a panicking runner is also reconciled correctly. + defer func() { + if errors.Is(context.Cause(claimContext), task.ErrClaimLost) { + tsk.ClaimToken = nil + } + }() defer func() { if err := recover(); err != nil { - log.LoggerFromContext(ctx).WithFields(log.Fields{"error": err, "stack": string(debug.Stack())}).Error("Unhandled panic") + lgr.WithFields(log.Fields{"error": err, "stack": string(debug.Stack())}).Error("Unhandled panic while running task") tsk.AppendError(errors.New("unhandled panic")) + RunPanicTotal.WithLabelValues(tsk.Type).Inc() } }() - if runner, ok := q.runners[tsk.Type]; ok { + // If runner does not respect its own maximum duration, then enforce a context-based timeout. This forces the task + // to cancel via the context. + runnerContext, cancel := context.WithTimeoutCause(claimContext, runner.GetRunnerTimeout(), ErrRunnerTimeoutExceeded) + defer cancel() + + // Watchdog for a runner that ignores cancellation. Go cannot preempt a goroutine, so if the runner blows past its + // timeout without returning, this worker stays blocked until the process restarts (the task itself is recovered by + // the deadline/unstick mechanism). We cannot unblock the worker, but we surface the condition via a log and metric + // so a non-cooperative runner is detectable rather than silent. The grace period lets a cooperative runner that + // returns promptly after the timeout cancellation avoid being reported as blocked here; such a run is instead + // logged and counted as "recovered" during reconciliation. + runnerWatchdog := time.AfterFunc(runner.GetRunnerTimeout()+q.config.RunnerWatchdogGracePeriod, func() { + lgr.Error("Task runner exceeded timeout without returning; worker is blocked until it returns") + RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType(), "blocked").Inc() + }) + defer runnerWatchdog.Stop() + + // Watch the claim for the duration of the run. The task is owned by the runner while it runs, so the watch gets its + // own copies of everything it needs rather than reading tsk. + go func(id string, typ string, claimToken string) { + if reason := q.monitorTaskForLostClaim(claimContext, id, claimToken); reason != nil { + log.LoggerFromContext(claimContext).Warnf("Task %s; canceling task run", *reason) + RunClaimLostTotal.WithLabelValues(typ, *reason).Inc() + claimCancel(task.ErrClaimLost) + } + }(tsk.ID, tsk.Type, *tsk.ClaimToken) - // If runner does not respect its own maximum duration, then enforce a context-based timeout. - // This forces the task to cancel via the context. - ctx, cancel := context.WithTimeout(ctx, runner.GetRunnerTimeout()) - defer cancel() + // Run the task via the runner + startTime := time.Now() + runner.Run(runnerContext, tsk) + duration := time.Since(startTime).Truncate(time.Millisecond) - startTime := time.Now() + // Immediately stop the runner watchdog + runnerWatchdog.Stop() - // Run the task via the runner - runner.Run(ctx, tsk) + RunDurationSeconds.WithLabelValues(runner.GetRunnerType()).Observe(duration.Seconds()) + if duration > runner.GetRunnerDurationMaximum() { + lgr.WithField("duration", duration.Seconds()).Warn("Task duration exceeds maximum") + } + + // The claim was lost mid-run; any write-back would miss the claim token, so skip state reconciliation. The outcome + // is discarded during completion. + if errors.Is(context.Cause(claimContext), task.ErrClaimLost) { + return + } - if taskDuration := time.Since(startTime); taskDuration > runner.GetRunnerDurationMaximum() { - log.LoggerFromContext(ctx).WithField("taskDuration", taskDuration.Truncate(time.Millisecond).Seconds()).Warn("Task duration exceeds maximum") + // If the runner left the task running, reconcile its state based on why the run ended. A claim lost after the check + // above matches neither branch; the outcome is discarded during completion. Any other run left running is failed + // with a missing terminal state error during completion. + if tsk.State == task.TaskStateRunning { + if context.Cause(ctx) != nil { + // The parent (worker) context was canceled by shutdown; make the task available again for retry rather than + // treating the interruption as a completion. + tsk.RepeatAvailableAfter(0) + } else if cause := context.Cause(runnerContext); errors.Is(cause, ErrRunnerTimeoutExceeded) { + // The runner exceeded its timeout but returned; record the cause so the failure is attributed to the + // timeout rather than the generic missing terminal state error. + lgr.Warn("Task runner exceeded timeout; task will be failed") + tsk.AppendError(cause) + RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType(), "recovered").Inc() } - } else { - tsk.AppendError(errors.New("runner not found for task")) - tsk.SetFailed() } } -func (q *queue) startManager(ctx context.Context) { - q.managerWaitGroup.Add(1) +// monitorTaskForLostClaim polls until the run's claim on the task is lost - the task is gone (deleted) or holds a +// different claim token (unstuck and possibly re-claimed) - and then returns the reason for the claim loss. If the +// context is canceled before the claim is lost, returns nil. +func (q *Queue) monitorTaskForLostClaim(ctx context.Context, id string, runningClaimToken string) *string { + ticker := time.NewTicker(q.config.MonitorTaskDelay) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if storeClaimToken, exists, err := q.repository.GetTaskClaimToken(ctx, id); err != nil { + if context.Cause(ctx) == nil { + log.LoggerFromContext(ctx).WithError(err).Error("Unable to get task claim token") + } + } else if !exists { + return pointer.From("deleted") + } else if storeClaimToken == nil || *storeClaimToken != runningClaimToken { + return pointer.From("reclaimed") + } + } + } +} - go func() { - defer q.managerWaitGroup.Done() +func (q *Queue) startManager(ctx context.Context) { + q.managerWaitGroup.Go(func() { + lgr := log.LoggerFromContext(ctx) - q.startTimer(time.Duration(rand.Int63n(int64(q.delay)) + 1)) - defer q.stopTimer() + lgr.Debug("Task queue manager started") - // pick a starting random time in a future cycle to ensure multiple daemons don't do this exactly at the same - // time, it is not an error condition if it does, but could stress the db if the collection gets large - nextUnstickTime := time.Now().Add(time.Duration(rand.Int63n(int64(q.delay * 15)))) + lgr.Debug("Task queue start manager delay initiated") + + // Start at a random future time to help prevent thundering herd problem + select { + case <-ctx.Done(): + lgr.WithError(context.Cause(ctx)).Debug("Task queue manager stopped before dispatching tasks") + return + case <-time.After(randomDuration(q.config.StartManagerDelay)): + lgr.Debug("Task queue start manager delay complete") + } + + // Start at a random future time to help prevent thundering herd problem + unstickTasksTime := time.Now().Add(randomDuration(q.config.UnstickTasksDelay)) for { - if nextUnstickTime.Before(time.Now()) { - q.unstickTasks(ctx) - nextUnstickTime = time.Now().Add(q.delay * 15) - } + if err := q.executeManager(ctx); err != nil { + lgr.WithError(err).Debug("Task queue manager stopping") - select { - case <-ctx.Done(): // Drain and complete any interrupted tasks + // Complete any remaining tasks concurrently so the total drain time is bounded by a single slow + // completion write rather than the sum across all workers' tasks, keeping shutdown within the stop wait + // timeout. + var completionWaitGroup sync.WaitGroup for tsk := range q.completionChannel { - q.completeTask(ctx, tsk) + if tsk != nil { + completionWaitGroup.Go(func() { q.completeTask(ctx, tsk) }) + } } + completionWaitGroup.Wait() + + lgr.WithError(err).Debug("Task queue manager stopped") return - case tsk := <-q.completionChannel: - if tsk != nil { - q.stopTimer() - q.completeTask(ctx, tsk) - q.startTimer(q.dispatchTasks(ctx)) - } - case <-q.timer.C: - q.startTimer(q.dispatchTasks(ctx)) + } + + if unstickTasksTime.Before(time.Now()) { + q.unstickTasks(ctx) + unstickTasksTime = time.Now().Add(durationWithJitter(q.config.UnstickTasksDelay)) } } - }() + }) } -func (q *queue) unstickTasks(ctx context.Context) { - repository := q.store.NewTaskRepository() - count, err := repository.UnstickTasks(ctx) - if err != nil { - q.logger.WithError(err).Error("Failure in unsticking tasks") +func (q *Queue) executeManager(ctx context.Context) error { + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: } - if count > 0 { - q.logger.WithField("unstickCount", count).Warn("Unstuck tasks") + + select { + case <-ctx.Done(): + return context.Cause(ctx) + case tsk := <-q.completionChannel: + if tsk != nil { + q.completeTask(ctx, tsk) + q.workersAvailable++ + WorkersAvailable.WithLabelValues(q.name).Set(float64(q.workersAvailable)) + q.dispatchTasks(ctx) + } + case <-time.After(durationWithJitter(q.config.DispatchTasksDelay)): + q.dispatchTasks(ctx) } + + return nil } -func (q *queue) dispatchTasks(ctx context.Context) time.Duration { - defer q.stopPendingIterator(ctx) - for q.workersAvailable > 0 { - iter, err := q.startPendingIterator(ctx) - if err != nil { - q.logger.WithError(err).Error("Failure starting pending iterator") - return q.delay - } +func (q *Queue) unstickTasks(ctx context.Context) { + // Delay availability of unstuck tasks to ensure that any still-running tasks have likely been canceled and exited + // before being re-dispatched. + availabilityDelay := q.config.MonitorTaskDelay + q.config.RunnerWatchdogGracePeriod + q.config.UnstickTasksAvailableGracePeriod + ids, err := q.repository.UnstickTasks(ctx, availabilityDelay) + if count := len(ids); count > 0 { + log.LoggerFromContext(ctx).WithFields(log.Fields{"count": count, "ids": ids}).Info("Unstuck tasks") + } - tsk := &task.Task{} - if iter.Next(ctx) { - if err := iter.Decode(tsk); err != nil { - q.logger.WithError(err).Error("Failure iterating tasks") - return q.delay - } - q.dispatchTask(ctx, tsk) - } else { - if err := iter.Err(); err != nil { - q.logger.WithError(err).Error("Failure iterating pending tasks") - } - return q.delay + // Log error unless context was canceled + if err != nil { + if context.Cause(ctx) == nil { + log.LoggerFromContext(ctx).WithError(err).Error("Unable to unstick tasks") } } - - return q.delay } -func (q *queue) dispatchTask(ctx context.Context, tsk *task.Task) { - ctx = log.ContextWithField(ctx, "taskId", tsk.ID) - - repository := q.store.NewTaskRepository() +func (q *Queue) dispatchTasks(ctx context.Context) { + if q.workersAvailable < 1 { + return + } - tsk.State = task.TaskStateRunning - tsk.AvailableTime = nil - tsk.RunTime = pointer.FromAny(time.Now()) + lgr := log.LoggerFromContext(ctx) - // we don't error here if missing, as the task will be failed during runTask - if runner, ok := q.runners[tsk.Type]; ok { - tsk.DeadlineTime = pointer.FromAny(runner.GetRunnerDeadline()) + // Iterate across all pending tasks + cursor, err := q.repository.IteratePending(ctx) + if err != nil { + if context.Cause(ctx) == nil { + lgr.WithError(err).Error("Unable to open task iterator") + } + return } + defer storeStructuredMongo.CloseCursor(ctx, cursor) - var err error - tsk, err = repository.UpdateFromState(context.WithoutCancel(ctx), tsk, task.TaskStatePending) - if err != nil { - if errors.Is(err, task.AlreadyClaimedTask) { - log.LoggerFromContext(ctx).Warnf("Failure to claim task %s (%s) as it is already in progress or is no longer available.", tsk.Name, tsk.ID) + // Loop until no more workers available or no more pending tasks + for q.workersAvailable > 0 && cursor.Next(ctx) { + tsk := &task.Task{} + if err = cursor.Decode(tsk); err != nil { + lgr.WithError(err).Error("Unable to decode task") + } else if err = q.dispatchTask(ctx, tsk); err != nil { + lgr.WithError(err).Error("Unable to dispatch task") return } - - log.LoggerFromContext(ctx).WithError(err).Error("Failure to update state during dispatch task") - return } - q.workersAvailable-- - q.dispatchChannel <- tsk + // Log cursor error unless context was canceled + if err := cursor.Err(); err != nil { + if context.Cause(ctx) == nil { + lgr.WithError(err).Error("Unable to iterate tasks") + } + } } -func (q *queue) completeTask(ctx context.Context, tsk *task.Task) { - ctx = log.ContextWithField(ctx, "taskId", tsk.ID) +func (q *Queue) dispatchTask(ctx context.Context, tsk *task.Task) error { + ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) - q.workersAvailable++ + // we don't error here if missing, as the task will be failed during runTask and the error persisted to the database + // when the task completes + var deadline time.Duration + if runner, ok := q.runners[tsk.Type]; ok { + deadline = runner.GetRunnerDeadline() + } else { + deadline = TaskDeadlineDefault + } + + // StartTask completes regardless of context cancellation, so its outcome is definitive: a non-nil startedTask means + // the claim committed with a known claim token. + startedTask, err := q.repository.StartTask(ctx, tsk.ID, tsk.Revision, deadline) + if err != nil { + return errors.Wrap(err, "unable to start task") + } else if startedTask == nil { + lgr.Info("Task no longer available to start") + return nil + } + + // Hand the task off to a worker. If the queue is shutting down before a worker can receive it, revert the task to + // pending rather than blocking the manager. The revert uses the started task claim token so it reliably matches. + select { + case <-ctx.Done(): + if err := q.repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.ClaimToken, task.TaskStatePending, nil, nil); err != nil { + return errors.Wrap(err, "unable to revert task to pending") + } + case q.dispatchChannel <- startedTask: + q.workersAvailable-- + WorkersAvailable.WithLabelValues(q.name).Set(float64(q.workersAvailable)) + } - repository := q.store.NewTaskRepository() + return nil +} - q.computeState(tsk) +// completeTask persists the task's completion. It deliberately does not touch workersAvailable (the caller accounts for +// the freed worker where relevant), so it is safe to call concurrently during the manager's shutdown drain. +func (q *Queue) completeTask(ctx context.Context, tsk *task.Task) { + ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) - if tsk.State != task.TaskStatePending { - tsk.AvailableTime = nil + // The claim was lost mid-run (task deleted or re-claimed): any write-back would miss the claim token, so discard + // the outcome rather than logging a spurious lost completion. + if tsk.ClaimToken == nil { + lgr.Warn("Task claim lost; task run outcome discarded") + return } - tsk.DeadlineTime = nil + + var duration *time.Duration if tsk.RunTime != nil { - tsk.Duration = pointer.FromFloat64(time.Since(*tsk.RunTime).Truncate(time.Millisecond).Seconds()) + // Clamp to a zero minimum: the run time round-trips through the database without a monotonic reading, so a + // backwards wall clock step could yield a negative elapsed time, which StopTask would reject, losing the + // completion. + duration = pointer.From(max(time.Since(*tsk.RunTime), 0)) } - // Without cancel to ensure task is updated in the database - _, err := repository.UpdateFromState(context.WithoutCancel(ctx), tsk, task.TaskStateRunning) - if err != nil { - log.LoggerFromContext(ctx).WithError(err).Error("Failure to update state during complete task") + q.computeState(ctx, tsk) + + // computeState has already settled the terminal state. A failed task is a genuine problem worth an error; a task + // that errored but will run again (e.g. reverted to pending for retry) is an expected, recoverable outcome, so log + // it at warning to avoid error-level noise from routine retries. + if err := tsk.GetError(); tsk.State == task.TaskStateFailed { + lgr.WithError(err).Error("Task failed while running") + } else if err != nil { + lgr.WithError(err).Warn("Error occurred while running task that did not fail") } - if tsk.HasError() { - log.LoggerFromContext(ctx).WithError(tsk.Error.Error).Error("Error occurred while running task") + // Data and Error use non-nil wrappers so that a task whose data or error was cleared during the run has the + // corresponding field unset in the database, rather than left stale (a nil wrapper means "leave unchanged" to + // StopTask). + update := &task.TaskUpdate{ + Data: pointer.From(tsk.Data), + AvailableTime: tsk.AvailableTime, + Error: &errors.Serializable{Error: tsk.GetError()}, + } + if err := q.repository.StopTask(ctx, tsk.ID, tsk.Revision, tsk.ClaimToken, tsk.State, duration, update); err != nil { + lgr.WithError(err).Error("Unable to complete task") } } -func (q *queue) computeState(tsk *task.Task) { +func (q *Queue) computeState(ctx context.Context, tsk *task.Task) { switch tsk.State { case task.TaskStatePending: - if tsk.AvailableTime == nil || time.Now().After(*tsk.AvailableTime) { - // This used to error out here, but was considered too defensive. - tsk.AvailableTime = pointer.FromAny(time.Now()) + now := time.Now().UTC() + if tsk.AvailableTime == nil { + log.LoggerFromContext(ctx).Warn("Available time missing for pending task") + tsk.AvailableTime = pointer.FromTime(now) + } else if tsk.AvailableTime.Before(now) { + if tsk.AvailableTime.Before(now.Add(-time.Minute)) { // Allow some leeway to prevent spurious warnings + log.LoggerFromContext(ctx).Warn("Available time significantly before now for pending task") + } + tsk.AvailableTime = pointer.FromTime(now) } case task.TaskStateRunning: + // The runner returned without moving the task out of the running state, violating the runner contract; fail the + // task rather than guessing whether it succeeded. if tsk.HasError() { tsk.SetFailed() } else { - tsk.SetCompleted() + tsk.SetFailedWithError(errors.New("runner failed to set state")) } case task.TaskStateFailed, task.TaskStateCompleted: + tsk.AvailableTime = nil default: - tsk.AppendError(errors.New("unknown state")) - tsk.SetFailed() + tsk.SetFailedWithError(errors.New("unknown task state")) } } -func (q *queue) startTimer(delay time.Duration) { - if delay > 0 { - if q.timer == nil { - q.timer = time.NewTimer(delay) - } else { - q.timer.Reset(delay) - } +// validateRunner enforces the runner duration contract, deadline > timeout > duration maximum > 0, so that a +// misconfigured runner fails queue construction rather than causing subtle runtime misbehavior (a non-positive deadline +// prevents the task from ever starting, while a deadline that does not exceed the timeout allows the unstick mechanism +// to reset a task that is still running). +func validateRunner(runner Runner) error { + if durationMaximum := runner.GetRunnerDurationMaximum(); durationMaximum <= 0 { + return errors.New("runner duration maximum is invalid") + } else if timeout := runner.GetRunnerTimeout(); timeout <= durationMaximum { + return errors.New("runner timeout is invalid") + } else if runner.GetRunnerDeadline() <= timeout { + return errors.New("runner deadline is invalid") + } else { + return nil } } -func (q *queue) stopTimer() { - if q.timer != nil { - if !q.timer.Stop() { - <-q.timer.C - } +// waitWithTimeout waits for the wait group to complete, returning true if it completed within the timeout and false +// otherwise. On timeout the internal waiter goroutine is left running until the wait group eventually completes (or the +// process exits). +func waitWithTimeout(waitGroup *sync.WaitGroup, timeout time.Duration) bool { + done := make(chan struct{}) + go func() { + waitGroup.Wait() + close(done) + }() + + select { + case <-done: + return true + case <-time.After(timeout): + return false } } -func (q *queue) startPendingIterator(ctx context.Context) (*mongo.Cursor, error) { - if q.taskRepository == nil { - q.taskRepository = q.store.NewTaskRepository() - } - if q.iterator == nil { - if iterator, err := q.taskRepository.IteratePending(ctx); err != nil { - return nil, err - } else { - q.iterator = iterator - } +func randomDuration(duration time.Duration) time.Duration { + if duration <= 0 { + return 0 } - return q.iterator, nil + return time.Duration(crypto.RandomInt64N(int64(duration))) } -func (q *queue) stopPendingIterator(ctx context.Context) { - if q.iterator != nil { - if err := q.iterator.Close(context.WithoutCancel(ctx)); err != nil { - q.logger.WithError(err).Warn("failure closing pending iterator") - } - q.iterator = nil - } - if q.taskRepository != nil { - q.taskRepository = nil +func durationWithJitter(duration time.Duration) time.Duration { + if duration <= 0 { + return 0 } + jitter := time.Duration(float64(duration) * DurationJitterFactor) + return duration + (randomDuration(jitter*2) - jitter) } + +var ( + // WorkersTotal reports the configured number of workers, per queue. + WorkersTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_task_workers_total", + Help: "The configured number of task queue workers, sorted by queue", + }, []string{"queue"}) + + // WorkersAvailable reports the number of available workers, per queue. A value pinned at zero indicates a saturated + // queue or workers wedged in non-cooperative runners. + WorkersAvailable = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_task_workers_available", + Help: "The number of available task queue workers, sorted by queue", + }, []string{"queue"}) + + // RunnerNotFoundTotal counts task runs for which no runner is registered for the task's type, sorted by type. A + // non-zero value indicates pending tasks of a type this queue cannot process (a configuration problem); such tasks + // are failed immediately. + RunnerNotFoundTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_runner_not_found_total", + Help: "The total number of task runs with no registered runner for the task type, sorted by type", + }, []string{"type"}) + + // RunDurationSeconds observes how long each task run took, sorted by type. Use it to track run latency percentiles + // and to alert when durations approach the runner timeout. + RunDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tidepool_task_run_duration_seconds", + Help: "The duration of task runs, in seconds, sorted by type", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 15), + }, []string{"type"}) + + // RunnerTimeoutExceededTotal counts task runs that exceeded the runner timeout, sorted by type and disposition, + // where disposition can be: + // - "blocked" - for runs the watchdog caught still running past the grace period (the worker is wedged until the + // runner eventually returns or the process restarts, so this is the severe case) + // - "recovered" - for runs that exceeded the timeout, but returned once their context was canceled (the timeout + // mechanism worked as designed) + RunnerTimeoutExceededTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_runner_timeout_exceeded_total", + Help: "The total number of task runs that exceeded the runner timeout, sorted by type and disposition", + }, []string{"type", "disposition"}) + + // RunClaimLostTotal counts task runs canceled because the run's claim was lost, sorted by type and reason, where + // reason can be: + // - "deleted" - the task document was deleted mid-run + // - "reclaimed" - the stored claim token no longer matches (the task was unstuck and possibly re-claimed + // elsewhere) + RunClaimLostTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_run_claim_lost_total", + Help: "The total number of task runs canceled because the task claim was lost, sorted by type and reason", + }, []string{"type", "reason"}) + + // RunPanicTotal counts task runs that panicked and were recovered, sorted by type. The task is then failed, unless + // the runner had already moved it out of running before panicking. + RunPanicTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_run_panic_total", + Help: "The total number of task runs that panicked, sorted by type", + }, []string{"type"}) +) diff --git a/task/queue/queue_internal_test.go b/task/queue/queue_internal_test.go index a6c0767cdd..f8b044d78c 100644 --- a/task/queue/queue_internal_test.go +++ b/task/queue/queue_internal_test.go @@ -3,49 +3,81 @@ package queue import ( "context" "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "github.com/tidepool-org/platform/log" logTest "github.com/tidepool-org/platform/log/test" - "github.com/tidepool-org/platform/task/store" + "github.com/tidepool-org/platform/pointer" + "github.com/tidepool-org/platform/task" + taskStore "github.com/tidepool-org/platform/task/store" + taskTest "github.com/tidepool-org/platform/task/test" + "github.com/tidepool-org/platform/test" ) -type failingIteratorStore struct { - store.Store -} - -func (f *failingIteratorStore) NewTaskRepository() store.TaskRepository { - return &failingIteratorRepository{} -} - -type failingIteratorRepository struct { - store.TaskRepository -} - -func (f *failingIteratorRepository) IteratePending(ctx context.Context) ( - *mongo.Cursor, error) { - - return mongo.NewCursorFromDocuments([]interface{}{bson.D{}}, - fmt.Errorf("batch fetch failed"), nil) -} - var _ = Describe("Queue", func() { Context("dispatchTasks", func() { + It("logs an error when the pending iterator cannot be opened", func() { + lgr := logTest.NewLogger() + ctx := log.NewContextWithLogger(context.Background(), lgr) + cfg := NewConfig() + que := &Queue{ + name: taskTest.RandomType(), + config: cfg, + logger: lgr, + repository: &failureOpeningIteratorRepository{}, + workersAvailable: 1, + } + que.dispatchTasks(ctx) + lgr.AssertError("Unable to open task iterator") + }) + It("logs an error when the pending iterator fails mid-iteration", func() { - logger := logTest.NewLogger() - q := &queue{ - logger: logger, - store: &failingIteratorStore{}, + lgr := logTest.NewLogger() + ctx := log.NewContextWithLogger(context.Background(), lgr) + cfg := NewConfig() + que := &Queue{ + name: taskTest.RandomType(), + config: cfg, + logger: lgr, + repository: &failureIteratingIteratorRepository{}, workersAvailable: 1, - delay: time.Minute, } + que.dispatchTasks(ctx) + lgr.AssertError("Unable to iterate tasks") + }) + }) - Expect(q.dispatchTasks(context.Background())).To(Equal(time.Minute)) - logger.AssertError("Failure iterating pending tasks") + Context("computeState", func() { + It("clears the available time for a completed task", func() { + completedTask := &task.Task{State: task.TaskStateCompleted, AvailableTime: pointer.FromTime(test.RandomTimeBeforeNow())} + (&Queue{}).computeState(context.Background(), completedTask) + Expect(completedTask.AvailableTime).To(BeNil()) + }) + + It("clears the available time for a failed task", func() { + failedTask := &task.Task{State: task.TaskStateFailed, AvailableTime: pointer.FromTime(test.RandomTimeBeforeNow())} + (&Queue{}).computeState(context.Background(), failedTask) + Expect(failedTask.AvailableTime).To(BeNil()) }) }) }) + +type failureOpeningIteratorRepository struct { + taskStore.TaskRepository +} + +func (f *failureOpeningIteratorRepository) IteratePending(ctx context.Context) (*mongo.Cursor, error) { + return nil, fmt.Errorf("failure opening iterator") +} + +type failureIteratingIteratorRepository struct { + taskStore.TaskRepository +} + +func (f *failureIteratingIteratorRepository) IteratePending(ctx context.Context) (*mongo.Cursor, error) { + return mongo.NewCursorFromDocuments([]any{}, fmt.Errorf("failure iterating iterator"), nil) +} diff --git a/task/queue/queue_test.go b/task/queue/queue_test.go index ff18fa0800..2d04f62fc1 100644 --- a/task/queue/queue_test.go +++ b/task/queue/queue_test.go @@ -1,8 +1,1103 @@ package queue_test import ( + "context" + "sync" + "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/prometheus/client_golang/prometheus/testutil" + "go.mongodb.org/mongo-driver/bson" + + configTest "github.com/tidepool-org/platform/config/test" + "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + metadataTest "github.com/tidepool-org/platform/metadata/test" + "github.com/tidepool-org/platform/pointer" + storeStructuredMongoTest "github.com/tidepool-org/platform/store/structured/mongo/test" + "github.com/tidepool-org/platform/task" + taskQueue "github.com/tidepool-org/platform/task/queue" + taskQueueTest "github.com/tidepool-org/platform/task/queue/test" + taskStore "github.com/tidepool-org/platform/task/store" + taskStoreMongo "github.com/tidepool-org/platform/task/store/mongo" + taskStoreTest "github.com/tidepool-org/platform/task/store/test" + taskTest "github.com/tidepool-org/platform/task/test" + "github.com/tidepool-org/platform/test" ) -var _ = Describe("queue", func() { +var _ = Describe("Queue", func() { + It("WorkersDefault is expected", func() { + Expect(taskQueue.WorkersDefault).To(Equal(5)) + }) + + It("StartManagerDelayDefault is expected", func() { + Expect(taskQueue.StartManagerDelayDefault).To(Equal(1 * time.Minute)) + }) + + It("DispatchTasksDelayDefault is expected", func() { + Expect(taskQueue.DispatchTasksDelayDefault).To(Equal(1 * time.Minute)) + }) + + It("MonitorTaskDelayDefault is expected", func() { + Expect(taskQueue.MonitorTaskDelayDefault).To(Equal(1 * time.Minute)) + }) + + It("RunnerWatchdogGracePeriodDefault is expected", func() { + Expect(taskQueue.RunnerWatchdogGracePeriodDefault).To(Equal(5 * time.Second)) + }) + + It("UnstickTasksDelayDefault is expected", func() { + Expect(taskQueue.UnstickTasksDelayDefault).To(Equal(5 * time.Minute)) + }) + + It("UnstickTasksAvailableGracePeriodDefault is expected", func() { + Expect(taskQueue.UnstickTasksAvailableGracePeriodDefault).To(Equal(1 * time.Minute)) + }) + + It("StopWaitTimeoutDefault is expected", func() { + Expect(taskQueue.StopWaitTimeoutDefault).To(Equal(10 * time.Second)) + }) + + It("TaskDeadlineDefault is expected", func() { + Expect(taskQueue.TaskDeadlineDefault).To(Equal(1 * time.Minute)) + }) + + It("DurationJitterFactor is expected", func() { + Expect(taskQueue.DurationJitterFactor).To(Equal(0.2)) + }) + + Context("Config", func() { + Context("NewConfig", func() { + It("returns successfully", func() { + cfg := taskQueue.NewConfig() + Expect(cfg).ToNot(BeNil()) + }) + + It("returns default values", func() { + cfg := taskQueue.NewConfig() + Expect(cfg).ToNot(BeNil()) + Expect(cfg.Workers).To(Equal(taskQueue.WorkersDefault)) + Expect(cfg.StartManagerDelay).To(Equal(taskQueue.StartManagerDelayDefault)) + Expect(cfg.DispatchTasksDelay).To(Equal(taskQueue.DispatchTasksDelayDefault)) + Expect(cfg.MonitorTaskDelay).To(Equal(taskQueue.MonitorTaskDelayDefault)) + Expect(cfg.UnstickTasksDelay).To(Equal(taskQueue.UnstickTasksDelayDefault)) + Expect(cfg.UnstickTasksAvailableGracePeriod).To(Equal(taskQueue.UnstickTasksAvailableGracePeriodDefault)) + Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(taskQueue.RunnerWatchdogGracePeriodDefault)) + Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) + }) + }) + + Context("with new config", func() { + var cfg *taskQueue.Config + + BeforeEach(func() { + cfg = taskQueue.NewConfig() + }) + + Context("Load", func() { + var configReporter *configTest.Reporter + + BeforeEach(func() { + configReporter = configTest.NewReporter() + }) + + It("returns an error when the config reporter is missing", func() { + Expect(cfg.Load(nil)).To(MatchError("config reporter is missing")) + }) + + It("returns an error when workers is not parsable", func() { + configReporter.Config["workers"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("workers is invalid")) + }) + + It("returns an error when start manager delay is not parsable", func() { + configReporter.Config["start_manager_delay"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("start manager delay is invalid")) + }) + + It("returns an error when dispatch tasks delay is not parsable", func() { + configReporter.Config["dispatch_tasks_delay"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("dispatch tasks delay is invalid")) + }) + + It("returns an error when monitor task delay is not parsable", func() { + configReporter.Config["monitor_task_delay"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("monitor task delay is invalid")) + }) + + It("returns an error when runner watchdog grace period is not parsable", func() { + configReporter.Config["runner_watchdog_grace_period"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("runner watchdog grace period is invalid")) + }) + + It("returns an error when unstick tasks delay is not parsable", func() { + configReporter.Config["unstick_tasks_delay"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("unstick tasks delay is invalid")) + }) + + It("returns an error when unstick tasks available grace period is not parsable", func() { + configReporter.Config["unstick_tasks_available_grace_period"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("unstick tasks available grace period is invalid")) + }) + + It("returns an error when stop wait timeout is not parsable", func() { + configReporter.Config["stop_wait_timeout"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("stop wait timeout is invalid")) + }) + + It("uses existing workers if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.Workers).To(Equal(taskQueue.WorkersDefault)) + }) + + It("uses existing start manager delay if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.StartManagerDelay).To(Equal(taskQueue.StartManagerDelayDefault)) + }) + + It("uses existing dispatch tasks delay if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.DispatchTasksDelay).To(Equal(taskQueue.DispatchTasksDelayDefault)) + }) + + It("uses existing monitor task delay if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.MonitorTaskDelay).To(Equal(taskQueue.MonitorTaskDelayDefault)) + }) + + It("uses existing runner watchdog grace period if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(taskQueue.RunnerWatchdogGracePeriodDefault)) + }) + + It("uses existing unstick tasks delay if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.UnstickTasksDelay).To(Equal(taskQueue.UnstickTasksDelayDefault)) + }) + + It("uses existing unstick tasks available grace period if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.UnstickTasksAvailableGracePeriod).To(Equal(taskQueue.UnstickTasksAvailableGracePeriodDefault)) + }) + + It("uses existing stop wait timeout if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) + }) + + It("returns successfully and uses values from the config reporter", func() { + configReporter.Config["workers"] = "5" + configReporter.Config["start_manager_delay"] = "45" + configReporter.Config["dispatch_tasks_delay"] = "30" + configReporter.Config["monitor_task_delay"] = "75" + configReporter.Config["runner_watchdog_grace_period"] = "20" + configReporter.Config["unstick_tasks_delay"] = "60" + configReporter.Config["unstick_tasks_available_grace_period"] = "10" + configReporter.Config["stop_wait_timeout"] = "15" + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.Workers).To(Equal(5)) + Expect(cfg.StartManagerDelay).To(Equal(45 * time.Second)) + Expect(cfg.DispatchTasksDelay).To(Equal(30 * time.Second)) + Expect(cfg.MonitorTaskDelay).To(Equal(75 * time.Second)) + Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(20 * time.Second)) + Expect(cfg.UnstickTasksDelay).To(Equal(60 * time.Second)) + Expect(cfg.UnstickTasksAvailableGracePeriod).To(Equal(10 * time.Second)) + Expect(cfg.StopWaitTimeout).To(Equal(15 * time.Second)) + }) + }) + + Context("Validate", func() { + It("returns an error when workers is less than 1", func() { + cfg.Workers = 0 + Expect(cfg.Validate()).To(MatchError("workers is invalid")) + }) + + It("returns an error when start manager delay is invalid", func() { + cfg.StartManagerDelay = 0 + Expect(cfg.Validate()).To(MatchError("start manager delay is invalid")) + }) + + It("returns an error when dispatch tasks delay is invalid", func() { + cfg.DispatchTasksDelay = 0 + Expect(cfg.Validate()).To(MatchError("dispatch tasks delay is invalid")) + }) + + It("returns an error when monitor task delay is invalid", func() { + cfg.MonitorTaskDelay = 0 + Expect(cfg.Validate()).To(MatchError("monitor task delay is invalid")) + }) + + It("returns an error when runner watchdog grace period is invalid", func() { + cfg.RunnerWatchdogGracePeriod = 0 + Expect(cfg.Validate()).To(MatchError("runner watchdog grace period is invalid")) + }) + + It("returns an error when unstick tasks delay is invalid", func() { + cfg.UnstickTasksDelay = 0 + Expect(cfg.Validate()).To(MatchError("unstick tasks delay is invalid")) + }) + + It("returns an error when unstick tasks available grace period is invalid", func() { + cfg.UnstickTasksAvailableGracePeriod = 0 + Expect(cfg.Validate()).To(MatchError("unstick tasks available grace period is invalid")) + }) + + It("returns an error when stop wait timeout is invalid", func() { + cfg.StopWaitTimeout = 0 + Expect(cfg.Validate()).To(MatchError("stop wait timeout is invalid")) + }) + + It("returns successfully", func() { + Expect(cfg.Validate()).To(Succeed()) + }) + }) + }) + }) + + Context("New", func() { + var cfg *taskQueue.Config + var lgr *logTest.Logger + var str *taskStoreTest.Store + + BeforeEach(func() { + cfg = taskQueue.NewConfig() + lgr = logTest.NewLogger() + str = taskStoreTest.NewStore() + }) + + It("returns an error when name is missing", func() { + que, err := taskQueue.New("", cfg, lgr, str) + Expect(err).To(MatchError("name is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when config is missing", func() { + que, err := taskQueue.New(taskTest.RandomType(), nil, lgr, str) + Expect(err).To(MatchError("config is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when logger is missing", func() { + que, err := taskQueue.New(taskTest.RandomType(), cfg, nil, str) + Expect(err).To(MatchError("logger is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when store is missing", func() { + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, nil) + Expect(err).To(MatchError("store is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when config is invalid", func() { + cfg.Workers = 0 + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str) + Expect(err).To(MatchError("config is invalid; workers is invalid")) + Expect(que).To(BeNil()) + }) + + It("returns an error when a runner is missing", func() { + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, nil) + Expect(err).To(MatchError("runner is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when multiple runners have the same type", func() { + typ := taskTest.RandomType() + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, taskQueueTest.NewCountingRunner(typ), taskQueueTest.NewCountingRunner(typ)) + Expect(err).To(MatchError("runner type already registered")) + Expect(que).To(BeNil()) + }) + + It("returns an error when a runner duration maximum is not positive", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDeadline(3 * time.Minute). + WithTimeout(2 * time.Minute). + WithDurationMaximum(0) + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner) + Expect(err).To(MatchError("runner duration maximum is invalid")) + Expect(que).To(BeNil()) + }) + + It("returns an error when a runner timeout does not exceed its duration maximum", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDeadline(3 * time.Minute). + WithTimeout(time.Minute). + WithDurationMaximum(time.Minute) + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner) + Expect(err).To(MatchError("runner timeout is invalid")) + Expect(que).To(BeNil()) + }) + + It("returns an error when a runner deadline does not exceed its timeout", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDeadline(2 * time.Minute). + WithTimeout(2 * time.Minute). + WithDurationMaximum(time.Minute) + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner) + Expect(err).To(MatchError("runner deadline is invalid")) + Expect(que).To(BeNil()) + }) + + It("returns successfully", func() { + str.NewTaskRepositoryOutputs = []taskStore.TaskRepository{nil} + que := test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str)) + Expect(que).ToNot(BeNil()) + }) + + It("returns successfully with multiple runners of different types", func() { + str.NewTaskRepositoryOutputs = []taskStore.TaskRepository{nil} + que := test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, taskQueueTest.NewCountingRunner(taskTest.RandomType()), taskQueueTest.NewCountingRunner(taskTest.RandomType()))) + Expect(que).ToNot(BeNil()) + }) + }) + + Context("with a new queue", func() { + var que *taskQueue.Queue + + BeforeEach(func() { + cfg := taskQueue.NewConfig() + lgr := logTest.NewLogger() + str := taskStoreTest.NewStore() + str.NewTaskRepositoryOutputs = []taskStore.TaskRepository{nil} + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, taskQueueTest.NewCountingRunner(taskTest.RandomType()))) + }) + + Context("Start", func() { + It("does nothing when called after Stop, since the queue is single-use", func() { + que.Stop() + Expect(func() { que.Start() }).ToNot(Panic()) + // A no-op Start launches no goroutines, so a subsequent Stop stays a no-op too. + Expect(func() { que.Stop() }).ToNot(Panic()) + }) + + It("is safe to call more than once", func() { + que.Start() + Expect(func() { que.Start() }).ToNot(Panic()) + que.Stop() + }) + }) + + Context("Stop", func() { + It("does not panic when called without a prior call to Start", func() { + Expect(func() { que.Stop() }).ToNot(Panic()) + }) + + It("succeeds if called twice", func() { + Expect(func() { que.Stop() }).ToNot(Panic()) + Expect(func() { que.Stop() }).ToNot(Panic()) + }) + }) + }) + + Context("with a store", func() { + var lgr *logTest.Logger + var ctx context.Context + var str *taskStoreMongo.Store + + BeforeEach(func() { + lgr = logTest.NewLogger() + ctx = log.NewContextWithLogger(context.Background(), lgr) + + cfg := storeStructuredMongoTest.NewConfig() + str = test.Must(taskStoreMongo.NewStore(cfg)) + _ = test.Must(str.GetRepository("tasks").DeleteMany(ctx, bson.M{})) + }) + + AfterEach(func() { + _ = test.Must(str.GetRepository("tasks").DeleteMany(ctx, bson.M{})) + Expect(str.Terminate(ctx)).To(Succeed()) + }) + + Context("with successful shutdown", func() { + var cfg *taskQueue.Config + var que *taskQueue.Queue + + BeforeEach(func() { + cfg = taskQueue.NewConfig() + cfg.Workers = 2 + cfg.StartManagerDelay = time.Millisecond + cfg.DispatchTasksDelay = time.Millisecond + }) + + AfterEach(func() { + if que != nil { + que.Stop() + } + lgr.AssertDebug("Task queue worker stopped", log.Fields{"error": errors.NewSerializable(context.Canceled)}) + lgr.AssertDebug("Task queue manager stopped", log.Fields{"error": errors.NewSerializable(context.Canceled)}) + }) + + It("fails a pending task that does not match any registered runner", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + unregisteredType := taskTest.RandomType() + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: unregisteredType})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("runner not found for task type")) + + lgr.AssertError("Runner not found for task type; task cannot be processed") + Expect(testutil.ToFloat64(taskQueue.RunnerNotFoundTotal.WithLabelValues(unregisteredType))).To(Equal(float64(1))) + }) + + It("dispatches, runs, and completes a pending task matching a registered runner", func() { + runner := taskQueueTest.NewCountingRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + Expect(runner.GetCount()).To(Equal(1)) + }) + + It("completes a task that the runner updated while it was running", func() { + updatedData := metadataTest.RandomMetadataMap() + + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + *runnerTask = *test.Must(str.NewTaskRepository().UpdateTask(runnerContext, runnerTask.ID, nil, &task.TaskUpdate{Data: &updatedData})) + runnerTask.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.Error).To(BeNil()) + Expect(actualTask.Data).To(Equal(updatedData)) + }) + + It("logs a warning if the runner update the task while it was running, but did not use the updated task", func() { + updatedData := metadataTest.RandomMetadataMap() + + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + test.Must(str.NewTaskRepository().UpdateTask(runnerContext, runnerTask.ID, nil, &task.TaskUpdate{Data: &updatedData})) + runnerTask.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.Error).To(BeNil()) + Expect(actualTask.Data).To(BeNil()) + + lgr.AssertWarn("Database task revision does not match running task revision; Runner contract broken or concurrent update") + }) + + It("does not complete a task whose claim token changed while it was running", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + // Simulate the task being unstuck and re-claimed elsewhere by changing the claim token out from + // under this run. The completion must then miss rather than falsely complete another run task. + test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": runnerTask.ID}, bson.M{"$set": bson.M{"claimToken": ""}})) + runnerTask.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertError("Unable to stop task; no running task matched the id and claim token") + return true + }, "5s", "100ms").To(BeTrue()) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateRunning)) + }) + + It("does not complete a task that was deleted while it was running", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": runnerTask.ID})) + runnerTask.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertError("Unable to stop task; no running task matched the id and claim token") + return true + }, "5s", "100ms").To(BeTrue()) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask).To(BeNil()) + }) + + It("cancels a running task when the task is deleted mid-run", func() { + cfg.MonitorTaskDelay = time.Millisecond + taskQueue.RunClaimLostTotal.Reset() + + canceled := make(chan error, 1) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": runnerTask.ID})) + <-runnerContext.Done() + canceled <- context.Cause(runnerContext) + runnerTask.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(canceled, "5s").Should(Receive(MatchError("task claim lost"))) + + // The outcome is discarded rather than written back as a lost completion. + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertWarn("Task claim lost; task run outcome discarded") + return true + }, "5s", "100ms").To(BeTrue()) + + Expect(test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil))).To(BeNil()) + Expect(testutil.ToFloat64(taskQueue.RunClaimLostTotal.WithLabelValues(runner.GetRunnerType(), "deleted"))).To(Equal(float64(1))) + }) + + It("cancels a running task deleted after earlier claim checks found the claim intact", func() { + cfg.MonitorTaskDelay = 50 * time.Millisecond + taskQueue.RunClaimLostTotal.Reset() + + canceled := make(chan error, 1) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + // Run past several claim checks before losing the claim, so the monitor must keep checking + // rather than settle after the first. + time.Sleep(5 * cfg.MonitorTaskDelay) + test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": runnerTask.ID})) + <-runnerContext.Done() + canceled <- context.Cause(runnerContext) + runnerTask.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(canceled, "5s").Should(Receive(MatchError("task claim lost"))) + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertWarn("Task claim lost; task run outcome discarded") + return true + }, "5s", "100ms").To(BeTrue()) + + Expect(test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil))).To(BeNil()) + Expect(testutil.ToFloat64(taskQueue.RunClaimLostTotal.WithLabelValues(runner.GetRunnerType(), "deleted"))).To(Equal(float64(1))) + }) + + It("cancels a running task when the task is re-claimed mid-run", func() { + cfg.MonitorTaskDelay = 100 * time.Millisecond + taskQueue.RunClaimLostTotal.Reset() + + canceled := make(chan error, 1) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + // Simulate the task being unstuck and re-claimed elsewhere by changing the claim token out from + // under this run. + test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": runnerTask.ID}, bson.M{"$set": bson.M{"claimToken": "other-claim-token"}})) + <-runnerContext.Done() + canceled <- context.Cause(runnerContext) + runnerTask.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(canceled, "5s").Should(Receive(MatchError("task claim lost"))) + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertWarn("Task claim lost; task run outcome discarded") + return true + }, "5s", "100ms").To(BeTrue()) + + // The foreign claim remains untouched; this run wrote nothing back. + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateRunning)) + Expect(testutil.ToFloat64(taskQueue.RunClaimLostTotal.WithLabelValues(runner.GetRunnerType(), "reclaimed"))).To(Equal(float64(1))) + }) + + It("does not cancel a running task whose claim is intact", func() { + cfg.MonitorTaskDelay = time.Millisecond + + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + // Linger across several claim checks; the claim is intact, so the run must not be canceled. + select { + case <-runnerContext.Done(): + runnerTask.AppendError(context.Cause(runnerContext)) + runnerTask.SetFailed() + case <-time.After(250 * time.Millisecond): + runnerTask.SetCompleted() + } + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.Error).To(BeNil()) + }) + + It("cleans up a task that panics during execution", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { panic("panic test") }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("unhandled panic")) + + lgr.AssertError("Unhandled panic while running task") + Expect(testutil.ToFloat64(taskQueue.RunPanicTotal.WithLabelValues(runner.GetRunnerType()))).To(Equal(float64(1))) + }) + + It("fails a task whose runner leaves it in an unknown state", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { runnerTask.State = "unknown-state" }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("unknown task state")) + }) + + It("warns and sets the available time for a pending task left without one", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + runnerTask.State = task.TaskStatePending + runnerTask.AvailableTime = nil + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + defer func() { _ = recover() }() + lgr.AssertWarn("Available time missing for pending task") + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStatePending)) + + que.Stop() + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + }) + + It("warns when a pending task available time is significantly in the past", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + runnerTask.RepeatAvailableAt(time.Now().Add(-2 * time.Minute)) + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + defer func() { _ = recover() }() + lgr.AssertWarn("Available time significantly before now for pending task") + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStatePending)) + + que.Stop() + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + }) + + It("unsticks and logs a task left running past its deadline", func() { + cfg.UnstickTasksDelay = time.Millisecond + cfg.UnstickTasksAvailableGracePeriod = 50 * time.Millisecond + + // Keep the unstuck task's availability delay short so it is re-dispatched within the wait below. + cfg.MonitorTaskDelay = time.Millisecond + cfg.RunnerWatchdogGracePeriod = 50 * time.Millisecond + + runner := taskQueueTest.NewCountingRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := &task.Task{ + ID: task.NewID(), + Type: runner.GetRunnerType(), + State: task.TaskStateRunning, + CreatedTime: time.Now(), + Revision: 1, + ClaimToken: pointer.FromString(taskTest.RandomType()), + DeadlineTime: pointer.FromTime(time.Now().Add(-time.Minute)), + } + test.Must(str.GetCollection("tasks").InsertOne(ctx, createdTask)) + + que.Start() + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertInfo("Unstuck tasks") + return true + }, "5s", "50ms").To(BeTrue()) + + // Once unstuck, the task returns to pending and is dispatched, run, and completed. + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + }) + + It("reverts a task that is still running to pending when the queue is stopped", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { <-runnerContext.Done() }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + // Wait until the runner has picked up the task and it is running. + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateRunning)) + + // Stopping cancels the worker context; the hanging runner returns leaving the task running, so the + // queue must revert it to pending for a later retry. + que.Stop() + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + Expect(actualTask.DeadlineTime).To(BeNil()) + }) + + It("cancels the context of a task that exceeds its timeout", func() { + runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), time.Minute, 100*time.Millisecond, 50*time.Millisecond, 10*time.Second) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "1m", "50ms").To(Equal(task.TaskStateCompleted)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("task runner timeout exceeded")) + }) + + It("fails a task whose runner returns without setting a terminal state", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("runner failed to set state")) + }) + + It("fails a task that exceeds its timeout without setting its own state", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDurationMaximum(100 * time.Millisecond). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { <-runnerContext.Done() }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("task runner timeout exceeded")) + + lgr.AssertWarn("Task runner exceeded timeout; task will be failed") + Expect(testutil.ToFloat64(taskQueue.RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType(), "recovered"))).To(Equal(float64(1))) + }) + + It("logs a warning if a task that exceeds its maximum duration", func() { + runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 2*time.Minute, time.Minute, time.Millisecond, 10*time.Millisecond) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "1m", "50ms").To(Equal(task.TaskStateCompleted)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.Error).To(BeNil()) + + lgr.AssertWarn("Task duration exceeds maximum") + }) + + It("does not race or panic when Stop is called concurrently", func() { + runner := taskQueueTest.NewCountingRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + var waitGroup sync.WaitGroup + for range 10 { + waitGroup.Go(func() { + defer GinkgoRecover() + que.Stop() + }) + } + waitGroup.Wait() + }) + }) + + Context("without successful shutdown", func() { + var cfg *taskQueue.Config + var que *taskQueue.Queue + + BeforeEach(func() { + cfg = taskQueue.NewConfig() + cfg.Workers = 2 + cfg.StartManagerDelay = time.Millisecond + cfg.DispatchTasksDelay = time.Millisecond + cfg.StopWaitTimeout = 250 * time.Millisecond + }) + + AfterEach(func() { + if que != nil { + que.Stop() + } + }) + + It("returns from Stop within the stop timeout when a runner ignores cancellation", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDurationMaximum(time.Minute). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { select {} }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + // Wait until the blocking runner has picked up the task and it is running. + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateRunning)) + + // Stop must return within roughly the stop timeout even though the runner never returns; it abandons + // the in-flight task rather than blocking forever. + stopped := make(chan struct{}) + go func() { + defer GinkgoRecover() + que.Stop() + close(stopped) + }() + Eventually(stopped, "5s").Should(BeClosed()) + + lgr.AssertError("Task queue workers did not stop within timeout; abandoning in-flight tasks; will be fixed with UnstickTasks later") + }) + + It("logs and counts the run once its timeout elapses", func() { + cfg.RunnerWatchdogGracePeriod = 50 * time.Millisecond + + // A short duration maximum yields a short runner timeout (2x), and a short grace period keeps the + // watchdog prompt, so the watchdog fires quickly while the runner is still blocked, without an unstick + // reclaiming the task. + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDurationMaximum(20 * time.Millisecond). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { select {} }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + runnerType := runner.GetRunnerType() + taskQueue.RunnerTimeoutExceededTotal.Reset() + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runnerType})) + + que.Start() + + // Wait until the runner has picked up the task and it is running. + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateRunning)) + + // The watchdog fires after the runner timeout, recording the stuck run as blocked. + Eventually(func() float64 { + return testutil.ToFloat64(taskQueue.RunnerTimeoutExceededTotal.WithLabelValues(runnerType, "blocked")) + }, "5s", "50ms").Should(BeNumerically(">=", float64(1))) + lgr.AssertError("Task runner exceeded timeout without returning; worker is blocked until it returns") + + // The runner never returns, so Stop abandons the in-flight worker. + stopped := make(chan struct{}) + go func() { + defer GinkgoRecover() + que.Stop() + close(stopped) + }() + Eventually(stopped, "5s").Should(BeClosed()) + }) + }) + + Context("with multiple queues", func() { + const queueCount = 10 + const workersCount = 10 + const taskCount = 2 * queueCount * workersCount + + var runner *taskQueueTest.StubRunner + var ques []*taskQueue.Queue + + BeforeEach(func() { + runner = taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { runnerTask.State = task.TaskStatePending }) + + cfg := taskQueue.NewConfig() + cfg.Workers = workersCount + cfg.StartManagerDelay = time.Millisecond + cfg.DispatchTasksDelay = time.Millisecond + cfg.UnstickTasksDelay = time.Millisecond + ques = make([]*taskQueue.Queue, queueCount) + for index := range len(ques) { + ques[index] = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + } + }) + + AfterEach(func() { + for _, que := range ques { + que.Stop() + } + }) + + It("completes all running tasks when stopped", func() { + tasks := make(task.Tasks, taskCount) + for index := range len(tasks) { + tasks[index] = test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + } + + for _, que := range ques { + que.Start() + } + + select { + case <-ctx.Done(): + case <-time.After(2 * time.Second): + } + + for _, que := range ques { + que.Stop() + } + + for _, tsk := range tasks { + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, tsk.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + } + }) + + It("eventually a queue attempts to run a task that is already running in another queue", func() { + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + for _, que := range ques { + que.Start() + } + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertInfo("Task no longer available to start") + return true + }, "10s", "100ms").To(BeTrue()) + + for _, que := range ques { + que.Stop() + } + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + }) + }) + }) }) diff --git a/task/queue/runner.go b/task/queue/runner.go new file mode 100644 index 0000000000..f3d4e1009e --- /dev/null +++ b/task/queue/runner.go @@ -0,0 +1,111 @@ +package queue + +import ( + "context" + "time" + + "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/task" +) + +// Runner processes one task type on the queue. This comment is the complete contract; no other queue code need be +// consulted to implement it. +// +// Registration: one Runner per task type, registered at queue construction (duplicate types fail construction). The +// queue owns the single instance for the queue's lifetime. +// +// Concurrency: the queue runs multiple workers (configurable; 5 by default) that share the one Runner instance, so Run +// is called concurrently from multiple goroutines - any state shared across calls must be synchronized. Each call's +// *task.Task is distinct and call-owned: don't retain it past Run or touch another call's task. The Get* methods are +// also called concurrently and must be stable (in practice, constants). +// +// Duration contract, validated at construction (violations fail construction): +// +// GetRunnerDeadline() > GetRunnerTimeout() > GetRunnerDurationMaximum() > 0 +// +// Run's obligation: the task arrives running and already claimed under a unique per-run claim token - a value the queue +// set when it claimed the task; a later write-back to the task only lands while that same token is still in place (see +// the lost-write note under Delivery). Before returning, Run MUST move it out of running by calling exactly one +// terminal helper on tsk: +// +// - tsk.SetCompleted() - success; done. +// - tsk.SetFailed() - permanent failure; done. +// - tsk.SetFailedWithError(e) - permanent failure with error; done. +// - tsk.RepeatAvailableAfter(d) - reschedule after duration d. +// - tsk.RepeatAvailableAt(t) - reschedule at time t. +// +// The task arrives with AvailableTime cleared, so reschedule via the Repeat* helpers (which set both pending state and +// a new available time); a reschedule time in the past is clamped to now (run ASAP). Setting pending without an +// available time is a bug the queue only papers over with a warning. If Run returns with the task still running, the +// queue: reverts it to pending if interrupted by shutdown (parent context canceled); else fails it, recording the +// timeout as cause if the context was canceled for timeout, otherwise a "runner failed to set state" error. +// +// Context: canceled (with cause) after GetRunnerTimeout, on shutdown, and on claim loss - cooperative runners select on +// ctx.Done() and return promptly. To tell them apart, inspect context.Cause(ctx): a normal shutdown with +// context.Canceled; a runner timeout with ErrRunnerTimeoutExceeded, and a lost claim (the task was deleted mid-run, or +// unstuck and its claim token replaced) with task.ErrClaimLost (use errors.Is in all cases). After a claim loss the +// run's outcome is discarded - no write-back can land - so return promptly and skip any remaining work. Claim loss is +// detected by a periodic check (MonitorTaskDelay, 1 minute by default), not instantly. The queue cannot preempt a +// runner that ignores cancellation; it stays blocked until it returns or the process restarts, recovered by the +// deadline/unstick mechanism (a periodic sweep that resets tasks still running past their deadline - see +// GetRunnerDeadline). The logger is on the context (log.LoggerFromContext(ctx)). So that cancellation doesn't abandon +// them, wrap must-complete writes in: +// +// ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) +// defer cancel() +// +// Delivery is at-least-once, so Run must tolerate re-execution of the same logical task: a runner that overruns its +// deadline is unstuck and re-dispatched (possibly while the original run is still in flight), and a crash or kill +// mid-run re-runs the task from the start. Conversely a final write can be lost - if the task was unstuck and +// re-claimed while Run was still working, the queue's write-back on return no longer matches the claim token and is +// dropped (logged and counted, not applied). Design terminal effects to be idempotent and safe to repeat. +// +// Errors: accumulate with tsk.AppendError(err), reset with tsk.ClearError() (commonly at run start). Setting an error +// does not change state - pair it with a terminal helper. Persisted with the task; logged at error level if the task +// failed, warning otherwise (e.g. rescheduled). A panic is recovered and an "unhandled panic" error attached; the task +// is then failed unless the runner had already moved it out of running before panicking (e.g. a task completed before +// the panic stays completed, with the error attached). Unlike a normal return, a panic skips the post-run +// reconciliation, so a panic during shutdown fails the task rather than reverting it to pending. +// +// Persistence: on return the queue persists only the terminal state, the available time (written only when +// rescheduling; a terminal task's available time was already cleared when the task was claimed), tsk.Data (mutate the +// map in place, or assign a new map if it arrived nil; the map is written as a whole-field replacement, so its final +// contents are exactly what is stored - keys dropped from the map are gone, and a nil map unsets the field), and the +// accumulated error - mutate only these. Run time and duration are recorded by the queue; other fields are +// queue-managed, don't set them. To persist Data mid-run, mutate tsk and let the queue write it back on return; a +// Runner that instead updates its OWN task via task.Client must replace tsk with the returned task or the queue's +// write-back clobbers it (see task.Client.UpdateTask). +// +// NOTE: The task system is superseded by the work system. New asynchronous processing implements work.Processor and +// registers a work.ProcessorFactory (see the work/base helpers, and oura/data/work for examples). +type Runner interface { + // GetRunnerType returns the task type this runner processes. Unique across a queue's runners; must match + // task.Task.Type of the tasks it handles. + GetRunnerType() string + + // GetRunnerDeadline returns the duration, measured from when the task starts running, after which a still-running + // task is forcibly reset to pending/available by the unstick mechanism, recovering tasks orphaned by crashes, + // kills, or a runner that never returns. The reset is not immediate: unstick runs on a periodic sweep, so a task is + // recovered on the first sweep after its deadline elapses. Must exceed GetRunnerTimeout so a task still within its + // timeout is never unstuck and re-claimed mid-run. + GetRunnerDeadline() time.Duration + + // GetRunnerTimeout returns the hard cap on a run; at elapse the Run context is canceled so a cooperative runner can + // return. Must exceed GetRunnerDurationMaximum. + GetRunnerTimeout() time.Duration + + // GetRunnerDurationMaximum returns the expected upper bound of a normal run. Advisory: exceeding it logs a warning + // but does not interrupt the run. Must be positive. + GetRunnerDurationMaximum() time.Duration + + // Run executes tsk within ctx. Before returning it must move tsk out of running (completed, failed, or rescheduled + // via Repeat*); a task left running is failed by the queue, unless interrupted by shutdown, in which case it is + // reverted to pending. ctx is canceled after GetRunnerTimeout, on shutdown, and when the run's claim on the task is + // lost. Called concurrently for distinct tasks; must be concurrency-safe. + Run(ctx context.Context, tsk *task.Task) +} + +// ErrRunnerTimeoutExceeded is the cancellation cause set on the Run context when a run exceeds the runner timeout. A +// runner distinguishes a timeout from a shutdown with errors.Is(context.Cause(ctx), ErrRunnerTimeoutExceeded); a +// shutdown instead cancels with context.Canceled. +var ErrRunnerTimeoutExceeded = errors.New("task runner timeout exceeded") diff --git a/task/queue/test/runner.go b/task/queue/test/runner.go index 751cb2c23e..5e0aba14f8 100644 --- a/task/queue/test/runner.go +++ b/task/queue/test/runner.go @@ -22,12 +22,13 @@ func NewCountingRunner(typ string) *CountingRunner { mu: &sync.Mutex{}, } } + func (c *CountingRunner) GetRunnerType() string { return c.Type } -func (c *CountingRunner) GetRunnerDeadline() time.Time { - return time.Now().Add(time.Second * 10) +func (c *CountingRunner) GetRunnerDeadline() time.Duration { + return time.Second * 10 } func (c *CountingRunner) GetRunnerTimeout() time.Duration { @@ -53,3 +54,119 @@ func (c *CountingRunner) GetCount() int { } var _ queue.Runner = &CountingRunner{} + +type SleepRunner struct { + Type string + Deadline time.Duration + Timeout time.Duration + Maximum time.Duration + Sleep time.Duration +} + +func NewSleepRunner(typ string, deadline time.Duration, timeout time.Duration, maximum time.Duration, sleep time.Duration) *SleepRunner { + return &SleepRunner{ + Type: typ, + Deadline: deadline, + Timeout: timeout, + Maximum: maximum, + Sleep: sleep, + } +} + +func (s *SleepRunner) GetRunnerType() string { + return s.Type +} + +func (s *SleepRunner) GetRunnerDeadline() time.Duration { + return s.Deadline +} + +func (s *SleepRunner) GetRunnerTimeout() time.Duration { + return s.Timeout +} + +func (s *SleepRunner) GetRunnerDurationMaximum() time.Duration { + return s.Maximum +} + +func (s *SleepRunner) Run(ctx context.Context, tsk *task.Task) { + select { + case <-ctx.Done(): + tsk.AppendError(context.Cause(ctx)) + tsk.SetCompleted() + case <-time.After(s.Sleep): + tsk.SetCompleted() + } +} + +var _ queue.Runner = &SleepRunner{} + +type StubRunner struct { + Type string + Stub func(runnerContext context.Context, runnerTask *task.Task) + deadline *time.Duration + timeout *time.Duration + durationMaximum *time.Duration +} + +func NewStubRunner(typ string) *StubRunner { + return &StubRunner{ + Type: typ, + } +} + +func (s *StubRunner) GetRunnerType() string { + return s.Type +} + +func (s *StubRunner) GetRunnerDeadline() time.Duration { + if s.deadline != nil { + return *s.deadline + } else { + return s.GetRunnerDurationMaximum() * 4 + } +} + +func (s *StubRunner) GetRunnerTimeout() time.Duration { + if s.timeout != nil { + return *s.timeout + } else { + return s.GetRunnerDurationMaximum() * 2 + } +} + +func (s *StubRunner) GetRunnerDurationMaximum() time.Duration { + if s.durationMaximum != nil { + return *s.durationMaximum + } else { + return time.Minute + } +} + +func (s *StubRunner) Run(ctx context.Context, tsk *task.Task) { + if s.Stub != nil { + s.Stub(ctx, tsk) + } +} + +func (s *StubRunner) WithStub(stub func(runnerContext context.Context, runnerTask *task.Task)) *StubRunner { + s.Stub = stub + return s +} + +func (s *StubRunner) WithDeadline(deadline time.Duration) *StubRunner { + s.deadline = &deadline + return s +} + +func (s *StubRunner) WithTimeout(timeout time.Duration) *StubRunner { + s.timeout = &timeout + return s +} + +func (s *StubRunner) WithDurationMaximum(durationMaximum time.Duration) *StubRunner { + s.durationMaximum = &durationMaximum + return s +} + +var _ queue.Runner = &StubRunner{} diff --git a/task/service/api/v1/v1.go b/task/service/api/v1/v1.go index de7033e74b..df3e09f401 100644 --- a/task/service/api/v1/v1.go +++ b/task/service/api/v1/v1.go @@ -93,7 +93,13 @@ func (r *Router) GetTask(res rest.ResponseWriter, req *rest.Request) { return } - tsk, err := r.TaskClient().GetTask(req.Context(), id) + condition := request.NewCondition() + if err := request.DecodeRequestQuery(req.Request, condition); err != nil { + responder.Error(http.StatusBadRequest, err) + return + } + + tsk, err := r.TaskClient().GetTask(req.Context(), id, condition) if err != nil { responder.InternalServerError(err) return @@ -114,13 +120,19 @@ func (r *Router) UpdateTask(res rest.ResponseWriter, req *rest.Request) { return } + condition := request.NewCondition() + if err := request.DecodeRequestQuery(req.Request, condition); err != nil { + responder.Error(http.StatusBadRequest, err) + return + } + update := task.NewTaskUpdate() if err := request.DecodeRequestBody(req.Request, update); err != nil { responder.Error(http.StatusBadRequest, err) return } - tsk, err := r.TaskClient().UpdateTask(req.Context(), id, update) + tsk, err := r.TaskClient().UpdateTask(req.Context(), id, condition, update) if err != nil { responder.InternalServerError(err) return @@ -141,7 +153,13 @@ func (r *Router) DeleteTask(res rest.ResponseWriter, req *rest.Request) { return } - err := r.TaskClient().DeleteTask(req.Context(), id) + condition := request.NewCondition() + if err := request.DecodeRequestQuery(req.Request, condition); err != nil { + responder.Error(http.StatusBadRequest, err) + return + } + + err := r.TaskClient().DeleteTask(req.Context(), id, condition) if err != nil { responder.InternalServerError(err) return diff --git a/task/service/service/client.go b/task/service/service/client.go index ff9c1ab69d..f212b3701c 100644 --- a/task/service/service/client.go +++ b/task/service/service/client.go @@ -5,6 +5,8 @@ import ( "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/page" + "github.com/tidepool-org/platform/request" + storeStructured "github.com/tidepool-org/platform/store/structured" "github.com/tidepool-org/platform/task" taskStore "github.com/tidepool-org/platform/task/store" ) @@ -33,17 +35,17 @@ func (c *Client) CreateTask(ctx context.Context, create *task.TaskCreate) (*task return repository.CreateTask(ctx, create) } -func (c *Client) GetTask(ctx context.Context, id string) (*task.Task, error) { +func (c *Client) GetTask(ctx context.Context, id string, condition *request.Condition) (*task.Task, error) { repository := c.taskStore.NewTaskRepository() - return repository.GetTask(ctx, id) + return repository.GetTask(ctx, id, storeStructured.MapCondition(condition)) } -func (c *Client) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { +func (c *Client) UpdateTask(ctx context.Context, id string, condition *request.Condition, update *task.TaskUpdate) (*task.Task, error) { repository := c.taskStore.NewTaskRepository() - return repository.UpdateTask(ctx, id, update) + return repository.UpdateTask(ctx, id, storeStructured.MapCondition(condition), update) } -func (c *Client) DeleteTask(ctx context.Context, id string) error { +func (c *Client) DeleteTask(ctx context.Context, id string, condition *request.Condition) error { repository := c.taskStore.NewTaskRepository() - return repository.DeleteTask(ctx, id) + return repository.DeleteTask(ctx, id, storeStructured.MapCondition(condition)) } diff --git a/task/service/service/service.go b/task/service/service/service.go index 6e8d776cf9..4fd343d7a0 100644 --- a/task/service/service/service.go +++ b/task/service/service/service.go @@ -36,7 +36,7 @@ type Service struct { dataClient dataClient.Client dataSourceClient dataSource.Client dexcomClient dexcom.Client - taskQueue queue.Queue + taskQueue *queue.MultiQueue clinicsClient clinics.Client } @@ -136,7 +136,7 @@ func (s *Service) initializeTaskStore() error { func (s *Service) terminateTaskStore() { if s.taskStore != nil { s.Logger().Debug("Closing task store") - s.taskStore.Terminate(context.Background()) + _ = s.taskStore.Terminate(context.Background()) s.Logger().Debug("Destroying task store") s.taskStore = nil @@ -283,15 +283,6 @@ func (s *Service) initializeTaskQueue() error { return errors.Wrap(err, "unable to load task queue config") } - s.Logger().Debug("Creating task queue") - - taskQueue, err := queue.NewMultiQueue(cfg, s.Logger(), s.TaskStore()) - if err != nil { - return errors.Wrap(err, "unable to create task queue") - } - - s.taskQueue = taskQueue - var runners []queue.Runner if s.dexcomClient != nil { @@ -327,13 +318,15 @@ func (s *Service) initializeTaskQueue() error { } runners = append(runners, ehrSyncRnnr) - for _, r := range runners { - r := r - if err := taskQueue.RegisterRunner(r); err != nil { - return errors.Wrapf(err, "unable to register runner %s", r.GetRunnerType()) - } + s.Logger().Debug("Creating task queue") + + taskQueue, err := queue.NewMultiQueue(cfg, s.Logger(), s.TaskStore(), runners...) + if err != nil { + return errors.Wrap(err, "unable to create task queue") } + s.taskQueue = taskQueue + s.Logger().Debug("Starting task queue") s.taskQueue.Start() diff --git a/task/store/mongo/mongo.go b/task/store/mongo/mongo.go index cd5798cc74..3e8c20d1b3 100644 --- a/task/store/mongo/mongo.go +++ b/task/store/mongo/mongo.go @@ -2,6 +2,8 @@ package mongo import ( "context" + "fmt" + "slices" "time" "github.com/prometheus/client_golang/prometheus" @@ -10,26 +12,27 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "github.com/tidepool-org/platform/ehr/reconcile" + ehrReconcile "github.com/tidepool-org/platform/ehr/reconcile" "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/id" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/page" "github.com/tidepool-org/platform/pointer" + storeStructured "github.com/tidepool-org/platform/store/structured" storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" structureValidator "github.com/tidepool-org/platform/structure/validator" summaryTask "github.com/tidepool-org/platform/summary/task" "github.com/tidepool-org/platform/task" - "github.com/tidepool-org/platform/task/store" + taskStore "github.com/tidepool-org/platform/task/store" ) -var ( - TasksStateTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tidepool_task_tasks_state_total", - Help: "The total number of tasks sorted by state and type", - }, []string{"state", "type"}) -) +const ( + MaxTaskCreationDuration = 30 * time.Second -const MaxTaskCreationDuration = 30 * time.Second + // TransitionTimeout bounds a task state-transition write (start or stop) that must complete regardless of the + // caller's context being canceled (e.g. during shutdown). + TransitionTimeout = 10 * time.Second +) type Store struct { *storeStructuredMongo.Store @@ -47,14 +50,14 @@ func NewStore(config *storeStructuredMongo.Config) (*Store, error) { }, nil } -func (s *Store) WithTypeFilter(typeFilter string) store.Store { +func (s *Store) WithTypeFilter(typeFilter string) taskStore.Store { return &Store{ Store: s.Store, typeFilter: &typeFilter, } } -func (s *Store) NewTaskRepository() store.TaskRepository { +func (s *Store) NewTaskRepository() taskStore.TaskRepository { repo := s.TaskRepository() repo.typeFilter = s.typeFilter return repo @@ -91,29 +94,13 @@ func (s *Store) EnsureDefaultTasks() error { return nil } -func (s *Store) EnsureSummaryMigrationTask() error { - ctx, cancel := context.WithTimeout(context.Background(), MaxTaskCreationDuration) - defer cancel() - - repository := s.TaskRepository() - return repository.EnsureSummaryMigrationTask(ctx) -} - -func (s *Store) EnsureEHRReconcileTask() error { - ctx, cancel := context.WithTimeout(context.Background(), MaxTaskCreationDuration) - defer cancel() - - repository := s.TaskRepository() - return repository.EnsureEHRReconcileTask(ctx) -} - type TaskRepository struct { *storeStructuredMongo.Repository typeFilter *string } func (t *TaskRepository) EnsureIndexes() error { - // Repositories operation only a subset of the tasks shouldn't invoke this method + // Repositories operating on a subset of the tasks shouldn't invoke this method if t.typeFilter != nil { return errors.New("calling EnsureIndexes() on a partitioned repository is not allowed") } @@ -122,35 +109,33 @@ func (t *TaskRepository) EnsureIndexes() error { { Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index(). - SetUnique(true). - SetBackground(true), + SetUnique(true), }, { Keys: bson.D{{Key: "name", Value: 1}}, Options: options.Index(). SetUnique(true). - SetSparse(true). - SetBackground(true), + SetSparse(true), }, { - Keys: bson.D{{Key: "priority", Value: 1}}, - Options: options.Index(). - SetBackground(true), + Keys: bson.D{{Key: "availableTime", Value: 1}}, }, { - Keys: bson.D{{Key: "availableTime", Value: 1}}, - Options: options.Index(). - SetBackground(true), + Keys: bson.D{{Key: "state", Value: 1}}, }, { - Keys: bson.D{{Key: "expirationTime", Value: 1}}, + // Used by IteratePending; type equality, then availableTime for range and sort; partial on pending since + // that is the only state it queries. + Keys: bson.D{{Key: "type", Value: 1}, {Key: "availableTime", Value: 1}}, Options: options.Index(). - SetBackground(true), + SetPartialFilterExpression(bson.D{{Key: "state", Value: task.TaskStatePending}}), }, { - Keys: bson.D{{Key: "state", Value: 1}}, + // Used by UnstickTasks; type equality, then deadlineTime for range and sort; partial on running since that + // is the only state it queries. + Keys: bson.D{{Key: "type", Value: 1}, {Key: "deadlineTime", Value: 1}}, Options: options.Index(). - SetBackground(true), + SetPartialFilterExpression(bson.D{{Key: "state", Value: task.TaskStateRunning}}), }, }) } @@ -178,7 +163,7 @@ func (t *TaskRepository) EnsureSummaryMigrationTask(ctx context.Context) error { } func (t *TaskRepository) EnsureEHRReconcileTask(ctx context.Context) error { - create := reconcile.NewTaskCreate() + create := ehrReconcile.NewTaskCreate() return t.ensureTask(ctx, create) } @@ -189,30 +174,14 @@ func (t *TaskRepository) ensureTask(ctx context.Context, create *task.TaskCreate } else if err = structureValidator.New(log.LoggerFromContext(ctx)).Validate(tsk); err != nil { return errors.Wrap(err, "task is invalid") } - if err := t.assertType(t.typeFilter, &tsk.Type); err != nil { - return err - } - - upsert := true - after := options.After - opts := options.FindOneAndUpdateOptions{ - ReturnDocument: &after, - Upsert: &upsert, - } - - res := t.FindOneAndUpdate(ctx, - bson.M{"name": tsk.Name}, - bson.M{"$setOnInsert": tsk}, - &opts, - ) - if res.Err() != nil && !errors.Is(res.Err(), mongo.ErrNoDocuments) { - return errors.Wrap(res.Err(), "unable to create task") + if result, err := t.UpdateOne(ctx, bson.M{"name": tsk.Name}, bson.M{"$setOnInsert": tsk}, options.Update().SetUpsert(true)); err != nil { + return errors.Wrap(err, "unable to create task") + } else if result.UpsertedCount > 0 { + TypeStateTotal.WithLabelValues(create.Type, task.TaskStatePending).Inc() } - TasksStateTotal.WithLabelValues(task.TaskStatePending, create.Type).Inc() - - return res.Err() + return nil } func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) { @@ -233,10 +202,11 @@ func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, return nil, err } - now := time.Now() logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"filter": filter, "pagination": pagination}) - tasks := task.Tasks{} + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("ListTasks") }() + selector := bson.M{} if filter.Name != nil { @@ -248,15 +218,21 @@ func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, if filter.State != nil { selector["state"] = *filter.State } - opts := storeStructuredMongo.FindWithPagination(pagination). - SetSort(bson.M{"createdTime": -1}) + + if t.typeFilter != nil { + selector["type"] = *t.typeFilter + } + + opts := storeStructuredMongo.FindWithPagination(pagination).SetSort(bson.M{"createdTime": -1}) cursor, err := t.Find(ctx, selector, opts) - logger.WithFields(log.Fields{"count": len(tasks), "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("ListTasks") if err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to list tasks") } + tasks := task.Tasks{} if err = cursor.All(ctx, &tasks); err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to decode tasks") } @@ -264,6 +240,12 @@ func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, tasks = task.Tasks{} } + taskIds := make([]string, len(tasks)) + for index, tsk := range tasks { + taskIds[index] = tsk.ID + } + logger = logger.WithField("taskIds", taskIds) + return tasks, nil } @@ -278,228 +260,471 @@ func (t *TaskRepository) CreateTask(ctx context.Context, create *task.TaskCreate } else if err = structureValidator.New(log.LoggerFromContext(ctx)).Validate(tsk); err != nil { return nil, errors.Wrap(err, "task is invalid") } - if err := t.assertType(t.typeFilter, &tsk.Type); err != nil { + if err = t.assertType(t.typeFilter, &tsk.Type); err != nil { return nil, err } - now := time.Now() logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"create": create}) - _, err = t.InsertOne(ctx, tsk) - logger.WithFields(log.Fields{"id": tsk.ID, "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("CreateTask") - if err != nil { + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("CreateTask") }() + + if _, err = t.InsertOne(ctx, tsk); err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to create task") } - TasksStateTotal.WithLabelValues(task.TaskStatePending, create.Type).Inc() + logger = logger.WithField("task", tsk.LogFields()) + + TypeStateTotal.WithLabelValues(create.Type, task.TaskStatePending).Inc() return tsk, nil } -func (t *TaskRepository) GetTask(ctx context.Context, id string) (*task.Task, error) { +func (t *TaskRepository) GetTask(ctx context.Context, id string, condition *storeStructured.Condition) (*task.Task, error) { if ctx == nil { return nil, errors.New("context is missing") } if id == "" { return nil, errors.New("id is missing") } - - now := time.Now() - logger := log.LoggerFromContext(ctx).WithField("id", id) - - var task *task.Task - - selector := bson.M{"id": id} - if t.typeFilter != nil { - selector["type"] = t.typeFilter + if condition == nil { + condition = &storeStructured.Condition{} + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return nil, errors.Wrap(err, "condition is invalid") } - err := t.FindOne(ctx, selector).Decode(&task) - logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("GetTask") + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "condition": condition}) + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("GetTask") }() + + tsk := &task.Task{} + err := t.FindOne(ctx, t.selector(id, condition)).Decode(tsk) if errors.Is(err, mongo.ErrNoDocuments) { return nil, nil } else if err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to get task") } - return task, nil + return tsk, nil } -func (t *TaskRepository) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { +func (t *TaskRepository) UpdateTask(ctx context.Context, id string, condition *storeStructured.Condition, update *task.TaskUpdate) (*task.Task, error) { if ctx == nil { return nil, errors.New("context is missing") } if id == "" { return nil, errors.New("id is missing") } + if condition == nil { + condition = &storeStructured.Condition{} + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return nil, errors.Wrap(err, "condition is invalid") + } if update == nil { return nil, errors.New("update is missing") } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(update); err != nil { return nil, errors.Wrap(err, "update is invalid") } - now := time.Now() - logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "update": update}) + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "condition": condition, "update": update}) - set := bson.M{ - "modifiedTime": now, + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("UpdateTask") }() + + set, unset := t.parseUpdate(update) + set["modifiedTime"] = now + + updatedTask := &task.Task{} + opts := options.FindOneAndUpdate().SetReturnDocument(options.After) + err := t.FindOneAndUpdate(ctx, t.selector(id, condition), t.ConstructUpdate(set, unset), opts).Decode(updatedTask) + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, nil + } else if err != nil { + logger = logger.WithError(err) + return nil, errors.Wrap(err, "unable to update task") } - if update.Priority != nil { - set["priority"] = *update.Priority + + return updatedTask, nil +} + +func (t *TaskRepository) DeleteTask(ctx context.Context, id string, condition *storeStructured.Condition) error { + if ctx == nil { + return errors.New("context is missing") } - if update.Data != nil { - set["data"] = *update.Data + if id == "" { + return errors.New("id is missing") } - if update.AvailableTime != nil { - set["availableTime"] = *update.AvailableTime + if condition == nil { + condition = &storeStructured.Condition{} + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return errors.Wrap(err, "condition is invalid") } - if update.ExpirationTime != nil { - set["expirationTime"] = *update.ExpirationTime + + logger := log.LoggerFromContext(ctx).WithField("id", id) + + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("DeleteTask") }() + + if changeInfo, err := t.DeleteOne(ctx, t.selector(id, condition)); err != nil { + logger = logger.WithError(err) + return errors.Wrap(err, "unable to delete task") + } else { + logger = logger.WithField("changeInfo", changeInfo) } - selector := bson.M{"id": id} - if t.typeFilter != nil { - selector["type"] = t.typeFilter + return nil +} + +func (t *TaskRepository) StartTask(ctx context.Context, id string, revision int, deadline time.Duration) (*task.Task, error) { + if ctx == nil { + return nil, errors.New("context is missing") + } + if id == "" { + return nil, errors.New("id is missing") + } else if !task.IsValidID(id) { + return nil, errors.New("id is invalid") + } + if deadline <= 0 { + return nil, errors.New("deadline is invalid") } - changeInfo, err := t.UpdateMany(ctx, selector, t.ConstructUpdate(set, bson.M{})) - logger.WithFields(log.Fields{"changeInfo": changeInfo, "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("UpdateTask") - if err != nil { - return nil, errors.Wrap(err, "unable to update task") + // Add a timeout, but ignore cancel from the parent context so the claim write completes and its outcome is known. A + // write abandoned mid-flight (e.g. on shutdown) can still commit in the database, leaving the task running with no + // reliable way to revert it. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), TransitionTimeout) + defer cancel() + + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "revision": revision, "deadline": deadline.String()}) + + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("StartTask") }() + + set := bson.M{ + "state": task.TaskStateRunning, + "runTime": now, + "deadlineTime": now.Add(deadline), + "modifiedTime": now, + "claimToken": newClaimTokenWithRevision(revision), + } + unset := bson.M{ + "availableTime": 1, + "duration": 1, + } + + selector := t.selector(id, storeStructured.NewConditionWithRevision(&revision)) + selector["state"] = task.TaskStatePending + + tsk := &task.Task{} + opts := options.FindOneAndUpdate().SetReturnDocument(options.After) + err := t.FindOneAndUpdate(ctx, selector, t.ConstructUpdate(set, unset), opts).Decode(tsk) + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, nil + } else if err != nil { + logger = logger.WithError(err) + return nil, errors.Wrap(err, "unable to start task") } - return t.GetTask(ctx, id) + TypeStateTotal.WithLabelValues(tsk.Type, task.TaskStateRunning).Inc() + return tsk, nil } -func (t *TaskRepository) DeleteTask(ctx context.Context, id string) error { +// Will only timeout after 10 seconds even if parent context is canceled. +func (t *TaskRepository) StopTask(ctx context.Context, id string, revision int, claimToken *string, state string, duration *time.Duration, update *task.TaskUpdate) error { if ctx == nil { return errors.New("context is missing") } if id == "" { return errors.New("id is missing") + } else if !task.IsValidID(id) { + return errors.New("id is invalid") + } + if claimToken == nil { + return errors.New("claim token is missing") + } else if *claimToken == "" { + return errors.New("claim token is invalid") + } + if state == "" { + return errors.New("state is missing") + } else if !slices.Contains(task.TaskStates(), state) { + return errors.New("state is invalid") + } + if duration != nil && *duration < 0 { + return errors.New("duration is invalid") + } + if update != nil { + if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(update); err != nil { + return errors.Wrap(err, "update is invalid") + } } - now := time.Now() - logger := log.LoggerFromContext(ctx).WithField("id", id) + // Add a timeout, but ignore cancel from parent context to ensure we stop task even exiting + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), TransitionTimeout) + defer cancel() - selector := bson.M{"id": id} - if t.typeFilter != nil { - selector["type"] = t.typeFilter + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "revision": revision, "claimToken": claimToken, "state": state, "duration": duration, "update": update}) + + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("StopTask") }() + + set, unset := t.parseUpdate(update) + set["modifiedTime"] = now + set["state"] = state + unset["deadlineTime"] = 1 + unset["claimToken"] = 1 + if duration != nil { + set["duration"] = duration.Truncate(time.Millisecond).Seconds() + } else { + // A nil duration means no run actually happened (e.g. a claimed task whose dispatch was reverted during + // shutdown), so also clear the run time recorded by StartTask, keeping the runTime/duration pair describing + // only the last actual run. + unset["duration"] = 1 + unset["runTime"] = 1 + } + + selector := t.selector(id, nil) + selector["state"] = task.TaskStateRunning + selector["claimToken"] = claimToken + + partial := &task.Task{} + opts := options.FindOneAndUpdate().SetProjection(bson.M{"_id": 0, "type": 1, "revision": 1}) + err := t.FindOneAndUpdate(ctx, selector, t.ConstructUpdate(set, unset), opts).Decode(partial) + if errors.Is(err, mongo.ErrNoDocuments) { + // The compare-and-swap missed: no running task matched the expected claim token (it was concurrently modified, + // unstuck, or deleted since it started). The state transition is dropped; the deadline and unstick mechanism + // will recover the task if it was left running. This is logged and counted so lost completions are observable + // rather than silently swallowed. + logger.Error("Unable to stop task; no running task matched the id and claim token") + TypeLostCompletionTotal.WithLabelValues(pointer.Default(t.typeFilter, "")).Inc() + return nil + } else if err != nil { + logger = logger.WithError(err) + return errors.Wrap(err, "unable to stop task") } - changeInfo, err := t.DeleteOne(ctx, selector) - logger.WithFields(log.Fields{"changeInfo": changeInfo, "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("DeleteTask") - if err != nil { - return errors.Wrap(err, "unable to delete task") + // If the on-disk task revision does not match the runner task revision, then either: + // - the runner did not follow the Runner contract; i.e. the runner updated the task during run, but it did not + // use the updated task, or, + // - the task was concurrently modified outside of the runner while the task was running. + if partial.Revision != revision { + logger.WithField("revision", log.Fields{"expected": revision, "actual": partial.Revision}).Warn("Database task revision does not match running task revision; Runner contract broken or concurrent update") + TypeRevisionMismatchTotal.WithLabelValues(partial.Type).Inc() } + TypeStateTotal.WithLabelValues(partial.Type, state).Inc() return nil } -// TODO: Consider using an "update only specific fields" approach, as above - -func (t *TaskRepository) UpdateFromState(ctx context.Context, tsk *task.Task, state string) (*task.Task, error) { +// UnstickTasks resets tasks still running past their deadline back to pending, making each available again +// availabilityDelay in the future - long enough for the (possibly still running) previous run to observe its claim loss +// and exit before the task is re-dispatched. +func (t *TaskRepository) UnstickTasks(ctx context.Context, availabilityDelay time.Duration) ([]string, error) { if ctx == nil { return nil, errors.New("context is missing") } - if tsk == nil { - return nil, errors.New("task is missing") + if availabilityDelay < 0 { + return nil, errors.New("availability delay is invalid") } - now := time.Now() - logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": tsk.ID, "state": state}) + logger := log.LoggerFromContext(ctx) - tsk.ModifiedTime = pointer.FromTime(now.Truncate(time.Millisecond)) + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("UnstickTasks") }() - selector := bson.M{ - "id": tsk.ID, - "state": state, + findSelector := bson.M{ + "state": task.TaskStateRunning, + "deadlineTime": bson.M{"$lt": now}, } if t.typeFilter != nil { - selector["type"] = t.typeFilter + findSelector["type"] = *t.typeFilter } - result, err := t.ReplaceOne(ctx, selector, tsk) - logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("UpdateFromState") + + opts := options.Find(). + SetProjection(bson.M{"_id": 0, "id": 1, "deadlineTime": 1}). + SetSort(bson.M{"deadlineTime": 1}) + cursor, err := t.Find(ctx, findSelector, opts) if err != nil { - return nil, errors.Wrap(err, "unable to update from state") + logger = logger.WithError(err) + return nil, errors.Wrap(err, "unable to list tasks") + } + defer storeStructuredMongo.CloseCursor(ctx, cursor) + + var ids []string + for cursor.Next(ctx) { + partial := &task.Task{} + if err = cursor.Decode(partial); err != nil { + logger = logger.WithError(err) + log.LoggerFromContext(ctx).WithError(err).Error("Unable to decode task") + continue + } + + // The state clause is defensive: a matching deadline time already implies the same running claim (every stop + // clears the deadline time and any re-claim records a strictly later one), but including it keeps the invariant + // local to this update. + updateSelector := bson.M{ + "id": partial.ID, + "state": task.TaskStateRunning, + "deadlineTime": partial.DeadlineTime, + } + set := bson.M{ + "state": task.TaskStatePending, + "availableTime": now.Add(availabilityDelay), + "modifiedTime": now, + } + unset := bson.M{ + "deadlineTime": 1, + "claimToken": 1, + } + if result, updateErr := t.UpdateOne(ctx, updateSelector, t.ConstructUpdate(set, unset)); updateErr != nil { + logger = logger.WithError(updateErr) + return ids, errors.Wrap(updateErr, "unable to update task") + } else if result.ModifiedCount > 0 { + ids = append(ids, partial.ID) + } } - if result.ModifiedCount != 1 { - return nil, task.AlreadyClaimedTask + + logger = logger.WithField("taskIds", ids) + + if err = cursor.Err(); err != nil { + logger = logger.WithError(err) + return ids, err // Still want to return the ids of tasks that were successfully unstuck } - TasksStateTotal.WithLabelValues(tsk.State, tsk.Type).Inc() - return tsk, nil + return ids, nil } -func (t *TaskRepository) UnstickTasks(ctx context.Context) (int64, error) { - selector := bson.M{ - "state": task.TaskStateRunning, - "deadlineTime": bson.M{"$lt": time.Now()}, +func (t *TaskRepository) GetTaskClaimToken(ctx context.Context, id string) (*string, bool, error) { + if ctx == nil { + return nil, false, errors.New("context is missing") } - if t.typeFilter != nil { - selector["type"] = t.typeFilter + if id == "" { + return nil, false, errors.New("id is missing") } - update := bson.M{ - "$set": bson.M{"state": task.TaskStatePending}, - "$unset": bson.M{"deadlineTime": ""}, - } + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id}) - result, err := t.UpdateMany(ctx, selector, update) - if err != nil { - return 0, err + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("GetTaskClaimToken") }() + + partial := &task.Task{} + opts := options.FindOne().SetProjection(bson.M{"_id": 0, "claimToken": 1}) + err := t.FindOne(ctx, t.selector(id, nil), opts).Decode(partial) + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, false, nil + } else if err != nil { + logger = logger.WithError(err) + return nil, false, errors.Wrap(err, "unable to get task claim token") } - return result.ModifiedCount, err + return partial.ClaimToken, true, nil } func (t *TaskRepository) IteratePending(ctx context.Context) (*mongo.Cursor, error) { - now := time.Now() + now := time.Now().UTC() selector := bson.M{ "state": task.TaskStatePending, - "$and": []bson.M{ + "$or": []bson.M{ { - "$or": []bson.M{ - { - "availableTime": bson.M{ - "$exists": false, - }, - }, - { - "availableTime": bson.M{ - "$lte": now, - }, - }, + "availableTime": bson.M{ + "$exists": false, }, }, { - "$or": []bson.M{ - { - "expirationTime": bson.M{ - "$exists": false, - }, - }, - { - "expirationTime": bson.M{ - "$gt": now, - }, - }, + "availableTime": bson.M{ + "$lte": now, }, }, }, } if t.typeFilter != nil { - selector["type"] = t.typeFilter + selector["type"] = *t.typeFilter } - opts := options.Find().SetSort(bson.M{"priority": -1}) + opts := options.Find().SetSort(bson.D{{Key: "availableTime", Value: 1}}) return t.Find(ctx, selector, opts) } +func (t *TaskRepository) selector(id string, condition *storeStructured.Condition) bson.M { + selector := bson.M{"id": id} + if condition != nil { + if condition.Revision != nil { + if *condition.Revision == 0 { + selector["revision"] = bson.M{"$in": bson.A{0, nil}} + } else { + selector["revision"] = *condition.Revision + } + } + } + if t.typeFilter != nil { + selector["type"] = *t.typeFilter + } + return selector +} + +func (t *TaskRepository) parseUpdate(update *task.TaskUpdate) (bson.M, bson.M) { + set := bson.M{} + unset := bson.M{} + + if update != nil { + if update.Data != nil { + if *update.Data != nil { + set["data"] = *update.Data + } else { + unset["data"] = true + } + } + if update.AvailableTime != nil { + set["availableTime"] = *update.AvailableTime + } + if update.Error != nil { + if update.Error.Error != nil { + set["error"] = *update.Error + } else { + unset["error"] = true + } + } + } + + return set, unset +} + // assertType return an error if the expected type doesn't match the actual type func (t *TaskRepository) assertType(expected *string, actual *string) error { if expected != nil && actual != nil && *expected != *actual { - return errors.Newf("expected task type %s but got %s", *expected, *actual) + return errors.Newf("expected task type %q, but got %q", *expected, *actual) } return nil } + +// newClaimTokenWithRevision embeds the claimed revision so successive claims of the same task can never collide +// (revision is strictly monotonic per document); the random suffix distinguishes claims across tasks. The token is +// opaque everywhere else. +func newClaimTokenWithRevision(revision int) string { + return fmt.Sprintf("%d:%s", revision, id.Must(id.New(16))) +} + +var ( + // TypeStateTotal counts the total number of tasks run, sorted by type and state. + TypeStateTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_type_state_total", + Help: "The total number of tasks run, sorted by type and state", + }, []string{"type", "state"}) + + // TypeLostCompletionTotal counts task completions dropped because the compare-and-swap in StopTask missed (task + // claim token was concurrently modified, task unstuck, or task deleted). The task is recovered by the + // deadline/unstick mechanism, but the intended terminal state is lost. The type label is populated only when the + // repository is type-filtered (as it is for each queue in a MultiQueue); otherwise it is empty. + TypeLostCompletionTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_type_lost_completion_total", + Help: "The total number of task completions dropped because the claim-token compare-and-swap missed, sorted by type", + }, []string{"type"}) + + // TypeRevisionMismatchTotal counts task completions where the task revision does not match the task revision in the + // database. This only occurs if the task runner does not follow the Runner contract when updating a task during + // run, or if there is a concurrent modification outside of the expected Runner behavior. + TypeRevisionMismatchTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_type_revision_mismatch_total", + Help: "The total number of task revisions that do not match the task revision in the database, sorted by type", + }, []string{"type"}) +) diff --git a/task/store/mongo/mongo_test.go b/task/store/mongo/mongo_test.go index 756cc33ed0..cb2ce78425 100644 --- a/task/store/mongo/mongo_test.go +++ b/task/store/mongo/mongo_test.go @@ -2,7 +2,7 @@ package mongo_test import ( "context" - "strings" + "strconv" "time" . "github.com/onsi/ginkgo/v2" @@ -13,7 +13,7 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" - "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/ehr/reconcile" "github.com/tidepool-org/platform/log" logTest "github.com/tidepool-org/platform/log/test" "github.com/tidepool-org/platform/pointer" @@ -22,6 +22,8 @@ import ( "github.com/tidepool-org/platform/task" taskStore "github.com/tidepool-org/platform/task/store" taskStoreMongo "github.com/tidepool-org/platform/task/store/mongo" + taskTest "github.com/tidepool-org/platform/task/test" + "github.com/tidepool-org/platform/test" ) var _ = Describe("Mongo", func() { @@ -37,7 +39,7 @@ var _ = Describe("Mongo", func() { AfterEach(func() { if str != nil { - str.Terminate(context.Background()) + _ = str.Terminate(context.Background()) } }) @@ -83,31 +85,31 @@ var _ = Describe("Mongo", func() { "Key": Equal(storeStructuredMongoTest.MakeKeySlice("_id")), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("id")), - "Background": Equal(true), - "Unique": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("id")), + "Unique": Equal(true), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("name")), - "Background": Equal(true), - "Unique": Equal(true), - "Sparse": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("name")), + "Unique": Equal(true), + "Sparse": Equal(true), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("priority")), - "Background": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("availableTime")), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("availableTime")), - "Background": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("state")), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("expirationTime")), - "Background": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("type", "availableTime")), + "PartialFilterExpression": Equal(bson.D{ + {Key: "state", Value: task.TaskStatePending}, + }), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("state")), - "Background": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("type", "deadlineTime")), + "PartialFilterExpression": Equal(bson.D{ + {Key: "state", Value: task.TaskStateRunning}, + }), }), )) }) @@ -129,152 +131,311 @@ var _ = Describe("Mongo", func() { ctx = log.NewContextWithLogger(context.Background(), logger) }) - Context("with an existing task", func() { - var tsk *task.Task + Context("StartTask", func() { + It("embeds the claimed revision in the claim token", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) - BeforeEach(func() { - var err error - tsk, err = task.NewTask(context.Background(), &task.TaskCreate{ - Name: pointer.FromString("test"), - Type: "fetch", - Priority: 0, - Data: nil, - AvailableTime: pointer.FromTime(time.Now()), - ExpirationTime: pointer.FromTime(time.Now().Add(5 * time.Minute)), - }) + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(startedTask.ClaimToken).To(PointTo(HavePrefix(strconv.Itoa(pendingTask.Revision) + ":"))) + }) + + It("generates a distinct claim token for successive claims of the same task", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + firstStartedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(firstStartedTask).ToNot(BeNil()) + Expect(repository.StopTask(ctx, firstStartedTask.ID, firstStartedTask.Revision, firstStartedTask.ClaimToken, task.TaskStatePending, nil, nil)).To(Succeed()) + + stoppedTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": pendingTask.ID}).Decode(stoppedTask)).To(Succeed()) + + secondStartedTask := test.Must(repository.StartTask(ctx, stoppedTask.ID, stoppedTask.Revision, time.Minute)) + Expect(secondStartedTask).ToNot(BeNil()) + Expect(secondStartedTask.ClaimToken).To(PointTo(Not(Equal(*firstStartedTask.ClaimToken)))) + }) + }) + + Context("StopTask", func() { + It("clears the run time and duration when stopping without a duration", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + Expect(pendingTask.AvailableTime).ToNot(BeNil()) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(startedTask.RunTime).ToNot(BeNil()) + Expect(startedTask.AvailableTime).To(BeNil()) + + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.ClaimToken, task.TaskStatePending, nil, nil)).To(Succeed()) + + actualTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + Expect(actualTask.RunTime).To(BeNil()) + Expect(actualTask.Duration).To(BeNil()) + Expect(actualTask.ClaimToken).To(BeNil()) + Expect(actualTask.DeadlineTime).To(BeNil()) + Expect(actualTask.AvailableTime).To(BeNil()) + }) + + It("retains the run time when stopping with a duration", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + Expect(pendingTask.AvailableTime).ToNot(BeNil()) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(startedTask.RunTime).ToNot(BeNil()) + + duration := time.Second + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.ClaimToken, task.TaskStateCompleted, &duration, nil)).To(Succeed()) + + actualTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.RunTime).To(PointTo(BeTemporally("~", *startedTask.RunTime, time.Millisecond))) + Expect(actualTask.Duration).To(PointTo(Equal(duration.Seconds()))) + Expect(actualTask.ClaimToken).To(BeNil()) + Expect(actualTask.DeadlineTime).To(BeNil()) + Expect(actualTask.AvailableTime).To(BeNil()) + }) + }) + + Context("UnstickTasks", func() { + It("returns an error when the context is missing", func() { + unstuckTaskIDs, err := repository.UnstickTasks(context.Context(nil), 0) + Expect(err).To(MatchError("context is missing")) + Expect(unstuckTaskIDs).To(BeNil()) + }) + + It("returns an error when the availability delay is negative", func() { + unstuckTaskIDs, err := repository.UnstickTasks(ctx, -time.Second) + Expect(err).To(MatchError("availability delay is invalid")) + Expect(unstuckTaskIDs).To(BeNil()) + }) + + It("returns no ids when there are no stuck tasks", func() { + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(BeEmpty()) + }) + + It("unsticks a running task with an expired deadline", func() { + stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) + + actualStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": stuckTask.ID}).Decode(actualStuckTask)).To(Succeed()) + Expect(actualStuckTask.State).To(Equal(task.TaskStatePending)) + Expect(actualStuckTask.AvailableTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) + Expect(actualStuckTask.ModifiedTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) + Expect(actualStuckTask.DeadlineTime).To(BeNil()) + }) + + It("offsets the available time by the availability delay", func() { + stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, time.Minute)) + Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) + + actualStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": stuckTask.ID}).Decode(actualStuckTask)).To(Succeed()) + Expect(actualStuckTask.State).To(Equal(task.TaskStatePending)) + Expect(actualStuckTask.AvailableTime).To(PointTo(BeTemporally("~", test.Now().Add(time.Minute), time.Second))) + }) + + It("does not unstick a running task with a deadline in the future", func() { + notStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(time.Hour))) + + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(BeEmpty()) + + actualNotStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": notStuckTask.ID}).Decode(actualNotStuckTask)).To(Succeed()) + Expect(actualNotStuckTask).To(Equal(notStuckTask)) + }) + + It("does not unstick a task that is not running", func() { + notStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, pointer.FromTime(test.Now().Add(-time.Minute))) + + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(BeEmpty()) + + actualNotStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": notStuckTask.ID}).Decode(actualNotStuckTask)).To(Succeed()) + Expect(actualNotStuckTask).To(Equal(notStuckTask)) + }) + + It("unsticks multiple running tasks with expired deadlines, ordered by deadline time", func() { + laterStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + earlierStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Hour))) + + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(Equal([]string{earlierStuckTask.ID, laterStuckTask.ID})) + }) + + It("only unsticks tasks matching the repository type filter", func() { + stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + otherTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + + filteredRepository := str.WithTypeFilter(stuckTask.Type).NewTaskRepository() + unstuckTaskIDs, err := filteredRepository.UnstickTasks(ctx, 0) + Expect(err).ToNot(HaveOccurred()) + Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) + + actualOtherTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": otherTask.ID}).Decode(actualOtherTask)).To(Succeed()) + Expect(actualOtherTask).To(Equal(otherTask)) + }) + }) + + Context("GetTaskClaimToken", func() { + It("returns an error when the context is missing", func() { + claimToken, exists, err := repository.GetTaskClaimToken(context.Context(nil), taskTest.RandomID()) + Expect(err).To(MatchError("context is missing")) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeFalse()) + }) + + It("returns an error when the id is missing", func() { + claimToken, exists, err := repository.GetTaskClaimToken(ctx, "") + Expect(err).To(MatchError("id is missing")) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeFalse()) + }) + + It("reports the task as not existing when the task does not exist", func() { + claimToken, exists, err := repository.GetTaskClaimToken(ctx, taskTest.RandomID()) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeFalse()) + }) + + It("returns a nil claim token when the task exists, but is not claimed", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + Expect(pendingTask.ClaimToken).To(BeNil()) + + claimToken, exists, err := repository.GetTaskClaimToken(ctx, pendingTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeTrue()) + }) + + It("returns the claim token of a claimed task", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(startedTask.ClaimToken).ToNot(BeNil()) + + claimToken, exists, err := repository.GetTaskClaimToken(ctx, startedTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(PointTo(Equal(*startedTask.ClaimToken))) + Expect(exists).To(BeTrue()) + }) + + It("returns a nil claim token once the task is stopped", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.ClaimToken, task.TaskStateCompleted, nil, nil)).To(Succeed()) + + claimToken, exists, err := repository.GetTaskClaimToken(ctx, startedTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeTrue()) + }) + + It("returns the new claim token once the task is re-claimed", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + firstStartedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(firstStartedTask).ToNot(BeNil()) + Expect(repository.StopTask(ctx, firstStartedTask.ID, firstStartedTask.Revision, firstStartedTask.ClaimToken, task.TaskStatePending, nil, nil)).To(Succeed()) + + stoppedTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": pendingTask.ID}).Decode(stoppedTask)).To(Succeed()) + + secondStartedTask := test.Must(repository.StartTask(ctx, stoppedTask.ID, stoppedTask.Revision, time.Minute)) + Expect(secondStartedTask).ToNot(BeNil()) + + claimToken, exists, err := repository.GetTaskClaimToken(ctx, secondStartedTask.ID) Expect(err).ToNot(HaveOccurred()) - tsk.State = task.TaskStateRunning - _, err = collection.InsertOne(ctx, tsk) + Expect(claimToken).To(PointTo(Equal(*secondStartedTask.ClaimToken))) + Expect(claimToken).To(PointTo(Not(Equal(*firstStartedTask.ClaimToken)))) + Expect(exists).To(BeTrue()) + }) + + It("does not modify the task", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + + _, _, err := repository.GetTaskClaimToken(ctx, startedTask.ID) Expect(err).ToNot(HaveOccurred()) + + actualTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) + Expect(actualTask).To(Equal(startedTask)) }) - Context("UpdateFromState", func() { - var updated *task.Task - - const defaultPrometheusOutput = ` - # HELP tidepool_task_tasks_state_total The total number of tasks sorted by state and type - # TYPE tidepool_task_tasks_state_total counter - tidepool_task_tasks_state_total{ state = "", type = "" } 1 - ` - const metricName = "tidepool_task_tasks_state_total" - - BeforeEach(func() { - taskStoreMongo.TasksStateTotal.Reset() - var err error - updated, err = task.NewTask(context.Background(), &task.TaskCreate{ - Name: pointer.FromString("updated"), - Type: "fetch", - Priority: 0, - Data: nil, - AvailableTime: pointer.FromTime(time.Now()), - ExpirationTime: pointer.FromTime(time.Now().Add(5 * time.Minute)), - }) - Expect(err).ToNot(HaveOccurred()) - updated.ID = tsk.ID - }) - - It("returns an error when the context is missing", func() { - ctx = nil - result, err := repository.UpdateFromState(ctx, updated, tsk.State) - Expect(err).To(MatchError("context is missing")) - Expect(result).To(BeNil()) - }) - - It("returns an error when the updated task is missing", func() { - updated = nil - result, err := repository.UpdateFromState(ctx, updated, tsk.State) - Expect(err).To(MatchError("task is missing")) - Expect(result).To(BeNil()) - }) - - It("successfully fails the task with multiple errors", func() { - updated.State = task.TaskStateFailed - updated.AppendError(errors.New("first error")) - updated.AppendError(errors.New("second error")) - _, err := repository.UpdateFromState(ctx, updated, tsk.State) - Expect(err).ToNot(HaveOccurred()) - - result := task.Task{} - err = collection.FindOne(ctx, bson.M{"id": tsk.ID}).Decode(&result) - Expect(err).ToNot(HaveOccurred()) - Expect(result.State).To(Equal(updated.State)) - Expect(result.Error).To(Equal(updated.Error)) - }) - - It("returns error if task is updated from the same state multiple times", func() { - updated.State = task.TaskStatePending - _, err := repository.UpdateFromState(ctx, updated, tsk.State) - Expect(err).ToNot(HaveOccurred()) - - _, err = repository.UpdateFromState(ctx, updated, tsk.State) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError("Task has already been claimed or is now unavailable.")) - }) - - It("records metrics of completed tasks", func() { - updated.State = task.TaskStateCompleted - completedTask, err := repository.UpdateFromState(ctx, updated, tsk.State) - - Expect(err).ToNot(HaveOccurred()) - - prometheusState := strings.ReplaceAll(defaultPrometheusOutput, "", task.TaskStateCompleted) - expectedOutput := strings.ReplaceAll(prometheusState, "", completedTask.Type) - - prometheusErr := testutil. - CollectAndCompare(taskStoreMongo.TasksStateTotal, strings.NewReader(expectedOutput), metricName) - Expect(prometheusErr).ToNot(HaveOccurred()) - }) - - It("records metrics of failed tasks", func() { - updated.State = task.TaskStateFailed - failedTask, err := repository.UpdateFromState(ctx, updated, tsk.State) - - Expect(err).ToNot(HaveOccurred()) - - prometheusState := strings.ReplaceAll(defaultPrometheusOutput, "", task.TaskStateFailed) - expectedOutput := strings.ReplaceAll(prometheusState, "", failedTask.Type) - - prometheusErr := testutil. - CollectAndCompare(taskStoreMongo.TasksStateTotal, strings.NewReader(expectedOutput), metricName) - Expect(prometheusErr).ToNot(HaveOccurred()) - }) - - It("records metrics of running tasks", func() { - updated.State = task.TaskStateRunning - runningTask, err := repository.UpdateFromState(ctx, updated, tsk.State) - - Expect(err).ToNot(HaveOccurred()) - - prometheusState := strings.ReplaceAll(defaultPrometheusOutput, "", task.TaskStateRunning) - expectedOutput := strings.ReplaceAll(prometheusState, "", runningTask.Type) - - prometheusErr := testutil. - CollectAndCompare(taskStoreMongo.TasksStateTotal, strings.NewReader(expectedOutput), metricName) - Expect(prometheusErr).ToNot(HaveOccurred()) - }) - - It("records metrics of pending tasks", func() { - tskCreate := &task.TaskCreate{ - Name: pointer.FromString("test"), - Type: "fetch", - Priority: 0, - Data: nil, - AvailableTime: pointer.FromTime(time.Now()), - ExpirationTime: pointer.FromTime(time.Now().Add(5 * time.Minute)), - } - _, err := repository.CreateTask(ctx, tskCreate) - Expect(err).ToNot(HaveOccurred()) - - prometheusState := strings.ReplaceAll(defaultPrometheusOutput, "", task.TaskStatePending) - expectedOutput := strings.ReplaceAll(prometheusState, "", tsk.Type) - - prometheusErr := testutil. - CollectAndCompare(taskStoreMongo.TasksStateTotal, strings.NewReader(expectedOutput), metricName) - Expect(prometheusErr).ToNot(HaveOccurred()) - }) + It("returns the claim token of a task matching the repository type filter", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + + filteredRepository := str.WithTypeFilter(startedTask.Type).NewTaskRepository() + claimToken, exists, err := filteredRepository.GetTaskClaimToken(ctx, startedTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(PointTo(Equal(*startedTask.ClaimToken))) + Expect(exists).To(BeTrue()) + }) + + It("reports the task as not existing when it does not match the repository type filter", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + + filteredRepository := str.WithTypeFilter(startedTask.Type + "-other").NewTaskRepository() + claimToken, exists, err := filteredRepository.GetTaskClaimToken(ctx, startedTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("EnsureEHRReconcileTask", func() { + BeforeEach(func() { + taskStoreMongo.TypeStateTotal.Reset() + }) + + It("creates the task and increments the pending metric only on the initial insert", func() { + repository := str.TaskRepository() + Expect(repository).ToNot(BeNil()) + + Expect(repository.EnsureEHRReconcileTask(ctx)).To(Succeed()) + Expect(testutil.ToFloat64(taskStoreMongo.TypeStateTotal)).To(Equal(1.0)) + + Expect(repository.EnsureEHRReconcileTask(ctx)).To(Succeed()) + Expect(testutil.ToFloat64(taskStoreMongo.TypeStateTotal)).To(Equal(1.0)) + + count := test.Must(collection.CountDocuments(context.Background(), bson.M{"type": reconcile.Type})) + Expect(count).To(Equal(int64(1))) }) }) }) }) }) + +func insertTaskWithStateAndDeadlineTime(ctx context.Context, collection *mongo.Collection, state string, deadlineTime *time.Time) *task.Task { + tsk := test.Must(task.NewTask(ctx, taskTest.RandomTaskCreate())) + tsk.State = state + tsk.DeadlineTime = deadlineTime + result := test.Must(collection.InsertOne(ctx, tsk)) + Expect(collection.FindOne(ctx, bson.M{"_id": result.InsertedID}).Decode(tsk)).To(Succeed()) + return tsk +} diff --git a/task/store/store.go b/task/store/store.go index 655acca98e..52cea0bfa2 100644 --- a/task/store/store.go +++ b/task/store/store.go @@ -2,9 +2,12 @@ package store import ( "context" + "time" "go.mongodb.org/mongo-driver/mongo" + "github.com/tidepool-org/platform/page" + storeStructured "github.com/tidepool-org/platform/store/structured" "github.com/tidepool-org/platform/task" ) @@ -15,10 +18,19 @@ type Store interface { } type TaskRepository interface { - task.TaskAccessor + ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) + CreateTask(ctx context.Context, create *task.TaskCreate) (*task.Task, error) + GetTask(ctx context.Context, id string, condition *storeStructured.Condition) (*task.Task, error) + UpdateTask(ctx context.Context, id string, condition *storeStructured.Condition, update *task.TaskUpdate) (*task.Task, error) + DeleteTask(ctx context.Context, id string, condition *storeStructured.Condition) error - UnstickTasks(ctx context.Context) (int64, error) + // Queue use only below + + StartTask(ctx context.Context, id string, revision int, deadline time.Duration) (*task.Task, error) + StopTask(ctx context.Context, id string, revision int, claimToken *string, state string, duration *time.Duration, update *task.TaskUpdate) error + + UnstickTasks(ctx context.Context, availabilityDelay time.Duration) ([]string, error) + GetTaskClaimToken(ctx context.Context, id string) (*string, bool, error) - UpdateFromState(ctx context.Context, tsk *task.Task, state string) (*task.Task, error) IteratePending(ctx context.Context) (*mongo.Cursor, error) } diff --git a/task/store/test/task_session.go b/task/store/test/task_session.go deleted file mode 100644 index 8cb7a8794a..0000000000 --- a/task/store/test/task_session.go +++ /dev/null @@ -1,17 +0,0 @@ -package test - -import "github.com/tidepool-org/platform/test" - -type TasksSession struct { - *test.Closer -} - -func NewTasksSession() *TasksSession { - return &TasksSession{ - Closer: test.NewCloser(), - } -} - -func (t *TasksSession) AssertOutputsEmpty() { - t.Closer.AssertOutputsEmpty() -} diff --git a/task/task.go b/task/task.go index 1c030cf949..be3fe458aa 100644 --- a/task/task.go +++ b/task/task.go @@ -19,15 +19,11 @@ import ( //go:generate mockgen -source=task.go -destination=test/task_mocks.go -package=test -typed type Client interface { - TaskAccessor -} - -type TaskAccessor interface { ListTasks(ctx context.Context, filter *TaskFilter, pagination *page.Pagination) (Tasks, error) CreateTask(ctx context.Context, create *TaskCreate) (*Task, error) - GetTask(ctx context.Context, id string) (*Task, error) - UpdateTask(ctx context.Context, id string, update *TaskUpdate) (*Task, error) - DeleteTask(ctx context.Context, id string) error + GetTask(ctx context.Context, id string, condition *request.Condition) (*Task, error) + UpdateTask(ctx context.Context, id string, condition *request.Condition, update *TaskUpdate) (*Task, error) + DeleteTask(ctx context.Context, id string, condition *request.Condition) error } const ( @@ -83,12 +79,10 @@ func (t *TaskFilter) MutateRequest(req *http.Request) error { } type TaskCreate struct { - Name *string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Priority int `json:"priority,omitempty"` - Data map[string]interface{} `json:"data,omitempty"` - AvailableTime *time.Time `json:"availableTime,omitempty"` - ExpirationTime *time.Time `json:"expirationTime,omitempty"` + Name *string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Data map[string]any `json:"data,omitempty"` + AvailableTime *time.Time `json:"availableTime,omitempty"` } func NewTaskCreate() *TaskCreate { @@ -100,31 +94,21 @@ func (t *TaskCreate) Parse(parser structure.ObjectParser) { if ptr := parser.String("type"); ptr != nil { t.Type = *ptr } - if ptr := parser.Int("priority"); ptr != nil { - t.Priority = *ptr - } if ptr := parser.Object("data"); ptr != nil { t.Data = *ptr } t.AvailableTime = parser.Time("availableTime", time.RFC3339Nano) - t.ExpirationTime = parser.Time("expirationTime", time.RFC3339Nano) } func (t *TaskCreate) Validate(validator structure.Validator) { validator.String("name", t.Name).NotEmpty() validator.String("type", &t.Type).NotEmpty() - expirationTimeValidator := validator.Time("expirationTime", t.ExpirationTime) - expirationTimeValidator.AfterNow(time.Second) - if t.AvailableTime != nil { - expirationTimeValidator.After(*t.AvailableTime) - } } type TaskUpdate struct { - Priority *int `json:"priority,omitempty" bson:"priority,omitempty"` - Data *map[string]interface{} `json:"data,omitempty" bson:"data,omitempty"` - AvailableTime *time.Time `json:"availableTime,omitempty" bson:"availableTime,omitempty"` - ExpirationTime *time.Time `json:"expirationTime,omitempty" bson:"expirationTime,omitempty"` + Data *map[string]any `json:"data,omitempty" bson:"data,omitempty"` + AvailableTime *time.Time `json:"availableTime,omitempty" bson:"availableTime,omitempty"` + Error *errors.Serializable `json:"error,omitempty" bson:"error,omitempty"` } func NewTaskUpdate() *TaskUpdate { @@ -132,22 +116,28 @@ func NewTaskUpdate() *TaskUpdate { } func (t *TaskUpdate) Parse(parser structure.ObjectParser) { - t.Priority = parser.Int("priority") t.Data = parser.Object("data") t.AvailableTime = parser.Time("availableTime", time.RFC3339Nano) - t.ExpirationTime = parser.Time("expirationTime", time.RFC3339Nano) + if parser.ReferenceExists("error") { + t.Error = &errors.Serializable{} + t.Error.Parse("error", parser) + } } func (t *TaskUpdate) Validate(validator structure.Validator) { - expirationTimeValidator := validator.Time("expirationTime", t.ExpirationTime) - expirationTimeValidator.AfterNow(time.Second) - if t.AvailableTime != nil { - expirationTimeValidator.After(*t.AvailableTime) + if t.Error != nil { + t.Error.Validate(validator.WithReference("error")) + } +} + +func (t *TaskUpdate) Normalize(normalizer structure.Normalizer) { + if t.Error != nil { + t.Error.Normalize(normalizer.WithReference("error")) } } func (t *TaskUpdate) IsEmpty() bool { - return t.Priority == nil && t.Data == nil && t.AvailableTime == nil && t.ExpirationTime == nil + return t.Data == nil && t.AvailableTime == nil && t.Error == nil } func NewID() string { @@ -178,20 +168,25 @@ func ErrorValueStringAsIDNotValid(value string) error { var idExpression = regexp.MustCompile("^[0-9a-f]{32}$") type Task struct { - ID string `json:"id,omitempty" bson:"id,omitempty"` - Name *string `json:"name,omitempty" bson:"name,omitempty"` - Type string `json:"type,omitempty" bson:"type,omitempty"` - Priority int `json:"priority,omitempty" bson:"priority,omitempty"` - Data map[string]interface{} `json:"data,omitempty" bson:"data,omitempty"` - AvailableTime *time.Time `json:"availableTime,omitempty" bson:"availableTime,omitempty"` - DeadlineTime *time.Time `json:"deadlineTime,omitempty" bson:"deadlineTime,omitempty"` - ExpirationTime *time.Time `json:"expirationTime,omitempty" bson:"expirationTime,omitempty"` - State string `json:"state,omitempty" bson:"state,omitempty"` - Error *errors.Serializable `json:"error,omitempty" bson:"error,omitempty"` - RunTime *time.Time `json:"runTime,omitempty" bson:"runTime,omitempty"` - Duration *float64 `json:"duration,omitempty" bson:"duration,omitempty"` - CreatedTime time.Time `json:"createdTime,omitempty" bson:"createdTime,omitempty"` - ModifiedTime *time.Time `json:"modifiedTime,omitempty" bson:"modifiedTime,omitempty"` + ID string `json:"id" bson:"id"` + Name *string `json:"name,omitempty" bson:"name,omitempty"` + Type string `json:"type" bson:"type"` + Data map[string]any `json:"data,omitempty" bson:"data,omitempty"` + AvailableTime *time.Time `json:"availableTime,omitempty" bson:"availableTime,omitempty"` + State string `json:"state" bson:"state"` + Error *errors.Serializable `json:"error,omitempty" bson:"error,omitempty"` + RunTime *time.Time `json:"runTime,omitempty" bson:"runTime,omitempty"` + Duration *float64 `json:"duration,omitempty" bson:"duration,omitempty"` + CreatedTime time.Time `json:"createdTime" bson:"createdTime"` + ModifiedTime *time.Time `json:"modifiedTime,omitempty" bson:"modifiedTime,omitempty"` + Revision int `json:"revision" bson:"revision"` + + // Database only + + // ClaimToken fences state transitions: a unique value set each time a run claims the task, which a later stop must + // match, so stale writers miss. + ClaimToken *string `json:"-" bson:"claimToken,omitempty"` + DeadlineTime *time.Time `json:"-" bson:"deadlineTime,omitempty"` } func NewTask(ctx context.Context, create *TaskCreate) (*Task, error) { @@ -201,16 +196,22 @@ func NewTask(ctx context.Context, create *TaskCreate) (*Task, error) { return nil, errors.Wrap(err, "create is invalid") } + now := time.Now().UTC() + + availableTime := create.AvailableTime + if availableTime == nil || availableTime.Before(now) { + availableTime = pointer.From(now) + } + return &Task{ - ID: NewID(), - Name: create.Name, - Type: create.Type, - Priority: create.Priority, - Data: create.Data, - AvailableTime: create.AvailableTime, - ExpirationTime: create.ExpirationTime, - State: TaskStatePending, - CreatedTime: time.Now(), + ID: NewID(), + Name: create.Name, + Type: create.Type, + Data: create.Data, + AvailableTime: availableTime, + State: TaskStatePending, + CreatedTime: now, + Revision: 1, }, nil } @@ -222,14 +223,10 @@ func (t *Task) Parse(parser structure.ObjectParser) { if ptr := parser.String("type"); ptr != nil { t.Type = *ptr } - if ptr := parser.Int("priority"); ptr != nil { - t.Priority = *ptr - } if ptr := parser.Object("data"); ptr != nil { t.Data = *ptr } t.AvailableTime = parser.Time("availableTime", time.RFC3339Nano) - t.ExpirationTime = parser.Time("expirationTime", time.RFC3339Nano) if ptr := parser.String("state"); ptr != nil { t.State = *ptr } @@ -243,17 +240,15 @@ func (t *Task) Parse(parser structure.ObjectParser) { t.CreatedTime = *ptr } t.ModifiedTime = parser.Time("modifiedTime", time.RFC3339Nano) + if ptr := parser.Int("revision"); ptr != nil { + t.Revision = *ptr + } } func (t *Task) Validate(validator structure.Validator) { validator.String("id", &t.ID).Using(IDValidator) validator.String("name", t.Name).NotEmpty() validator.String("type", &t.Type).NotEmpty() - expirationTimeValidator := validator.Time("expirationTime", t.ExpirationTime) - expirationTimeValidator.AfterNow(time.Second) - if t.AvailableTime != nil { - expirationTimeValidator.After(*t.AvailableTime) - } validator.String("state", &t.State).OneOf(TaskStates()...) if t.Error != nil { t.Error.Validate(validator.WithReference("error")) @@ -262,6 +257,7 @@ func (t *Task) Validate(validator structure.Validator) { validator.Float64("duration", t.Duration).GreaterThanOrEqualTo(0) validator.Time("createdTime", &t.CreatedTime).NotZero().BeforeNow(time.Second) validator.Time("modifiedTime", t.ModifiedTime).After(t.CreatedTime).BeforeNow(time.Second) + validator.Int("revision", &t.Revision).GreaterThanOrEqualTo(0) } func (t *Task) Normalize(normalizer structure.Normalizer) { @@ -280,7 +276,6 @@ func (t *Task) Sanitize(details request.AuthDetails) error { func (t *Task) RepeatAvailableAt(availableTime time.Time) { t.State = TaskStatePending t.AvailableTime = pointer.FromTime(availableTime) - t.DeadlineTime = nil } func (t *Task) RepeatAvailableAfter(availableDuration time.Duration) { @@ -293,6 +288,12 @@ func (t *Task) IsFailed() bool { func (t *Task) SetFailed() { t.State = TaskStateFailed + t.AvailableTime = nil +} + +func (t *Task) SetFailedWithError(err error) { + t.AppendError(err) + t.SetFailed() } func (t *Task) IsCompleted() bool { @@ -327,6 +328,16 @@ func (t *Task) ClearError() { t.Error = nil } +func (t *Task) LogFields() log.Fields { + return log.Fields{ + "id": t.ID, + "type": t.Type, + "state": t.State, + "revision": t.Revision, + "claimToken": t.ClaimToken, + } +} + type Tasks []*Task func (t Tasks) Sanitize(details request.AuthDetails) error { @@ -338,4 +349,7 @@ func (t Tasks) Sanitize(details request.AuthDetails) error { return nil } -var AlreadyClaimedTask = errors.New("Task has already been claimed or is now unavailable.") +// ErrClaimLost is the cancellation cause set on the Run context when the run's claim on the task was lost mid-run: the +// task was deleted, or it was unstuck and possibly re-claimed (its claim token no longer matches). A runner +// distinguishes it with errors.Is(context.Cause(ctx), ErrClaimLost). +var ErrClaimLost = errors.New("task claim lost") diff --git a/task/task_test.go b/task/task_test.go index 8021a84d89..6bd583b448 100644 --- a/task/task_test.go +++ b/task/task_test.go @@ -1,13 +1,22 @@ package task_test import ( + "context" + "time" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + "github.com/tidepool-org/platform/errors" errorsTest "github.com/tidepool-org/platform/errors/test" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + "github.com/tidepool-org/platform/pointer" structureTest "github.com/tidepool-org/platform/structure/test" structureValidator "github.com/tidepool-org/platform/structure/validator" "github.com/tidepool-org/platform/task" + taskTest "github.com/tidepool-org/platform/task/test" "github.com/tidepool-org/platform/test" ) @@ -48,4 +57,78 @@ var _ = Describe("Task", func() { Entry("is ErrorValueStringAsIDNotValid with non-empty string", task.ErrorValueStringAsIDNotValid("0123456789abcdef0123456789abcdef"), "value-not-valid", "value is not valid", `value "0123456789abcdef0123456789abcdef" is not valid as task id`), ) }) + + Context("Task", func() { + Context("NewTask", func() { + var ctx context.Context + var create *task.TaskCreate + + BeforeEach(func() { + ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) + create = taskTest.RandomTaskCreate(test.AllowOptionals()) + }) + + It("returns an error when the create is missing", func() { + result, err := task.NewTask(ctx, nil) + errorsTest.ExpectEqual(err, errors.New("create is missing")) + Expect(result).To(BeNil()) + }) + + It("returns an error when the create is invalid", func() { + create.Type = "" + result, err := task.NewTask(ctx, create) + errorsTest.ExpectEqual(err, errors.New("create is invalid")) + Expect(result).To(BeNil()) + }) + + It("returns a new pending task using the current time as the available time when the create does not specify one", func() { + create.AvailableTime = nil + + result := test.Must(task.NewTask(ctx, create)) + Expect(result).To(PointTo(MatchAllFields(Fields{ + "ID": MatchRegexp("^[0-9a-f]{32}$"), + "Name": Equal(create.Name), + "Type": Equal(create.Type), + "Data": Equal(create.Data), + "AvailableTime": PointTo(BeTemporally("~", time.Now(), time.Second)), + "DeadlineTime": BeNil(), + "State": Equal(task.TaskStatePending), + "Error": BeNil(), + "RunTime": BeNil(), + "Duration": BeNil(), + "CreatedTime": BeTemporally("~", time.Now(), time.Second), + "ModifiedTime": BeNil(), + "Revision": Equal(1), + "ClaimToken": BeNil(), + }))) + }) + + It("returns a new pending task using the current time as the available time when the create specifies one in the past", func() { + create.AvailableTime = pointer.From(test.RandomTimeBeforeNow()) + + result := test.Must(task.NewTask(ctx, create)) + Expect(result.AvailableTime).To(PointTo(BeTemporally("~", time.Now(), time.Second))) + }) + + It("returns a new pending task using the specified available time when it is in the future", func() { + create.AvailableTime = pointer.From(test.RandomTimeAfterNow()) + + result := test.Must(task.NewTask(ctx, create)) + Expect(result.AvailableTime).To(PointTo(Equal(*create.AvailableTime))) + }) + }) + + Context("LogFields", func() { + It("returns the id, type, and state as log fields", func() { + tsk := taskTest.RandomTask() + Expect(tsk.LogFields()).To(Equal(log.Fields{ + "id": tsk.ID, + "type": tsk.Type, + "state": tsk.State, + "revision": tsk.Revision, + "claimToken": tsk.ClaimToken, + })) + }) + }) + }) }) diff --git a/task/test/client.go b/task/test/client.go index f160c98d5f..d458cf353a 100644 --- a/task/test/client.go +++ b/task/test/client.go @@ -1,15 +1,151 @@ package test +import ( + "context" + + "github.com/onsi/gomega" + + "github.com/tidepool-org/platform/page" + request "github.com/tidepool-org/platform/request" + "github.com/tidepool-org/platform/task" +) + +type ListTasksInput struct { + Context context.Context //nolint:containedctx // Test only + Filter *task.TaskFilter + Pagination *page.Pagination +} + +type ListTasksOutput struct { + Tasks task.Tasks + Error error +} + +type CreateTaskInput struct { + Context context.Context //nolint:containedctx // Test only + Create *task.TaskCreate +} + +type CreateTaskOutput struct { + Task *task.Task + Error error +} + +type GetTaskInput struct { + Context context.Context //nolint:containedctx // Test only + ID string + Condition *request.Condition +} + +type GetTaskOutput struct { + Task *task.Task + Error error +} + +type UpdateTaskInput struct { + Context context.Context //nolint:containedctx // Test only + ID string + Condition *request.Condition + Update *task.TaskUpdate +} + +type UpdateTaskOutput struct { + Task *task.Task + Error error +} + +type DeleteTaskInput struct { + Context context.Context //nolint:containedctx // Test only + ID string + Condition *request.Condition +} + type Client struct { - *TaskAccessor + ListTasksInvocations int + ListTasksInputs []ListTasksInput + ListTasksOutputs []ListTasksOutput + CreateTaskInvocations int + CreateTaskInputs []CreateTaskInput + CreateTaskOutputs []CreateTaskOutput + GetTaskInvocations int + GetTaskInputs []GetTaskInput + GetTaskOutputs []GetTaskOutput + UpdateTaskInvocations int + UpdateTaskInputs []UpdateTaskInput + UpdateTaskOutputs []UpdateTaskOutput + DeleteTaskInvocations int + DeleteTaskInputs []DeleteTaskInput + DeleteTaskOutputs []error } func NewClient() *Client { - return &Client{ - TaskAccessor: NewTaskAccessor(), - } + return &Client{} +} + +func (t *Client) ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) { + t.ListTasksInvocations++ + + t.ListTasksInputs = append(t.ListTasksInputs, ListTasksInput{Context: ctx, Filter: filter, Pagination: pagination}) + + gomega.Expect(t.ListTasksOutputs).ToNot(gomega.BeEmpty()) + + output := t.ListTasksOutputs[0] + t.ListTasksOutputs = t.ListTasksOutputs[1:] + return output.Tasks, output.Error +} + +func (t *Client) CreateTask(ctx context.Context, create *task.TaskCreate) (*task.Task, error) { + t.CreateTaskInvocations++ + + t.CreateTaskInputs = append(t.CreateTaskInputs, CreateTaskInput{Context: ctx, Create: create}) + + gomega.Expect(t.CreateTaskOutputs).ToNot(gomega.BeEmpty()) + + output := t.CreateTaskOutputs[0] + t.CreateTaskOutputs = t.CreateTaskOutputs[1:] + return output.Task, output.Error +} + +func (t *Client) GetTask(ctx context.Context, id string, condition *request.Condition) (*task.Task, error) { + t.GetTaskInvocations++ + + t.GetTaskInputs = append(t.GetTaskInputs, GetTaskInput{Context: ctx, ID: id, Condition: condition}) + + gomega.Expect(t.GetTaskOutputs).ToNot(gomega.BeEmpty()) + + output := t.GetTaskOutputs[0] + t.GetTaskOutputs = t.GetTaskOutputs[1:] + return output.Task, output.Error +} + +func (t *Client) UpdateTask(ctx context.Context, id string, condition *request.Condition, update *task.TaskUpdate) (*task.Task, error) { + t.UpdateTaskInvocations++ + + t.UpdateTaskInputs = append(t.UpdateTaskInputs, UpdateTaskInput{Context: ctx, ID: id, Condition: condition, Update: update}) + + gomega.Expect(t.UpdateTaskOutputs).ToNot(gomega.BeEmpty()) + + output := t.UpdateTaskOutputs[0] + t.UpdateTaskOutputs = t.UpdateTaskOutputs[1:] + return output.Task, output.Error +} + +func (t *Client) DeleteTask(ctx context.Context, id string, condition *request.Condition) error { + t.DeleteTaskInvocations++ + + t.DeleteTaskInputs = append(t.DeleteTaskInputs, DeleteTaskInput{Context: ctx, ID: id, Condition: condition}) + + gomega.Expect(t.DeleteTaskOutputs).ToNot(gomega.BeEmpty()) + + output := t.DeleteTaskOutputs[0] + t.DeleteTaskOutputs = t.DeleteTaskOutputs[1:] + return output } -func (c *Client) Expectations() { - c.TaskAccessor.Expectations() +func (t *Client) Expectations() { + gomega.Expect(t.ListTasksOutputs).To(gomega.BeEmpty()) + gomega.Expect(t.CreateTaskOutputs).To(gomega.BeEmpty()) + gomega.Expect(t.GetTaskOutputs).To(gomega.BeEmpty()) + gomega.Expect(t.UpdateTaskOutputs).To(gomega.BeEmpty()) + gomega.Expect(t.DeleteTaskOutputs).To(gomega.BeEmpty()) } diff --git a/task/test/task.go b/task/test/task.go new file mode 100644 index 0000000000..f0b11b88b4 --- /dev/null +++ b/task/test/task.go @@ -0,0 +1,92 @@ +package test + +import ( + "time" + + errorsTest "github.com/tidepool-org/platform/errors/test" + metadataTest "github.com/tidepool-org/platform/metadata/test" + "github.com/tidepool-org/platform/pointer" + "github.com/tidepool-org/platform/task" + "github.com/tidepool-org/platform/test" +) + +func RandomID() string { + return task.NewID() +} + +func RandomName() string { + return test.RandomString() +} + +func RandomType() string { + return test.RandomString() +} + +func RandomData(options ...test.Option) map[string]any { + return metadataTest.RandomOptionalMetadataMap(options...) +} + +func RandomState() string { + return test.RandomStringFromArray(task.TaskStates()) +} + +func RandomTaskCreate(options ...test.Option) *task.TaskCreate { + now := test.Now() + availableTime := test.RandomTimeAfter(now) + return &task.TaskCreate{ + Name: test.RandomOptional(RandomName, options...), + Type: RandomType(), + Data: RandomData(options...), + AvailableTime: test.RandomOptional(test.Constant(availableTime), options...), + } +} + +func RandomTask(options ...test.Option) *task.Task { + now := test.Now() + + tsk := &task.Task{ + ID: RandomID(), + Name: test.RandomOptional(RandomName, options...), + Type: RandomType(), + Data: RandomData(options...), + State: RandomState(), + CreatedTime: test.RandomTimeBefore(now), + } + + switch tsk.State { + case task.TaskStatePending: + tsk.AvailableTime = pointer.From(test.RandomTimeAfterNow()) + tsk.ModifiedTime = test.RandomOptional(func() time.Time { return test.RandomTimeFromRange(tsk.CreatedTime, now) }, options...) + if tsk.ModifiedTime != nil { + tsk.Error = test.RandomOptionalPointer(errorsTest.RandomSerializable, options...) + tsk.RunTime = tsk.ModifiedTime + tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + } + tsk.DeadlineTime = nil + case task.TaskStateRunning: + tsk.AvailableTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, now)) + tsk.ModifiedTime = pointer.From(test.RandomTimeFromRange(*tsk.AvailableTime, now)) + if test.RandomBool() { + tsk.Error = test.RandomOptionalPointer(errorsTest.RandomSerializable, options...) + tsk.RunTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, *tsk.AvailableTime)) + tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + } + tsk.DeadlineTime = pointer.From(test.RandomTimeAfterNow()) + case task.TaskStateFailed: + tsk.AvailableTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, now)) + tsk.ModifiedTime = pointer.From(test.RandomTimeFromRange(*tsk.AvailableTime, now)) + tsk.Error = errorsTest.RandomSerializable() + tsk.RunTime = tsk.ModifiedTime + tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + tsk.DeadlineTime = nil + case task.TaskStateCompleted: + tsk.AvailableTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, now)) + tsk.ModifiedTime = pointer.From(test.RandomTimeFromRange(*tsk.AvailableTime, now)) + tsk.Error = test.RandomOptionalPointer(errorsTest.RandomSerializable, options...) + tsk.RunTime = tsk.ModifiedTime + tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + tsk.DeadlineTime = nil + } + + return tsk +} diff --git a/task/test/task_accessor.go b/task/test/task_accessor.go deleted file mode 100644 index f71b10aa9f..0000000000 --- a/task/test/task_accessor.go +++ /dev/null @@ -1,146 +0,0 @@ -package test - -import ( - "context" - - "github.com/onsi/gomega" - - "github.com/tidepool-org/platform/page" - "github.com/tidepool-org/platform/task" -) - -type ListTasksInput struct { - Context context.Context - Filter *task.TaskFilter - Pagination *page.Pagination -} - -type ListTasksOutput struct { - Tasks task.Tasks - Error error -} - -type CreateTaskInput struct { - Context context.Context - Create *task.TaskCreate -} - -type CreateTaskOutput struct { - Task *task.Task - Error error -} - -type GetTaskInput struct { - Context context.Context - ID string -} - -type GetTaskOutput struct { - Task *task.Task - Error error -} - -type UpdateTaskInput struct { - Context context.Context - ID string - Update *task.TaskUpdate -} - -type UpdateTaskOutput struct { - Task *task.Task - Error error -} - -type DeleteTaskInput struct { - Context context.Context - ID string -} - -type TaskAccessor struct { - ListTasksInvocations int - ListTasksInputs []ListTasksInput - ListTasksOutputs []ListTasksOutput - CreateTaskInvocations int - CreateTaskInputs []CreateTaskInput - CreateTaskOutputs []CreateTaskOutput - GetTaskInvocations int - GetTaskInputs []GetTaskInput - GetTaskOutputs []GetTaskOutput - UpdateTaskInvocations int - UpdateTaskInputs []UpdateTaskInput - UpdateTaskOutputs []UpdateTaskOutput - DeleteTaskInvocations int - DeleteTaskInputs []DeleteTaskInput - DeleteTaskOutputs []error -} - -func NewTaskAccessor() *TaskAccessor { - return &TaskAccessor{} -} - -func (t *TaskAccessor) ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) { - t.ListTasksInvocations++ - - t.ListTasksInputs = append(t.ListTasksInputs, ListTasksInput{Context: ctx, Filter: filter, Pagination: pagination}) - - gomega.Expect(t.ListTasksOutputs).ToNot(gomega.BeEmpty()) - - output := t.ListTasksOutputs[0] - t.ListTasksOutputs = t.ListTasksOutputs[1:] - return output.Tasks, output.Error -} - -func (t *TaskAccessor) CreateTask(ctx context.Context, create *task.TaskCreate) (*task.Task, error) { - t.CreateTaskInvocations++ - - t.CreateTaskInputs = append(t.CreateTaskInputs, CreateTaskInput{Context: ctx, Create: create}) - - gomega.Expect(t.CreateTaskOutputs).ToNot(gomega.BeEmpty()) - - output := t.CreateTaskOutputs[0] - t.CreateTaskOutputs = t.CreateTaskOutputs[1:] - return output.Task, output.Error -} - -func (t *TaskAccessor) GetTask(ctx context.Context, id string) (*task.Task, error) { - t.GetTaskInvocations++ - - t.GetTaskInputs = append(t.GetTaskInputs, GetTaskInput{Context: ctx, ID: id}) - - gomega.Expect(t.GetTaskOutputs).ToNot(gomega.BeEmpty()) - - output := t.GetTaskOutputs[0] - t.GetTaskOutputs = t.GetTaskOutputs[1:] - return output.Task, output.Error -} - -func (t *TaskAccessor) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { - t.UpdateTaskInvocations++ - - t.UpdateTaskInputs = append(t.UpdateTaskInputs, UpdateTaskInput{Context: ctx, ID: id, Update: update}) - - gomega.Expect(t.UpdateTaskOutputs).ToNot(gomega.BeEmpty()) - - output := t.UpdateTaskOutputs[0] - t.UpdateTaskOutputs = t.UpdateTaskOutputs[1:] - return output.Task, output.Error -} - -func (t *TaskAccessor) DeleteTask(ctx context.Context, id string) error { - t.DeleteTaskInvocations++ - - t.DeleteTaskInputs = append(t.DeleteTaskInputs, DeleteTaskInput{Context: ctx, ID: id}) - - gomega.Expect(t.DeleteTaskOutputs).ToNot(gomega.BeEmpty()) - - output := t.DeleteTaskOutputs[0] - t.DeleteTaskOutputs = t.DeleteTaskOutputs[1:] - return output -} - -func (t *TaskAccessor) Expectations() { - gomega.Expect(t.ListTasksOutputs).To(gomega.BeEmpty()) - gomega.Expect(t.CreateTaskOutputs).To(gomega.BeEmpty()) - gomega.Expect(t.GetTaskOutputs).To(gomega.BeEmpty()) - gomega.Expect(t.UpdateTaskOutputs).To(gomega.BeEmpty()) -} diff --git a/task/test/task_mocks.go b/task/test/task_mocks.go index c68f2e35b0..d4401f7706 100644 --- a/task/test/task_mocks.go +++ b/task/test/task_mocks.go @@ -16,6 +16,7 @@ import ( gomock "go.uber.org/mock/gomock" page "github.com/tidepool-org/platform/page" + request "github.com/tidepool-org/platform/request" task "github.com/tidepool-org/platform/task" ) @@ -83,17 +84,17 @@ func (c *MockClientCreateTaskCall) DoAndReturn(f func(context.Context, *task.Tas } // DeleteTask mocks base method. -func (m *MockClient) DeleteTask(ctx context.Context, id string) error { +func (m *MockClient) DeleteTask(ctx context.Context, id string, condition *request.Condition) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteTask", ctx, id) + ret := m.ctrl.Call(m, "DeleteTask", ctx, id, condition) ret0, _ := ret[0].(error) return ret0 } // DeleteTask indicates an expected call of DeleteTask. -func (mr *MockClientMockRecorder) DeleteTask(ctx, id any) *MockClientDeleteTaskCall { +func (mr *MockClientMockRecorder) DeleteTask(ctx, id, condition any) *MockClientDeleteTaskCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTask", reflect.TypeOf((*MockClient)(nil).DeleteTask), ctx, id) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTask", reflect.TypeOf((*MockClient)(nil).DeleteTask), ctx, id, condition) return &MockClientDeleteTaskCall{Call: call} } @@ -109,30 +110,30 @@ func (c *MockClientDeleteTaskCall) Return(arg0 error) *MockClientDeleteTaskCall } // Do rewrite *gomock.Call.Do -func (c *MockClientDeleteTaskCall) Do(f func(context.Context, string) error) *MockClientDeleteTaskCall { +func (c *MockClientDeleteTaskCall) Do(f func(context.Context, string, *request.Condition) error) *MockClientDeleteTaskCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientDeleteTaskCall) DoAndReturn(f func(context.Context, string) error) *MockClientDeleteTaskCall { +func (c *MockClientDeleteTaskCall) DoAndReturn(f func(context.Context, string, *request.Condition) error) *MockClientDeleteTaskCall { c.Call = c.Call.DoAndReturn(f) return c } // GetTask mocks base method. -func (m *MockClient) GetTask(ctx context.Context, id string) (*task.Task, error) { +func (m *MockClient) GetTask(ctx context.Context, id string, condition *request.Condition) (*task.Task, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTask", ctx, id) + ret := m.ctrl.Call(m, "GetTask", ctx, id, condition) ret0, _ := ret[0].(*task.Task) ret1, _ := ret[1].(error) return ret0, ret1 } // GetTask indicates an expected call of GetTask. -func (mr *MockClientMockRecorder) GetTask(ctx, id any) *MockClientGetTaskCall { +func (mr *MockClientMockRecorder) GetTask(ctx, id, condition any) *MockClientGetTaskCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTask", reflect.TypeOf((*MockClient)(nil).GetTask), ctx, id) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTask", reflect.TypeOf((*MockClient)(nil).GetTask), ctx, id, condition) return &MockClientGetTaskCall{Call: call} } @@ -148,13 +149,13 @@ func (c *MockClientGetTaskCall) Return(arg0 *task.Task, arg1 error) *MockClientG } // Do rewrite *gomock.Call.Do -func (c *MockClientGetTaskCall) Do(f func(context.Context, string) (*task.Task, error)) *MockClientGetTaskCall { +func (c *MockClientGetTaskCall) Do(f func(context.Context, string, *request.Condition) (*task.Task, error)) *MockClientGetTaskCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientGetTaskCall) DoAndReturn(f func(context.Context, string) (*task.Task, error)) *MockClientGetTaskCall { +func (c *MockClientGetTaskCall) DoAndReturn(f func(context.Context, string, *request.Condition) (*task.Task, error)) *MockClientGetTaskCall { c.Call = c.Call.DoAndReturn(f) return c } @@ -199,18 +200,18 @@ func (c *MockClientListTasksCall) DoAndReturn(f func(context.Context, *task.Task } // UpdateTask mocks base method. -func (m *MockClient) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { +func (m *MockClient) UpdateTask(ctx context.Context, id string, condition *request.Condition, update *task.TaskUpdate) (*task.Task, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateTask", ctx, id, update) + ret := m.ctrl.Call(m, "UpdateTask", ctx, id, condition, update) ret0, _ := ret[0].(*task.Task) ret1, _ := ret[1].(error) return ret0, ret1 } // UpdateTask indicates an expected call of UpdateTask. -func (mr *MockClientMockRecorder) UpdateTask(ctx, id, update any) *MockClientUpdateTaskCall { +func (mr *MockClientMockRecorder) UpdateTask(ctx, id, condition, update any) *MockClientUpdateTaskCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTask", reflect.TypeOf((*MockClient)(nil).UpdateTask), ctx, id, update) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTask", reflect.TypeOf((*MockClient)(nil).UpdateTask), ctx, id, condition, update) return &MockClientUpdateTaskCall{Call: call} } @@ -226,231 +227,13 @@ func (c *MockClientUpdateTaskCall) Return(arg0 *task.Task, arg1 error) *MockClie } // Do rewrite *gomock.Call.Do -func (c *MockClientUpdateTaskCall) Do(f func(context.Context, string, *task.TaskUpdate) (*task.Task, error)) *MockClientUpdateTaskCall { +func (c *MockClientUpdateTaskCall) Do(f func(context.Context, string, *request.Condition, *task.TaskUpdate) (*task.Task, error)) *MockClientUpdateTaskCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientUpdateTaskCall) DoAndReturn(f func(context.Context, string, *task.TaskUpdate) (*task.Task, error)) *MockClientUpdateTaskCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// MockTaskAccessor is a mock of TaskAccessor interface. -type MockTaskAccessor struct { - ctrl *gomock.Controller - recorder *MockTaskAccessorMockRecorder - isgomock struct{} -} - -// MockTaskAccessorMockRecorder is the mock recorder for MockTaskAccessor. -type MockTaskAccessorMockRecorder struct { - mock *MockTaskAccessor -} - -// NewMockTaskAccessor creates a new mock instance. -func NewMockTaskAccessor(ctrl *gomock.Controller) *MockTaskAccessor { - mock := &MockTaskAccessor{ctrl: ctrl} - mock.recorder = &MockTaskAccessorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockTaskAccessor) EXPECT() *MockTaskAccessorMockRecorder { - return m.recorder -} - -// CreateTask mocks base method. -func (m *MockTaskAccessor) CreateTask(ctx context.Context, create *task.TaskCreate) (*task.Task, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTask", ctx, create) - ret0, _ := ret[0].(*task.Task) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateTask indicates an expected call of CreateTask. -func (mr *MockTaskAccessorMockRecorder) CreateTask(ctx, create any) *MockTaskAccessorCreateTaskCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTask", reflect.TypeOf((*MockTaskAccessor)(nil).CreateTask), ctx, create) - return &MockTaskAccessorCreateTaskCall{Call: call} -} - -// MockTaskAccessorCreateTaskCall wrap *gomock.Call -type MockTaskAccessorCreateTaskCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorCreateTaskCall) Return(arg0 *task.Task, arg1 error) *MockTaskAccessorCreateTaskCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorCreateTaskCall) Do(f func(context.Context, *task.TaskCreate) (*task.Task, error)) *MockTaskAccessorCreateTaskCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorCreateTaskCall) DoAndReturn(f func(context.Context, *task.TaskCreate) (*task.Task, error)) *MockTaskAccessorCreateTaskCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// DeleteTask mocks base method. -func (m *MockTaskAccessor) DeleteTask(ctx context.Context, id string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteTask", ctx, id) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteTask indicates an expected call of DeleteTask. -func (mr *MockTaskAccessorMockRecorder) DeleteTask(ctx, id any) *MockTaskAccessorDeleteTaskCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTask", reflect.TypeOf((*MockTaskAccessor)(nil).DeleteTask), ctx, id) - return &MockTaskAccessorDeleteTaskCall{Call: call} -} - -// MockTaskAccessorDeleteTaskCall wrap *gomock.Call -type MockTaskAccessorDeleteTaskCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorDeleteTaskCall) Return(arg0 error) *MockTaskAccessorDeleteTaskCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorDeleteTaskCall) Do(f func(context.Context, string) error) *MockTaskAccessorDeleteTaskCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorDeleteTaskCall) DoAndReturn(f func(context.Context, string) error) *MockTaskAccessorDeleteTaskCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// GetTask mocks base method. -func (m *MockTaskAccessor) GetTask(ctx context.Context, id string) (*task.Task, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTask", ctx, id) - ret0, _ := ret[0].(*task.Task) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTask indicates an expected call of GetTask. -func (mr *MockTaskAccessorMockRecorder) GetTask(ctx, id any) *MockTaskAccessorGetTaskCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTask", reflect.TypeOf((*MockTaskAccessor)(nil).GetTask), ctx, id) - return &MockTaskAccessorGetTaskCall{Call: call} -} - -// MockTaskAccessorGetTaskCall wrap *gomock.Call -type MockTaskAccessorGetTaskCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorGetTaskCall) Return(arg0 *task.Task, arg1 error) *MockTaskAccessorGetTaskCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorGetTaskCall) Do(f func(context.Context, string) (*task.Task, error)) *MockTaskAccessorGetTaskCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorGetTaskCall) DoAndReturn(f func(context.Context, string) (*task.Task, error)) *MockTaskAccessorGetTaskCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// ListTasks mocks base method. -func (m *MockTaskAccessor) ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListTasks", ctx, filter, pagination) - ret0, _ := ret[0].(task.Tasks) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ListTasks indicates an expected call of ListTasks. -func (mr *MockTaskAccessorMockRecorder) ListTasks(ctx, filter, pagination any) *MockTaskAccessorListTasksCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTasks", reflect.TypeOf((*MockTaskAccessor)(nil).ListTasks), ctx, filter, pagination) - return &MockTaskAccessorListTasksCall{Call: call} -} - -// MockTaskAccessorListTasksCall wrap *gomock.Call -type MockTaskAccessorListTasksCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorListTasksCall) Return(arg0 task.Tasks, arg1 error) *MockTaskAccessorListTasksCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorListTasksCall) Do(f func(context.Context, *task.TaskFilter, *page.Pagination) (task.Tasks, error)) *MockTaskAccessorListTasksCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorListTasksCall) DoAndReturn(f func(context.Context, *task.TaskFilter, *page.Pagination) (task.Tasks, error)) *MockTaskAccessorListTasksCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// UpdateTask mocks base method. -func (m *MockTaskAccessor) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateTask", ctx, id, update) - ret0, _ := ret[0].(*task.Task) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// UpdateTask indicates an expected call of UpdateTask. -func (mr *MockTaskAccessorMockRecorder) UpdateTask(ctx, id, update any) *MockTaskAccessorUpdateTaskCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTask", reflect.TypeOf((*MockTaskAccessor)(nil).UpdateTask), ctx, id, update) - return &MockTaskAccessorUpdateTaskCall{Call: call} -} - -// MockTaskAccessorUpdateTaskCall wrap *gomock.Call -type MockTaskAccessorUpdateTaskCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorUpdateTaskCall) Return(arg0 *task.Task, arg1 error) *MockTaskAccessorUpdateTaskCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorUpdateTaskCall) Do(f func(context.Context, string, *task.TaskUpdate) (*task.Task, error)) *MockTaskAccessorUpdateTaskCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorUpdateTaskCall) DoAndReturn(f func(context.Context, string, *task.TaskUpdate) (*task.Task, error)) *MockTaskAccessorUpdateTaskCall { +func (c *MockClientUpdateTaskCall) DoAndReturn(f func(context.Context, string, *request.Condition, *task.TaskUpdate) (*task.Task, error)) *MockClientUpdateTaskCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/test/http/http.go b/test/http/http.go index 0603530ef2..303cd41f10 100644 --- a/test/http/http.go +++ b/test/http/http.go @@ -248,3 +248,18 @@ func (r *ResponseWriter) Flush() { } r.ResponseRecorder.Flush() } + +type RoundTripper struct { + Request *http.Request + Response *http.Response + Error error +} + +func NewRoundTripper() *RoundTripper { + return &RoundTripper{} +} + +func (t *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + t.Request = req + return t.Response, t.Error +} diff --git a/test/test.go b/test/test.go index 652a2c7180..59e0c88612 100644 --- a/test/test.go +++ b/test/test.go @@ -67,6 +67,6 @@ func (m *MockMatcher) String() string { } func Must[T any](value T, err error) T { - gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(err).WithOffset(1).ToNot(gomega.HaveOccurred()) return value } diff --git a/test/time.go b/test/time.go index c3070cf4d6..57cc1ab9ba 100644 --- a/test/time.go +++ b/test/time.go @@ -9,20 +9,28 @@ import ( gomegaTypes "github.com/onsi/gomega/types" ) +func Now() time.Time { + return timeInNormalizedLocation(time.Now().Truncate(time.Millisecond).UTC()) +} + +func NowStable() time.Time { + return nowStable +} + func PastFarTime() time.Time { - return now.AddDate(-30, 0, 0) + return NowStable().AddDate(-30, 0, 0) } func PastNearTime() time.Time { - return now.AddDate(0, -1, 0) + return NowStable().AddDate(0, -1, 0) } func FutureNearTime() time.Time { - return now.AddDate(0, 1, 0) + return NowStable().AddDate(0, 1, 0) } func FutureFarTime() time.Time { - return now.AddDate(30, 0, 0) + return NowStable().AddDate(30, 0, 0) } func MustTime(value time.Time, err error) time.Time { @@ -33,11 +41,11 @@ func MustTime(value time.Time, err error) time.Time { } func RandomTimeBeforeNow() time.Time { - return RandomTimeBefore(now) + return RandomTimeBefore(Now()) } func RandomTimeAfterNow() time.Time { - return RandomTimeAfter(now) + return RandomTimeAfter(Now()) } func RandomTimeBefore(value time.Time) time.Time { @@ -73,11 +81,11 @@ func RandomTimeFromRange(minimum time.Time, maximum time.Time) time.Time { } func RandomTimeMaximum() time.Time { - return now.Add(RandomDurationMaximum()).Truncate(time.Millisecond) + return NowStable().Add(RandomDurationMaximum()).Truncate(time.Millisecond) } func RandomTimeMinimum() time.Time { - return now.Add(RandomDurationMinimum()).Truncate(time.Millisecond) + return NowStable().Add(RandomDurationMinimum()).Truncate(time.Millisecond) } func NewObjectFromTime(value time.Time, objectFormat ObjectFormat) interface{} { @@ -98,7 +106,7 @@ func PinnedTime(value time.Time) time.Time { } else if value.After(RandomTimeMaximum()) { return RandomTimeMaximum() } else { - return normalizeLocation(value.Truncate(time.Millisecond)) + return timeInNormalizedLocation(value.Truncate(time.Millisecond)) } } @@ -109,11 +117,11 @@ func MatchTime(datum *time.Time) gomegaTypes.GomegaMatcher { return gomegaGstruct.PointTo(gomega.BeTemporally("==", *datum)) } -func normalizeLocation(value time.Time) time.Time { +func timeInNormalizedLocation(value time.Time) time.Time { if value.Location() == time.Local && time.Local.String() == "UTC" { value = value.In(time.UTC) } return value } -var now = normalizeLocation(time.Now().Truncate(time.Millisecond).UTC()) +var nowStable = Now()