From 60a02d9a84de4de1135bd1614cfe6188b53e6353 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Fri, 17 Jul 2026 12:35:28 -0700 Subject: [PATCH 01/20] Refactor task queue to resolve task-related failures - Refactor task queue to resolve task-related failures - Remove priority and expiration time from task as unused - Add task timeout watchdog - Add name to queue for debugging purposes - Add revision condition to task Client and Repository API - Set default available time for a new task to now - Update runner deadline to return duration, not time - Add Prometheus metrics to Dexcom client - Add Prometheus metrics to task queue - Add serialization mutex to test logger - Require OAuth client to specify configured http client - Capture response metrics on all outgoing OAuth client requests - Add Prometheus helpers for outgoing clients - Add common CloseCursor function to properly close Mongo cursors and log errors - Add and update tests - Fix typos - https://tidepool.atlassian.net/browse/BACK-4523 --- client/prometheus.go | 147 ++++ client/prometheus_test.go | 281 ++++++ client/round_tripper.go | 32 + client/round_tripper_test.go | 113 +++ data/service/service/standard.go | 2 +- data/store/mongo/mongo_datum.go | 2 +- dexcom/client/client.go | 17 +- dexcom/device.go | 2 +- dexcom/fetch/runner.go | 4 +- dexcom/fetch/runner_test.go | 2 +- dexcom/provider/provider.go | 65 +- dexcom/provider/provider_test.go | 105 +++ ehr/reconcile/runner.go | 8 +- ehr/reconcile/runner_test.go | 2 +- ehr/reconcile/task.go | 7 +- ehr/reconcile/task_test.go | 3 - ehr/sync/runner.go | 4 +- ehr/sync/task.go | 5 +- ehr/sync/task_test.go | 1 - go.mod | 2 +- log/test/serializer.go | 12 +- oauth/provider/client/client.go | 10 +- oauth/provider/provider.go | 10 +- oura/client/client.go | 33 +- oura/client/client_test.go | 62 +- oura/oura.go | 9 + oura/oura_test.go | 28 + oura/provider/provider.go | 31 +- oura/provider/provider_test.go | 26 + pointer/default.go | 7 + pointer/default_test.go | 17 +- prometheus/test/prometheus.go | 34 + request/condition.go | 6 + request/inspector.go | 54 -- .../tools/dexcom_analyze/dexcom_analyze.go | 18 - store/structured/condition.go | 6 + store/structured/mongo/config.go | 2 +- store/structured/mongo/result.go | 21 + store/structured/mongo/result_test.go | 84 +- summary/task/migrationrunner.go | 10 +- summary/task/migrationrunner_test.go | 2 - summary/task/updaterunner.go | 10 +- summary/task/updaterunner_test.go | 2 - task/client/client.go | 27 +- task/queue/multi.go | 73 +- task/queue/multi_test.go | 119 +-- task/queue/queue.go | 716 ++++++++++----- task/queue/queue_internal_test.go | 51 -- task/queue/queue_test.go | 816 +++++++++++++++++- task/queue/test/runner.go | 225 ++++- task/service/api/v1/v1.go | 24 +- task/service/service/client.go | 14 +- task/service/service/service.go | 23 +- task/store/mongo/mongo.go | 468 ++++++---- task/store/mongo/mongo_test.go | 229 ++--- task/store/store.go | 15 +- task/task.go | 142 +-- task/task_test.go | 83 ++ task/test/client.go | 147 +++- task/test/task.go | 92 ++ task/test/task_accessor.go | 146 ---- task/test/task_mocks.go | 255 +----- test/http/http.go | 15 + test/time.go | 4 + twiist/provider/provider.go | 17 +- 65 files changed, 3622 insertions(+), 1377 deletions(-) create mode 100644 client/prometheus.go create mode 100644 client/prometheus_test.go create mode 100644 client/round_tripper.go create mode 100644 client/round_tripper_test.go create mode 100644 prometheus/test/prometheus.go delete mode 100644 task/queue/queue_internal_test.go create mode 100644 task/test/task.go delete mode 100644 task/test/task_accessor.go diff --git a/client/prometheus.go b/client/prometheus.go new file mode 100644 index 0000000000..45d579d35b --- /dev/null +++ b/client/prometheus.go @@ -0,0 +1,147 @@ +package client + +import ( + "fmt" + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + "github.com/tidepool-org/platform/pointer" +) + +const PathPatternAny = "/" + +var ( + DurationBucketsDefault = []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60} + + PrometheusLabelNameMethod = "method" + PrometheusLabelNamePath = "path" + PrometheusLabelNameStatus = "status" + + PrometheusLabelValueError = "ERROR" +) + +func PrometheusLabelNames() []string { + return []string{ + PrometheusLabelNameMethod, + PrometheusLabelNamePath, + PrometheusLabelNameStatus, + } +} + +// Where there are numerous discrete paths possible, then must simplify via patterns to prevent overwhelming Prometheus. +// If no patterns are specified, then all paths are recorded as-is. If one or more patterns are specified, but a path +// does not match one of the patterns, then the path is NOT captured by Prometheus. If you wish to match "all other" paths +// and records those paths as-is, then add the pattern PathPatternAny at the end of your patterns. +// +// Uses standard Go HTTP pattern matching. See https://go.dev/src/net/http/pattern.go. +// +// For example: /one/{id}, /two/{id} +type PrometheusRequestURLPathMatcher struct { + pathPatternMux *http.ServeMux +} + +func NewPrometheusRequestURLPathPatternMatcher(pathPatterns ...string) *PrometheusRequestURLPathMatcher { + var pathPatternMux *http.ServeMux + + if len(pathPatterns) > 0 { + pathPatternMux = http.NewServeMux() + for _, pattern := range pathPatterns { + pathPatternMux.HandleFunc(pattern, func(http.ResponseWriter, *http.Request) {}) + } + } + + return &PrometheusRequestURLPathMatcher{ + pathPatternMux: pathPatternMux, + } +} + +func (p *PrometheusRequestURLPathMatcher) MatchPath(req *http.Request) *string { + path := req.URL.Path + if p.pathPatternMux != nil { + if _, pattern := p.pathPatternMux.Handler(req); pattern == "" { + return nil + } else if pattern != PathPatternAny { + path = pattern + } + } + return &path +} + +type PrometheusRequestRoundTripper struct { + *RoundTripper + *PrometheusRequestURLPathMatcher +} + +func NewPrometheusRequestRoundTripper(pathPatterns ...string) *PrometheusRequestRoundTripper { + return &PrometheusRequestRoundTripper{ + RoundTripper: NewRoundTripper(nil), + PrometheusRequestURLPathMatcher: NewPrometheusRequestURLPathPatternMatcher(pathPatterns...), + } +} + +type PrometheusRequestMetricsRoundTripper struct { + *PrometheusRequestRoundTripper + requestCountCounterVec *prometheus.CounterVec + requestDurationHistogramVec *prometheus.HistogramVec +} + +func NewPrometheusRequestMetricsRoundTripper(name string, help string) *PrometheusRequestMetricsRoundTripper { + return NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name, help, nil, nil) +} + +func NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name string, help string, pathPatterns []string, durationBuckets []float64) *PrometheusRequestMetricsRoundTripper { + return &PrometheusRequestMetricsRoundTripper{ + PrometheusRequestRoundTripper: NewPrometheusRequestRoundTripper(pathPatterns...), + requestCountCounterVec: promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: fmt.Sprintf("%s_request_count", name), + Help: fmt.Sprintf("%s request count", help), + }, + PrometheusLabelNames(), + ), + requestDurationHistogramVec: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: fmt.Sprintf("%s_request_duration_seconds", name), + Help: fmt.Sprintf("%s request duration (seconds)", help), + Buckets: pointer.DefaultArray(durationBuckets, DurationBucketsDefault), + }, + PrometheusLabelNames(), + ), + } +} + +func (p *PrometheusRequestMetricsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + start := time.Now() + res, err := p.ResolvedRoundTripper().RoundTrip(req) + duration := time.Since(start) + + if labels := p.Labels(req, res); labels != nil { + p.requestCountCounterVec.With(*labels).Inc() + p.requestDurationHistogramVec.With(*labels).Observe(duration.Seconds()) + } + + return res, err +} + +func (p *PrometheusRequestMetricsRoundTripper) Labels(req *http.Request, res *http.Response) *prometheus.Labels { + path := p.MatchPath(req) + if path == nil { + return nil + } + + labels := prometheus.Labels{ + PrometheusLabelNameMethod: req.Method, + PrometheusLabelNamePath: *path, + } + if res != nil { + labels[PrometheusLabelNameStatus] = strconv.Itoa(res.StatusCode) + } else { + labels[PrometheusLabelNameStatus] = PrometheusLabelValueError + } + + return &labels +} diff --git a/client/prometheus_test.go b/client/prometheus_test.go new file mode 100644 index 0000000000..af75a91f25 --- /dev/null +++ b/client/prometheus_test.go @@ -0,0 +1,281 @@ +package client_test + +import ( + "net/http" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/tidepool-org/platform/client" + errorsTest "github.com/tidepool-org/platform/errors/test" + prometheusTest "github.com/tidepool-org/platform/prometheus/test" + "github.com/tidepool-org/platform/test" + testHttp "github.com/tidepool-org/platform/test/http" +) + +var _ = Describe("Prometheus", func() { + It("PathPatternAny is expected", func() { + Expect(client.PathPatternAny).To(Equal("/")) + }) + + It("DurationBucketsDefault is expected", func() { + Expect(client.DurationBucketsDefault).To(Equal([]float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 20, 30, 60})) + }) + + It("PrometheusLabelNameMethod is expected", func() { + Expect(client.PrometheusLabelNameMethod).To(Equal("method")) + }) + + It("PrometheusLabelNamePath is expected", func() { + Expect(client.PrometheusLabelNamePath).To(Equal("path")) + }) + + It("PrometheusLabelNameStatus is expected", func() { + Expect(client.PrometheusLabelNameStatus).To(Equal("status")) + }) + + It("PrometheusLabelValueError is expected", func() { + Expect(client.PrometheusLabelValueError).To(Equal("ERROR")) + }) + + Context("PrometheusLabelNames", func() { + It("returns the expected label names", func() { + Expect(client.PrometheusLabelNames()).To(Equal([]string{ + client.PrometheusLabelNameMethod, + client.PrometheusLabelNamePath, + client.PrometheusLabelNameStatus, + })) + }) + }) + + Context("PrometheusRequestURLPathPatternMatcher", func() { + Context("NewPrometheusRequestURLPathPatternMatcher", func() { + It("returns successfully with no path patterns", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher() + Expect(matcher).ToNot(BeNil()) + }) + It("returns successfully with path patterns", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher("/one/{id}", client.PathPatternAny) + Expect(matcher).ToNot(BeNil()) + }) + }) + + Context("MatchPath", func() { + It("returns the request path unchanged when there are no path patterns", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher() + request := testHttp.NewRequest() + Expect(matcher.MatchPath(request)).To(PointTo(Equal(request.URL.Path))) + }) + + It("returns nil when the path does not match any path pattern", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher("/one/{id}") + request := test.Must(http.NewRequest(http.MethodGet, "http://example.com/two/456", nil)) + Expect(matcher.MatchPath(request)).To(BeNil()) + }) + + It("returns the matched path pattern when the path matches a specific pattern", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher("/one/{id}", client.PathPatternAny) + request := test.Must(http.NewRequest(http.MethodGet, "http://example.com/one/123", nil)) + Expect(matcher.MatchPath(request)).To(PointTo(Equal("/one/{id}"))) + }) + + It("returns the request path unchanged when the path matches the any path pattern", func() { + matcher := client.NewPrometheusRequestURLPathPatternMatcher("/one/{id}", client.PathPatternAny) + request := test.Must(http.NewRequest(http.MethodGet, "http://example.com/two/456", nil)) + Expect(matcher.MatchPath(request)).To(PointTo(Equal("/two/456"))) + }) + }) + }) + + Context("PrometheusRequestRoundTripper", func() { + Context("NewPrometheusRequestRoundTripper", func() { + It("returns successfully", func() { + roundTripper := client.NewPrometheusRequestRoundTripper() + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.RoundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestURLPathMatcher).ToNot(BeNil()) + }) + + It("returns successfully with path patterns", func() { + roundTripper := client.NewPrometheusRequestRoundTripper("/one/{id}", client.PathPatternAny) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.RoundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestURLPathMatcher).ToNot(BeNil()) + }) + }) + }) + + Context("PrometheusRequestMetricsRoundTripper", func() { + Context("NewPrometheusRequestMetricsRoundTripper", func() { + It("returns successfully", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(prometheusTest.RandomMetricName(), prometheusTest.RandomMetricHelp()) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestRoundTripper).ToNot(BeNil()) + }) + }) + + Context("NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets", func() { + It("returns successfully", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(prometheusTest.RandomMetricName(), prometheusTest.RandomMetricHelp(), []string{"/one/{id}"}, []float64{1, 2, 3}) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestRoundTripper).ToNot(BeNil()) + }) + }) + + Context("with a metrics round tripper", func() { + var testRoundTripper *testHttp.RoundTripper + var name string + var help string + var request *http.Request + + BeforeEach(func() { + testRoundTripper = testHttp.NewRoundTripper() + name = prometheusTest.RandomMetricName() + help = prometheusTest.RandomMetricHelp() + request = testHttp.NewRequest() + }) + + Context("RoundTrip", func() { + It("returns the response from the resolved round tripper", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + + result := test.Must(roundTripper.RoundTrip(request)) + Expect(result).To(BeIdenticalTo(testRoundTripper.Response)) + Expect(testRoundTripper.Request).To(BeIdenticalTo(request)) + }) + + It("returns the error from the resolved round tripper", func() { + testErr := errorsTest.RandomError() + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Error = testErr + + result, err := roundTripper.RoundTrip(request) + Expect(err).To(Equal(testErr)) + Expect(result).To(BeNil()) + }) + + It("records request count and duration metrics for a successful request", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + roundTripper.WithRoundTripper(testRoundTripper) + statusCode := testHttp.NewStatusCode() + testRoundTripper.Response = &http.Response{StatusCode: statusCode} + + _ = test.Must(roundTripper.RoundTrip(request)) + + expectedLabels := map[string]string{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: strconv.Itoa(statusCode), + } + + countFamily := prometheusTest.MetricFamilyFromName(name + "_request_count") + Expect(countFamily).ToNot(BeNil()) + Expect(countFamily.GetMetric()).To(HaveLen(1)) + countMetric := countFamily.GetMetric()[0] + Expect(countMetric.GetCounter().GetValue()).To(Equal(float64(1))) + Expect(prometheusTest.LabelPairsToMap(countMetric.GetLabel())).To(Equal(expectedLabels)) + + durationFamily := prometheusTest.MetricFamilyFromName(name + "_request_duration_seconds") + Expect(durationFamily).ToNot(BeNil()) + Expect(durationFamily.GetMetric()).To(HaveLen(1)) + durationMetric := durationFamily.GetMetric()[0] + Expect(durationMetric.GetHistogram().GetSampleCount()).To(Equal(uint64(1))) + Expect(prometheusTest.LabelPairsToMap(durationMetric.GetLabel())).To(Equal(expectedLabels)) + }) + + It("records request count and duration metrics with an ERROR status for a failed request", func() { + testErr := errorsTest.RandomError() + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Error = testErr + + _, err := roundTripper.RoundTrip(request) + Expect(err).To(Equal(testErr)) + + expectedLabels := map[string]string{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: client.PrometheusLabelValueError, + } + + countFamily := prometheusTest.MetricFamilyFromName(name + "_request_count") + Expect(countFamily).ToNot(BeNil()) + Expect(countFamily.GetMetric()).To(HaveLen(1)) + Expect(prometheusTest.LabelPairsToMap(countFamily.GetMetric()[0].GetLabel())).To(Equal(expectedLabels)) + + durationFamily := prometheusTest.MetricFamilyFromName(name + "_request_duration_seconds") + Expect(durationFamily).ToNot(BeNil()) + Expect(durationFamily.GetMetric()).To(HaveLen(1)) + Expect(prometheusTest.LabelPairsToMap(durationFamily.GetMetric()[0].GetLabel())).To(Equal(expectedLabels)) + }) + + It("does not record metrics when the path does not match a path pattern", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name, help, []string{"/one/{id}"}, nil) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + request = test.Must(http.NewRequest(http.MethodGet, "http://example.com/two/456", nil)) + + _ = test.Must(roundTripper.RoundTrip(request)) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_count")).To(BeNil()) + Expect(prometheusTest.MetricFamilyFromName(name + "_request_duration_seconds")).To(BeNil()) + }) + + It("uses the custom duration buckets when specified", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name, help, nil, []float64{1, 2, 3}) + roundTripper.WithRoundTripper(testRoundTripper) + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + + _ = test.Must(roundTripper.RoundTrip(request)) + + durationFamily := prometheusTest.MetricFamilyFromName(name + "_request_duration_seconds") + Expect(durationFamily).ToNot(BeNil()) + Expect(durationFamily.GetMetric()).To(HaveLen(1)) + buckets := durationFamily.GetMetric()[0].GetHistogram().GetBucket() + Expect(buckets).To(HaveLen(3)) + Expect(buckets[0].GetUpperBound()).To(Equal(1.0)) + Expect(buckets[1].GetUpperBound()).To(Equal(2.0)) + Expect(buckets[2].GetUpperBound()).To(Equal(3.0)) + }) + }) + + Context("Labels", func() { + It("returns labels with the response status code", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + statusCode := testHttp.NewStatusCode() + response := &http.Response{StatusCode: statusCode} + + Expect(roundTripper.Labels(request, response)).To(PointTo(Equal(prometheus.Labels{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: strconv.Itoa(statusCode), + }))) + }) + + It("returns labels with an ERROR status when the response is nil", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripper(name, help) + + Expect(roundTripper.Labels(request, nil)).To(PointTo(Equal(prometheus.Labels{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: client.PrometheusLabelValueError, + }))) + }) + + It("returns nil when the path does not match a path pattern", func() { + roundTripper := client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(name, help, []string{"/one/{id}"}, nil) + request := test.Must(http.NewRequest(http.MethodGet, "http://example.com/two/456", nil)) + + Expect(roundTripper.Labels(request, nil)).To(BeNil()) + }) + }) + }) + }) +}) diff --git a/client/round_tripper.go b/client/round_tripper.go new file mode 100644 index 0000000000..4fd484b3d2 --- /dev/null +++ b/client/round_tripper.go @@ -0,0 +1,32 @@ +package client + +import "net/http" + +type RoundTripper struct { + roundTripper http.RoundTripper +} + +func NewRoundTripper(roundTripper http.RoundTripper) *RoundTripper { + return &RoundTripper{ + roundTripper: roundTripper, + } +} + +func (p *RoundTripper) ResolvedRoundTripper() http.RoundTripper { + roundTripper := p.roundTripper + if roundTripper == nil { + roundTripper = http.DefaultClient.Transport + if roundTripper == nil { + roundTripper = http.DefaultTransport + } + } + return roundTripper +} + +func (p *RoundTripper) WithRoundTripper(roundTripper http.RoundTripper) { + p.roundTripper = roundTripper +} + +func (p *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return p.ResolvedRoundTripper().RoundTrip(req) +} diff --git a/client/round_tripper_test.go b/client/round_tripper_test.go new file mode 100644 index 0000000000..0cef7f2deb --- /dev/null +++ b/client/round_tripper_test.go @@ -0,0 +1,113 @@ +package client_test + +import ( + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tidepool-org/platform/client" + errorsTest "github.com/tidepool-org/platform/errors/test" + testHttp "github.com/tidepool-org/platform/test/http" +) + +var _ = Describe("RoundTripper", func() { + var testRoundTripper *testHttp.RoundTripper + var roundTripper *client.RoundTripper + + BeforeEach(func() { + testRoundTripper = testHttp.NewRoundTripper() + }) + + Context("NewRoundTripper", func() { + It("returns successfully with a round tripper", func() { + roundTripper = client.NewRoundTripper(testRoundTripper) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + + It("returns successfully without a round tripper", func() { + roundTripper = client.NewRoundTripper(nil) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.ResolvedRoundTripper()).ToNot(BeNil()) + }) + }) + + Context("ResolvedRoundTripper", func() { + var originalDefaultTransport http.RoundTripper + var originalDefaultClientTransport http.RoundTripper + + BeforeEach(func() { + originalDefaultTransport = http.DefaultTransport + originalDefaultClientTransport = http.DefaultClient.Transport + }) + + AfterEach(func() { + http.DefaultClient.Transport = originalDefaultClientTransport + http.DefaultTransport = originalDefaultTransport + }) + + It("returns the round tripper if it is set", func() { + roundTripper = client.NewRoundTripper(testRoundTripper) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + + It("returns http.DefaultClient.Transport if the round tripper is not set and http.DefaultClient.Transport is set", func() { + http.DefaultClient.Transport = testRoundTripper + roundTripper = client.NewRoundTripper(nil) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + + It("returns http.DefaultTransport if the round tripper is not set and http.DefaultClient.Transport is not set", func() { + http.DefaultClient.Transport = nil + http.DefaultTransport = testRoundTripper + roundTripper = client.NewRoundTripper(nil) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + }) + + Context("WithRoundTripper", func() { + BeforeEach(func() { + roundTripper = client.NewRoundTripper(nil) + }) + + It("sets the round tripper when it was previously unset", func() { + roundTripper.WithRoundTripper(testRoundTripper) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(testRoundTripper))) + }) + + It("replaces the round tripper when it was previously set", func() { + roundTripper.WithRoundTripper(testRoundTripper) + replacement := testHttp.NewRoundTripper() + roundTripper.WithRoundTripper(replacement) + Expect(roundTripper.ResolvedRoundTripper()).To(BeIdenticalTo(http.RoundTripper(replacement))) + }) + }) + + Context("RoundTrip", func() { + var request *http.Request + + BeforeEach(func() { + request = testHttp.NewRequest() + }) + + It("returns the response from the resolved round tripper", func() { + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + roundTripper = client.NewRoundTripper(testRoundTripper) + + result, err := roundTripper.RoundTrip(request) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeIdenticalTo(testRoundTripper.Response)) + Expect(testRoundTripper.Request).To(BeIdenticalTo(request)) + }) + + It("returns the error from the resolved round tripper", func() { + testRoundTripper.Error = errorsTest.RandomError() + roundTripper = client.NewRoundTripper(testRoundTripper) + + result, err := roundTripper.RoundTrip(request) + Expect(err).To(Equal(testRoundTripper.Error)) + Expect(result).To(BeNil()) + }) + }) +}) diff --git a/data/service/service/standard.go b/data/service/service/standard.go index 0d5151504a..a309a01d5f 100644 --- a/data/service/service/standard.go +++ b/data/service/service/standard.go @@ -197,7 +197,7 @@ func (s *Standard) Initialize(provider application.Provider) error { func (s *Standard) Terminate() { if s.server != nil { if err := s.server.Shutdown(); err != nil { - s.Logger().Errorf("Error while terminating the the server: %v", err) + s.Logger().Errorf("Error while terminating the server: %v", err) } s.server = nil } diff --git a/data/store/mongo/mongo_datum.go b/data/store/mongo/mongo_datum.go index 02260c661d..f080fc8102 100644 --- a/data/store/mongo/mongo_datum.go +++ b/data/store/mongo/mongo_datum.go @@ -437,7 +437,7 @@ func (d *DatumRepository) ArchiveDeviceDataUsingHashesFromDataSet(ctx context.Co var updateInfo *mongo.UpdateResult - // Note that the "DeduplicatorHash" index is NOT used here as the fields in the query don't match the the index definition. On average an upload only has one device anyways (P90 ~ 1). However the "DeduplicatorHash" index is still useful for the UpdateMany operation that follows. + // Note that the "DeduplicatorHash" index is NOT used here as the fields in the query don't match the index definition. On average an upload only has one device anyways (P90 ~ 1). However the "DeduplicatorHash" index is still useful for the UpdateMany operation that follows. selector := bson.M{ "_userId": dataSet.UserID, "uploadId": dataSet.UploadID, diff --git a/dexcom/client/client.go b/dexcom/client/client.go index 83af300841..29a219d8a3 100644 --- a/dexcom/client/client.go +++ b/dexcom/client/client.go @@ -10,7 +10,6 @@ import ( "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/oauth" oauthClient "github.com/tidepool-org/platform/oauth/client" - "github.com/tidepool-org/platform/request" ) type Client struct { @@ -114,17 +113,9 @@ func (c *Client) sendDexcomRequestWithDataRange(ctx context.Context, startTime t } func (c *Client) sendDexcomRequest(ctx context.Context, method string, url string, responseBody interface{}, tokenSource oauth.TokenSource) error { - startTime := time.Now() - - err := c.client.SendOAuthRequest(ctx, method, url, nil, nil, responseBody, []request.ResponseInspector{prometheusCodePathResponseInspector}, tokenSource) - - if requestDuration := time.Since(startTime); requestDuration > requestDurationMaximum { - log.LoggerFromContext(ctx).WithField("requestDuration", requestDuration.Truncate(time.Millisecond).Seconds()).Warn("Request duration exceeds maximum") - } - - return err + return log.WarnIfDurationExceedsMaximum(ctx, requestDurationMaximum, url, func(ctx context.Context) error { + return c.client.SendOAuthRequest(ctx, method, url, nil, nil, responseBody, nil, tokenSource) + }) } -const requestDurationMaximum = 30 * time.Second - -var prometheusCodePathResponseInspector = request.NewPrometheusCodePathResponseInspector("tidepool_dexcom_api_client_requests", "Dexcom API client requests") +const requestDurationMaximum = 60 * time.Second diff --git a/dexcom/device.go b/dexcom/device.go index 0934e5dbaf..456d46e6c5 100644 --- a/dexcom/device.go +++ b/dexcom/device.go @@ -33,7 +33,7 @@ const ( DeviceTransmitterGenerationG4 = "g4" DeviceTransmitterGenerationG5 = "g5" DeviceTransmitterGenerationG6 = "g6" - DeviceTransmitterGenerationG6Pro = "g6 pro" // NOTE: Not specfied in API specs but found during actual usage + DeviceTransmitterGenerationG6Pro = "g6 pro" // NOTE: Not specified in API specs but found during actual usage DeviceTransmitterGenerationG6Plus = "g6+" DeviceTransmitterGenerationPro = "dexcomPro" DeviceTransmitterGenerationG7 = "g7" diff --git a/dexcom/fetch/runner.go b/dexcom/fetch/runner.go index 982676e28b..ba559ad6f3 100644 --- a/dexcom/fetch/runner.go +++ b/dexcom/fetch/runner.go @@ -110,8 +110,8 @@ func (r *Runner) GetRunnerType() string { return Type } -func (r *Runner) GetRunnerDeadline() time.Time { - return time.Now().Add(TaskDurationMaximum * 3) +func (r *Runner) GetRunnerDeadline() time.Duration { + return TaskDurationMaximum * 3 } func (r *Runner) GetRunnerTimeout() time.Duration { diff --git a/dexcom/fetch/runner_test.go b/dexcom/fetch/runner_test.go index ccd1d627a2..5d0b55bb20 100644 --- a/dexcom/fetch/runner_test.go +++ b/dexcom/fetch/runner_test.go @@ -109,7 +109,7 @@ var _ = Describe("Runner", func() { }) It("returns the runner deadline", func() { - Expect(runner.GetRunnerDeadline()).Should(BeTemporally("~", time.Now().Add(45*time.Minute), time.Second)) + Expect(runner.GetRunnerDeadline()).To(Equal(45 * time.Minute)) }) It("returns the runner timeout", func() { diff --git a/dexcom/provider/provider.go b/dexcom/provider/provider.go index 33a3cb5cb7..edc8bdb16e 100644 --- a/dexcom/provider/provider.go +++ b/dexcom/provider/provider.go @@ -2,8 +2,15 @@ package provider import ( "context" + "fmt" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/tidepool-org/platform/auth" + "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/config" dataSource "github.com/tidepool-org/platform/data/source" "github.com/tidepool-org/platform/dexcom" @@ -41,7 +48,18 @@ func New(configReporter config.Reporter, dataSourceClient dataSource.Client, tas return nil, errors.Wrap(err, "unable to create provider config") } - prvdr, err := oauthProvider.New(dexcom.ProviderName, cfg, nil) + // Attach prometheus round tripper to default client transport + prometheusRequestMetricsRoundTripper.WithRoundTripper(http.DefaultClient.Transport) + + // Create http client + httpClient := &http.Client{ + Transport: prometheusRequestMetricsRoundTripper, + CheckRedirect: http.DefaultClient.CheckRedirect, + Jar: http.DefaultClient.Jar, + Timeout: 2 * time.Minute, + } + + prvdr, err := oauthProvider.New(dexcom.ProviderName, cfg, httpClient, nil) if err != nil { return nil, err } @@ -117,7 +135,7 @@ func (p *Provider) OnCreate(ctx context.Context, providerSession *auth.ProviderS if _, err = p.dataSourceClient.Update(ctx, source.ID, nil, update); err != nil { // Attempt to delete task if data source not marked as connected - if taskErr := p.taskClient.DeleteTask(context.WithoutCancel(ctx), task.ID); taskErr != nil { + if taskErr := p.taskClient.DeleteTask(context.WithoutCancel(ctx), task.ID, nil); taskErr != nil { logger.WithError(taskErr).Error("Failure deleting task after failed data source update") } @@ -151,9 +169,50 @@ func (p *Provider) OnDelete(ctx context.Context, providerSession *auth.ProviderS logger.WithError(err).WithField(dexcom.DataKeyDataSourceID, dataSourceID).Error("Unable to update data source while deleting provider session") } } - if err = p.taskClient.DeleteTask(ctx, task.ID); err != nil { + if err = p.taskClient.DeleteTask(ctx, task.ID, nil); err != nil { logger.WithError(err).WithField("taskId", task.ID).Error("unable to delete task while deleting provider session") } } return nil } + +// Some Dexcom API responses include a "request-time" header with, supposedly, the internal duration of the request. +// This could be useful for debugging connection issues if compared against the calculated request duration. +// See client/prometheus.go for details on how the request duration is calculated and recorded. + +const RequestTimeHeaderName = "request-time" + +type PrometheusRequestMetricsRoundTripper struct { + *client.PrometheusRequestMetricsRoundTripper + requestTimeHistogramVec *prometheus.HistogramVec +} + +func NewPrometheusRequestMetricsRoundTripper(name string, help string) *PrometheusRequestMetricsRoundTripper { + return &PrometheusRequestMetricsRoundTripper{ + PrometheusRequestMetricsRoundTripper: client.NewPrometheusRequestMetricsRoundTripper(name, help), + requestTimeHistogramVec: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: fmt.Sprintf("%s_request_time_seconds", name), + Help: fmt.Sprintf("%s request time (seconds)", help), + Buckets: client.DurationBucketsDefault, + }, + client.PrometheusLabelNames(), + ), + } +} + +func (p *PrometheusRequestMetricsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + res, err := p.ResolvedRoundTripper().RoundTrip(req) + + if res != nil { + if labels := p.Labels(req, res); labels != nil { + if requestTime, parseErr := time.ParseDuration(res.Header.Get(RequestTimeHeaderName)); parseErr == nil { + p.requestTimeHistogramVec.With(*labels).Observe(requestTime.Seconds()) + } + } + } + + return res, err +} + +var prometheusRequestMetricsRoundTripper = NewPrometheusRequestMetricsRoundTripper("tidepool_dexcom_api", "Tidepool Dexcom API") diff --git a/dexcom/provider/provider_test.go b/dexcom/provider/provider_test.go index 887329603b..4a72702be5 100644 --- a/dexcom/provider/provider_test.go +++ b/dexcom/provider/provider_test.go @@ -1,8 +1,113 @@ package provider_test import ( + "net/http" + "strconv" + "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tidepool-org/platform/client" + dexcomProvider "github.com/tidepool-org/platform/dexcom/provider" + errorsTest "github.com/tidepool-org/platform/errors/test" + prometheusTest "github.com/tidepool-org/platform/prometheus/test" + "github.com/tidepool-org/platform/test" + testHttp "github.com/tidepool-org/platform/test/http" ) var _ = Describe("Provider", func() { + It("PathPatternAny is expected", func() { + Expect(dexcomProvider.RequestTimeHeaderName).To(Equal("request-time")) + }) + + Context("PrometheusRequestMetricsRoundTripper", func() { + Context("NewPrometheusRequestMetricsRoundTripper", func() { + It("returns successfully", func() { + roundTripper := dexcomProvider.NewPrometheusRequestMetricsRoundTripper(prometheusTest.RandomMetricName(), prometheusTest.RandomMetricHelp()) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestMetricsRoundTripper).ToNot(BeNil()) + }) + }) + + Context("RoundTrip", func() { + var testRoundTripper *testHttp.RoundTripper + var name string + var roundTripper *dexcomProvider.PrometheusRequestMetricsRoundTripper + var request *http.Request + + BeforeEach(func() { + testRoundTripper = testHttp.NewRoundTripper() + name = prometheusTest.RandomMetricName() + roundTripper = dexcomProvider.NewPrometheusRequestMetricsRoundTripper(name, prometheusTest.RandomMetricHelp()) + roundTripper.WithRoundTripper(testRoundTripper) + request = testHttp.NewRequest() + }) + + It("returns the response from the resolved round tripper", func() { + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + + result := test.Must(roundTripper.RoundTrip(request)) + Expect(result).To(BeIdenticalTo(testRoundTripper.Response)) + Expect(testRoundTripper.Request).To(BeIdenticalTo(request)) + }) + + It("returns the error from the resolved round tripper", func() { + testErr := errorsTest.RandomError() + testRoundTripper.Error = testErr + + result, err := roundTripper.RoundTrip(request) + Expect(err).To(Equal(testErr)) + Expect(result).To(BeNil()) + }) + + It("does not record a request time metric when the resolved round tripper returns an error", func() { + testRoundTripper.Error = errorsTest.RandomError() + + _, _ = roundTripper.RoundTrip(request) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) + }) + + It("records a request time metric when the response has a valid request-time header", func() { + statusCode := testHttp.NewStatusCode() + requestTime := time.Duration(test.RandomIntFromRange(1, 60*1000)) * time.Millisecond + header := http.Header{} + header.Set(dexcomProvider.RequestTimeHeaderName, requestTime.String()) + testRoundTripper.Response = &http.Response{StatusCode: statusCode, Header: header} + + _ = test.Must(roundTripper.RoundTrip(request)) + + family := prometheusTest.MetricFamilyFromName(name + "_request_time_seconds") + Expect(family).ToNot(BeNil()) + Expect(family.GetMetric()).To(HaveLen(1)) + metric := family.GetMetric()[0] + Expect(metric.GetHistogram().GetSampleCount()).To(Equal(uint64(1))) + Expect(metric.GetHistogram().GetSampleSum()).To(Equal(requestTime.Seconds())) + Expect(prometheusTest.LabelPairsToMap(metric.GetLabel())).To(Equal(map[string]string{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: strconv.Itoa(statusCode), + })) + }) + + It("does not record a request time metric when the response does not have a request-time header", func() { + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode(), Header: http.Header{}} + + _ = test.Must(roundTripper.RoundTrip(request)) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) + }) + + It("does not record a request time metric when the request-time header is not a valid duration", func() { + header := http.Header{} + header.Set(dexcomProvider.RequestTimeHeaderName, test.RandomString()) + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode(), Header: header} + + _ = test.Must(roundTripper.RoundTrip(request)) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) + }) + }) + }) }) diff --git a/ehr/reconcile/runner.go b/ehr/reconcile/runner.go index 5c9e61b7cb..08f79322e8 100644 --- a/ehr/reconcile/runner.go +++ b/ehr/reconcile/runner.go @@ -41,8 +41,8 @@ func (r *Runner) GetRunnerType() string { return Type } -func (r *Runner) GetRunnerDeadline() time.Time { - return time.Now().Add(TaskDurationMaximum * 3) +func (r *Runner) GetRunnerDeadline() time.Duration { + return TaskDurationMaximum * 3 } func (r *Runner) GetRunnerTimeout() time.Duration { @@ -117,7 +117,7 @@ func (r *Runner) getSyncTasks(ctx context.Context) (map[string]task.Task, error) func (r *Runner) reconcileTasks(ctx context.Context, tsk *task.Task, plan ReconciliationPlan) { for _, t := range plan.ToDelete { - if err := r.taskClient.DeleteTask(ctx, t.ID); err != nil { + if err := r.taskClient.DeleteTask(ctx, t.ID, nil); err != nil { tsk.AppendError(errors.Wrap(err, "unable to delete task")) } } @@ -127,7 +127,7 @@ func (r *Runner) reconcileTasks(ctx context.Context, tsk *task.Task, plan Reconc } } for id, update := range plan.ToUpdate { - if _, err := r.taskClient.UpdateTask(ctx, id, update); err != nil { + if _, err := r.taskClient.UpdateTask(ctx, id, nil, update); err != nil { tsk.AppendError(errors.Wrap(err, "unable to update task")) } } diff --git a/ehr/reconcile/runner_test.go b/ehr/reconcile/runner_test.go index ad499aa3ae..4ec44b2d17 100644 --- a/ehr/reconcile/runner_test.go +++ b/ehr/reconcile/runner_test.go @@ -82,7 +82,7 @@ var _ = Describe("Runner", func() { clinicsClient.EXPECT().ListEHREnabledClinics(gomock.Any()).Return(clinics, nil) taskClient.EXPECT().ListTasks(gomock.Any(), gomock.Any(), gomock.Any()).Return(tasksList, nil) - taskClient.EXPECT().DeleteTask(gomock.Any(), gomock.Eq(tasks[*toBeDeleted.Id].ID)).Return(nil) + taskClient.EXPECT().DeleteTask(gomock.Any(), gomock.Eq(tasks[*toBeDeleted.Id].ID), gomock.Nil()).Return(nil) taskClient.EXPECT().CreateTask(gomock.Any(), gomock.Any()).Return(nil, nil) runner.Run(context.Background(), t) }) diff --git a/ehr/reconcile/task.go b/ehr/reconcile/task.go index 6d5df324c3..1b4e55245b 100644 --- a/ehr/reconcile/task.go +++ b/ehr/reconcile/task.go @@ -1,8 +1,6 @@ package reconcile import ( - "time" - "github.com/tidepool-org/platform/pointer" "github.com/tidepool-org/platform/task" ) @@ -13,8 +11,7 @@ const ( func NewTaskCreate() *task.TaskCreate { return &task.TaskCreate{ - Name: pointer.FromString(Type), - Type: Type, - AvailableTime: pointer.FromAny(time.Now().UTC()), + Name: pointer.FromString(Type), + Type: Type, } } diff --git a/ehr/reconcile/task_test.go b/ehr/reconcile/task_test.go index 5b0dd6bc0e..72daa99119 100644 --- a/ehr/reconcile/task_test.go +++ b/ehr/reconcile/task_test.go @@ -1,8 +1,6 @@ package reconcile_test import ( - "time" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" @@ -17,7 +15,6 @@ var _ = Describe("Task", func() { Expect(create).ToNot(BeNil()) Expect(create.Name).To(PointTo(Equal(reconcile.Type))) Expect(create.Type).To(Equal(reconcile.Type)) - Expect(create.AvailableTime).To(PointTo(BeTemporally("~", time.Now(), 3*time.Second))) }) }) }) diff --git a/ehr/sync/runner.go b/ehr/sync/runner.go index 428f7a27c7..e3ec5c7360 100644 --- a/ehr/sync/runner.go +++ b/ehr/sync/runner.go @@ -31,8 +31,8 @@ func (r *Runner) GetRunnerType() string { return Type } -func (r *Runner) GetRunnerDeadline() time.Time { - return time.Now().Add(TaskDurationMaximum * 3) +func (r *Runner) GetRunnerDeadline() time.Duration { + return TaskDurationMaximum * 3 } func (r *Runner) GetRunnerTimeout() time.Duration { diff --git a/ehr/sync/task.go b/ehr/sync/task.go index 7d32118966..88db979785 100644 --- a/ehr/sync/task.go +++ b/ehr/sync/task.go @@ -21,9 +21,8 @@ func TaskName(clinicId string) string { func NewTaskCreate(clinicId string, cadence time.Duration) *task.TaskCreate { tsk := &task.TaskCreate{ - Name: pointer.FromString(TaskName(clinicId)), - Type: Type, - AvailableTime: pointer.FromAny(time.Now().UTC()), + Name: pointer.FromString(TaskName(clinicId)), + Type: Type, Data: map[string]interface{}{ "clinicId": clinicId, }, diff --git a/ehr/sync/task_test.go b/ehr/sync/task_test.go index e561aca722..e288c83f98 100644 --- a/ehr/sync/task_test.go +++ b/ehr/sync/task_test.go @@ -19,7 +19,6 @@ var _ = Describe("Task", func() { Expect(create).ToNot(BeNil()) Expect(create.Name).To(PointTo(Equal(sync.TaskName(*clinic.Id)))) Expect(create.Type).To(Equal(sync.Type)) - Expect(create.AvailableTime).ToNot(BeNil()) }) It("stores the clinic id in the data", func() { diff --git a/go.mod b/go.mod index adc989e205..b0103de64e 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/onsi/ginkgo/v2 v2.27.5 github.com/onsi/gomega v1.39.0 github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/client_model v0.6.1 github.com/rinchsan/device-check-go v1.3.0 github.com/solworktech/md2pdf/v2 v2.2.18 github.com/tidepool-org/clinic/client v0.0.0-20250122123230-f89e2b1540dc @@ -105,7 +106,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect - github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/radovskyb/watcher v1.0.7 // indirect diff --git a/log/test/serializer.go b/log/test/serializer.go index ae8851777e..960d2907e2 100644 --- a/log/test/serializer.go +++ b/log/test/serializer.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "regexp" + "sync" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" @@ -12,10 +13,13 @@ import ( type Serializer struct { SerializedFields []log.Fields + mutex *sync.Mutex } func NewSerializer() *Serializer { - return &Serializer{} + return &Serializer{ + mutex: &sync.Mutex{}, + } } func (s *Serializer) Serialize(fields log.Fields) error { @@ -23,6 +27,9 @@ func (s *Serializer) Serialize(fields log.Fields) error { return errors.New("fields are missing") } + s.mutex.Lock() + defer s.mutex.Unlock() + s.SerializedFields = append(s.SerializedFields, fields) return nil @@ -76,6 +83,9 @@ func (s *Serializer) AssertErrorExpression(messageExpression *regexp.Regexp, con } func (s *Serializer) assertContainsFields(containsFields []log.Fields, matcher func(serializedFields log.Fields) bool) { + s.mutex.Lock() + defer s.mutex.Unlock() + joinedContainsFields := s.joinContainsFields(containsFields) for _, serializedFields := range s.SerializedFields { if s.serializedFieldsContainsFields(serializedFields, joinedContainsFields) && (matcher == nil || matcher(serializedFields)) { diff --git a/oauth/provider/client/client.go b/oauth/provider/client/client.go index 02d39998c7..14ad91571a 100644 --- a/oauth/provider/client/client.go +++ b/oauth/provider/client/client.go @@ -1,6 +1,8 @@ package client import ( + "net/http" + "github.com/lestrrat-go/jwx/v2/jwk" "github.com/tidepool-org/platform/client" @@ -14,11 +16,11 @@ type Provider struct { *oauthClient.Client } -func New(name string, config *Config, jwks jwk.Set) (*Provider, error) { - return NewWithErrorParser(name, config, jwks, nil) +func New(name string, config *Config, httpClient *http.Client, jwks jwk.Set) (*Provider, error) { + return NewWithErrorParser(name, config, httpClient, jwks, nil) } -func NewWithErrorParser(name string, config *Config, jwks jwk.Set, errorResponseParser client.ErrorResponseParser) (*Provider, error) { +func NewWithErrorParser(name string, config *Config, httpClient *http.Client, jwks jwk.Set, errorResponseParser client.ErrorResponseParser) (*Provider, error) { if name == "" { return nil, errors.New("name is missing") } @@ -26,7 +28,7 @@ func NewWithErrorParser(name string, config *Config, jwks jwk.Set, errorResponse return nil, errors.Wrap(err, "config is invalid") } - prvdr, err := oauthProvider.New(name, config.ProviderConfig, jwks) + prvdr, err := oauthProvider.New(name, config.ProviderConfig, httpClient, jwks) if err != nil { return nil, err } diff --git a/oauth/provider/provider.go b/oauth/provider/provider.go index 117eb2a8eb..42ed36a78d 100644 --- a/oauth/provider/provider.go +++ b/oauth/provider/provider.go @@ -3,6 +3,7 @@ package provider import ( "context" "fmt" + "net/http" "github.com/golang-jwt/jwt/v4" "github.com/lestrrat-go/jwx/v2/jwk" @@ -18,11 +19,12 @@ import ( type Provider struct { name string config Config + httpClient *http.Client jwks jwk.Set oauth2Config *oauth2.Config } -func New(name string, config *Config, jwks jwk.Set) (*Provider, error) { +func New(name string, config *Config, httpClient *http.Client, jwks jwk.Set) (*Provider, error) { if name == "" { return nil, errors.New("name is missing") } @@ -31,6 +33,9 @@ func New(name string, config *Config, jwks jwk.Set) (*Provider, error) { } else if err := config.Validate(); err != nil { return nil, errors.Wrap(err, "config is invalid") } + if httpClient == nil { + return nil, errors.New("http client is missing") + } oauth2Config := &oauth2.Config{ ClientID: config.ClientID, @@ -49,6 +54,7 @@ func New(name string, config *Config, jwks jwk.Set) (*Provider, error) { return &Provider{ name: name, config: *config, + httpClient: httpClient, jwks: jwks, oauth2Config: oauth2Config, }, nil @@ -113,7 +119,7 @@ func (p *Provider) TokenSource(ctx context.Context, token *auth.OAuthToken) (oau return nil, errors.New("token is missing") } - tknSrc := p.oauth2Config.TokenSource(ctx, token.RawToken()) + tknSrc := p.oauth2Config.TokenSource(context.WithValue(ctx, oauth2.HTTPClient, p.httpClient), token.RawToken()) if tknSrc == nil { return nil, errors.New("unable to create token source") } diff --git a/oura/client/client.go b/oura/client/client.go index 4475c6eeb4..9588ffbe51 100644 --- a/oura/client/client.go +++ b/oura/client/client.go @@ -182,7 +182,7 @@ func (c *Client) GetData(ctx context.Context, dataType string, timeRange *times. mutators := []request.RequestMutator{request.NewParametersMutator(parameters)} // Possible response status codes (see below for details): 200 (DataResponse), 400, 401, 403, 422, 429 - url := c.client.ConstructURL("v2", "usercollection", DataTypeToPath(dataType)) + url := c.client.ConstructURL("v2", "usercollection", oura.DataTypeToPath(dataType)) dataResponse := &oura.DataResponse{} if err := c.sendOAuthRequest(ctx, http.MethodGet, url, mutators, nil, dataResponse, tokenSource); err != nil { return nil, errors.Wrap(err, "unable to get data") @@ -203,7 +203,7 @@ func (c *Client) GetDatum(ctx context.Context, dataType string, dataID string, t } // Possible response status codes (see below for details): 200 (DataResponse), 400, 401, 403, 422, 429 - url := c.client.ConstructURL("v2", "usercollection", DataTypeToPath(dataType), dataID) + url := c.client.ConstructURL("v2", "usercollection", oura.DataTypeToPath(dataType), dataID) dataResponse := oura.Datum{} if err := c.sendOAuthRequest(ctx, http.MethodGet, url, nil, nil, &dataResponse, tokenSource); err != nil { return nil, errors.Wrap(err, "unable to get datum") @@ -229,7 +229,7 @@ func (c *Client) RevokeOAuthToken(ctx context.Context, oauthToken *auth.OAuthTok func (c *Client) sendOAuthRequest(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody any, responseBody any, tokenSource oauth.TokenSource) error { return log.WarnIfDurationExceedsMaximum(ctx, requestDurationMaximum, url, func(ctx context.Context) error { - return c.client.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, []request.ResponseInspector{prometheusCodePathResponseInspector}, tokenSource) + return c.client.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) }) } @@ -243,34 +243,11 @@ func (c *Client) sendClientRequest(ctx context.Context, method string, url strin func (c *Client) sendBaseRequest(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody any, responseBody any, inspectors []request.ResponseInspector) error { return log.WarnIfDurationExceedsMaximum(ctx, requestDurationMaximum, url, func(ctx context.Context) error { - return c.client.Client().RequestDataWithHTTPClient(ctx, method, url, mutators, requestBody, responseBody, append(inspectors, prometheusCodePathResponseInspector), http.DefaultClient) + return c.client.Client().RequestDataWithHTTPClient(ctx, method, url, mutators, requestBody, responseBody, inspectors, http.DefaultClient) }) } -func DataTypeToPath(dataType string) string { - switch dataType { - case oura.DataTypeVO2Max: - return "vO2_max" // Capitalization inconsistency - default: - return dataType - } -} - -func PrometheusCodePathPatterns() []string { - var patterns []string - for _, eventDataType := range oura.EventDataTypes() { - patterns = append(patterns, fmt.Sprintf("/v2/usercollection/%s/{document_id}", DataTypeToPath(eventDataType))) - } - return append(patterns, - "/v2/webhook/subscription/{id}", - "/v2/webhook/subscription/renew/{id}", - request.PatternAny, - ) -} - -const requestDurationMaximum = 30 * time.Second - -var prometheusCodePathResponseInspector = request.NewPrometheusCodePathResponseInspectorWithPatterns("tidepool_oura_api_client_requests", "Oura API client requests", PrometheusCodePathPatterns()...) +const requestDurationMaximum = 60 * time.Second // Possible response status codes from Oura API: // 200: successful get/list; response body contains the requested resource(s) diff --git a/oura/client/client_test.go b/oura/client/client_test.go index b38181bb50..957bbaf50b 100644 --- a/oura/client/client_test.go +++ b/oura/client/client_test.go @@ -526,7 +526,7 @@ var _ = Describe("client", func() { mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(false, nil) server.AppendHandlers( CombineHandlers( - VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s", ouraClient.DataTypeToPath(dataType)), expectedQuery.Encode()), + VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s", oura.DataTypeToPath(dataType)), expectedQuery.Encode()), VerifyHeader(http.Header{}), VerifyBody(nil), RespondWith(http.StatusInternalServerError, nil), @@ -544,7 +544,7 @@ var _ = Describe("client", func() { mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(false, nil) server.AppendHandlers( CombineHandlers( - VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s", ouraClient.DataTypeToPath(dataType)), expectedQuery.Encode()), + VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s", oura.DataTypeToPath(dataType)), expectedQuery.Encode()), VerifyBody(nil), RespondWithJSONEncoded(http.StatusOK, expectedData), ), @@ -614,7 +614,7 @@ var _ = Describe("client", func() { mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(false, nil) server.AppendHandlers( CombineHandlers( - VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s/%s", ouraClient.DataTypeToPath(dataType), dataID)), + VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s/%s", oura.DataTypeToPath(dataType), dataID)), VerifyHeader(http.Header{}), VerifyBody(nil), RespondWith(http.StatusInternalServerError, nil), @@ -632,7 +632,7 @@ var _ = Describe("client", func() { mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(false, nil) server.AppendHandlers( CombineHandlers( - VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s/%s", ouraClient.DataTypeToPath(dataType), dataID)), + VerifyRequest("GET", fmt.Sprintf("/v2/usercollection/%s/%s", oura.DataTypeToPath(dataType), dataID)), VerifyBody(nil), RespondWithJSONEncoded(http.StatusOK, expectedDatum), ), @@ -699,58 +699,4 @@ var _ = Describe("client", func() { }) }) }) - - Context("DataTypeToPath", func() { - It("returns expected path for valid data type", func() { - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyActivity)).To(Equal(oura.DataTypeDailyActivity)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyCardiovascularAge)).To(Equal(oura.DataTypeDailyCardiovascularAge)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyCyclePhases)).To(Equal(oura.DataTypeDailyCyclePhases)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyReadiness)).To(Equal(oura.DataTypeDailyReadiness)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyResilience)).To(Equal(oura.DataTypeDailyResilience)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailySleep)).To(Equal(oura.DataTypeDailySleep)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailySpO2)).To(Equal(oura.DataTypeDailySpO2)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeDailyStress)).To(Equal(oura.DataTypeDailyStress)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeEnhancedTag)).To(Equal(oura.DataTypeEnhancedTag)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeHeartRate)).To(Equal(oura.DataTypeHeartRate)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeRestModePeriod)).To(Equal(oura.DataTypeRestModePeriod)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeRingBatteryLevel)).To(Equal(oura.DataTypeRingBatteryLevel)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeRingConfiguration)).To(Equal(oura.DataTypeRingConfiguration)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeSession)).To(Equal(oura.DataTypeSession)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeSleep)).To(Equal(oura.DataTypeSleep)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeSleepTime)).To(Equal(oura.DataTypeSleepTime)) - Expect(ouraClient.DataTypeToPath(oura.DataTypeVO2Max)).To(Equal("vO2_max")) - Expect(ouraClient.DataTypeToPath(oura.DataTypeWorkout)).To(Equal(oura.DataTypeWorkout)) - }) - - It("returns data type for unknown data type", func() { - dataType := test.RandomString() - Expect(ouraClient.DataTypeToPath(dataType)).To(Equal(dataType)) - }) - }) - - Context("PrometheusCodePathPatterns", func() { - It("returns expected patterns", func() { - Expect(ouraClient.PrometheusCodePathPatterns()).To(Equal([]string{ - "/v2/usercollection/daily_activity/{document_id}", - "/v2/usercollection/daily_cardiovascular_age/{document_id}", - "/v2/usercollection/daily_cycle_phases/{document_id}", - "/v2/usercollection/daily_readiness/{document_id}", - "/v2/usercollection/daily_resilience/{document_id}", - "/v2/usercollection/daily_sleep/{document_id}", - "/v2/usercollection/daily_spo2/{document_id}", - "/v2/usercollection/daily_stress/{document_id}", - "/v2/usercollection/enhanced_tag/{document_id}", - "/v2/usercollection/rest_mode_period/{document_id}", - "/v2/usercollection/ring_configuration/{document_id}", - "/v2/usercollection/session/{document_id}", - "/v2/usercollection/sleep/{document_id}", - "/v2/usercollection/sleep_time/{document_id}", - "/v2/usercollection/vO2_max/{document_id}", - "/v2/usercollection/workout/{document_id}", - "/v2/webhook/subscription/{id}", - "/v2/webhook/subscription/renew/{id}", - "/", - })) - }) - }) }) diff --git a/oura/oura.go b/oura/oura.go index 2840ef7422..37314e5942 100644 --- a/oura/oura.go +++ b/oura/oura.go @@ -218,6 +218,15 @@ func DataTypeInScope(dataType string, scope string) bool { return slices.Contains(ScopesForDataType(dataType), scope) } +func DataTypeToPath(dataType string) string { + switch dataType { + case DataTypeVO2Max: + return "vO2_max" // Capitalization inconsistency + default: + return dataType + } +} + type BaseClient interface { ClientID() string ClientSecret() string diff --git a/oura/oura_test.go b/oura/oura_test.go index d029a4ce4f..505bc1d5e1 100644 --- a/oura/oura_test.go +++ b/oura/oura_test.go @@ -375,6 +375,34 @@ var _ = Describe("oura", func() { }) }) + Context("DataTypeToPath", func() { + It("returns expected path for valid data type", func() { + Expect(oura.DataTypeToPath(oura.DataTypeDailyActivity)).To(Equal(oura.DataTypeDailyActivity)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyCardiovascularAge)).To(Equal(oura.DataTypeDailyCardiovascularAge)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyCyclePhases)).To(Equal(oura.DataTypeDailyCyclePhases)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyReadiness)).To(Equal(oura.DataTypeDailyReadiness)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyResilience)).To(Equal(oura.DataTypeDailyResilience)) + Expect(oura.DataTypeToPath(oura.DataTypeDailySleep)).To(Equal(oura.DataTypeDailySleep)) + Expect(oura.DataTypeToPath(oura.DataTypeDailySpO2)).To(Equal(oura.DataTypeDailySpO2)) + Expect(oura.DataTypeToPath(oura.DataTypeDailyStress)).To(Equal(oura.DataTypeDailyStress)) + Expect(oura.DataTypeToPath(oura.DataTypeEnhancedTag)).To(Equal(oura.DataTypeEnhancedTag)) + Expect(oura.DataTypeToPath(oura.DataTypeHeartRate)).To(Equal(oura.DataTypeHeartRate)) + Expect(oura.DataTypeToPath(oura.DataTypeRestModePeriod)).To(Equal(oura.DataTypeRestModePeriod)) + Expect(oura.DataTypeToPath(oura.DataTypeRingBatteryLevel)).To(Equal(oura.DataTypeRingBatteryLevel)) + Expect(oura.DataTypeToPath(oura.DataTypeRingConfiguration)).To(Equal(oura.DataTypeRingConfiguration)) + Expect(oura.DataTypeToPath(oura.DataTypeSession)).To(Equal(oura.DataTypeSession)) + Expect(oura.DataTypeToPath(oura.DataTypeSleep)).To(Equal(oura.DataTypeSleep)) + Expect(oura.DataTypeToPath(oura.DataTypeSleepTime)).To(Equal(oura.DataTypeSleepTime)) + Expect(oura.DataTypeToPath(oura.DataTypeVO2Max)).To(Equal("vO2_max")) + Expect(oura.DataTypeToPath(oura.DataTypeWorkout)).To(Equal(oura.DataTypeWorkout)) + }) + + It("returns data type for unknown data type", func() { + dataType := test.RandomString() + Expect(oura.DataTypeToPath(dataType)).To(Equal(dataType)) + }) + }) + Context("CreateSubscription", func() { DescribeTable("serializes the datum as expected", func(mutator func(datum *oura.CreateSubscription)) { diff --git a/oura/provider/provider.go b/oura/provider/provider.go index 561c60a3f8..aeb352f39d 100644 --- a/oura/provider/provider.go +++ b/oura/provider/provider.go @@ -2,10 +2,14 @@ package provider import ( "context" + "fmt" + "net/http" "slices" + "time" "github.com/tidepool-org/platform/auth" authProviderSession "github.com/tidepool-org/platform/auth/providersession" + "github.com/tidepool-org/platform/client" dataSource "github.com/tidepool-org/platform/data/source" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" @@ -63,7 +67,18 @@ func New(dependencies Dependencies) (*Provider, error) { return nil, errors.Wrap(err, "dependencies is invalid") } - oauthProviderClient, err := oauthProviderClient.NewWithErrorParser(oura.ProviderName, dependencies.Config.Config, nil, &ouraClient.ErrorResponseParser{}) + // Attach prometheus round tripper to default client transport + prometheusRequestMetricsRoundTripper.WithRoundTripper(http.DefaultClient.Transport) + + // Create http client + httpClient := &http.Client{ + Transport: prometheusRequestMetricsRoundTripper, + CheckRedirect: http.DefaultClient.CheckRedirect, + Jar: http.DefaultClient.Jar, + Timeout: 2 * time.Minute, + } + + oauthProviderClient, err := oauthProviderClient.NewWithErrorParser(oura.ProviderName, dependencies.Config.Config, httpClient, nil, &ouraClient.ErrorResponseParser{}) if err != nil { return nil, err } @@ -298,3 +313,17 @@ func (p *Provider) createUserRevokeWork(ctx context.Context, providerSession *au log.LoggerFromContext(ctx).Debug("created user revoke work") return nil } + +func PrometheusPathPatterns() []string { + var pathPatterns []string + for _, eventDataType := range oura.EventDataTypes() { + pathPatterns = append(pathPatterns, fmt.Sprintf("/v2/usercollection/%s/{document_id}", oura.DataTypeToPath(eventDataType))) + } + return append(pathPatterns, + "/v2/webhook/subscription/{id}", + "/v2/webhook/subscription/renew/{id}", + client.PathPatternAny, + ) +} + +var prometheusRequestMetricsRoundTripper = client.NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets("tidepool_oura_api", "Tidepool Oura API", PrometheusPathPatterns(), nil) diff --git a/oura/provider/provider_test.go b/oura/provider/provider_test.go index 677c4499a3..6c3f86ddfc 100644 --- a/oura/provider/provider_test.go +++ b/oura/provider/provider_test.go @@ -257,4 +257,30 @@ var _ = Describe("provider", func() { }) }) }) + + Context("PrometheusCodePathPatterns", func() { + It("returns expected patterns", func() { + Expect(ouraProvider.PrometheusPathPatterns()).To(Equal([]string{ + "/v2/usercollection/daily_activity/{document_id}", + "/v2/usercollection/daily_cardiovascular_age/{document_id}", + "/v2/usercollection/daily_cycle_phases/{document_id}", + "/v2/usercollection/daily_readiness/{document_id}", + "/v2/usercollection/daily_resilience/{document_id}", + "/v2/usercollection/daily_sleep/{document_id}", + "/v2/usercollection/daily_spo2/{document_id}", + "/v2/usercollection/daily_stress/{document_id}", + "/v2/usercollection/enhanced_tag/{document_id}", + "/v2/usercollection/rest_mode_period/{document_id}", + "/v2/usercollection/ring_configuration/{document_id}", + "/v2/usercollection/session/{document_id}", + "/v2/usercollection/sleep/{document_id}", + "/v2/usercollection/sleep_time/{document_id}", + "/v2/usercollection/vO2_max/{document_id}", + "/v2/usercollection/workout/{document_id}", + "/v2/webhook/subscription/{id}", + "/v2/webhook/subscription/renew/{id}", + "/", + })) + }) + }) }) diff --git a/pointer/default.go b/pointer/default.go index edbbec0037..92acddbdc6 100644 --- a/pointer/default.go +++ b/pointer/default.go @@ -9,6 +9,13 @@ func Default[T any, S *T](value S, defaultValue T) T { return *value } +func DefaultArray[T any](value []T, defaultValue []T) []T { + if value == nil { + return defaultValue + } + return value +} + func DefaultPointer[T any](value *T, defaultValue *T) *T { if value == nil { return defaultValue diff --git a/pointer/default_test.go b/pointer/default_test.go index 376d4603ee..daf51348d0 100644 --- a/pointer/default_test.go +++ b/pointer/default_test.go @@ -25,8 +25,23 @@ var _ = Describe("Default", func() { }) }) + Context("DefaultArray", func() { + It("returns the default value if the pointer to the value is nil", func() { + defaultValue := test.RandomStringArray() + result := pointer.DefaultArray(nil, defaultValue) + Expect(result).To(Equal(defaultValue)) + }) + + It("returns the dereferenced pointer to the value if the pointer to the value is not nil", func() { + value := test.RandomStringArray() + defaultValue := test.RandomStringArray() + result := pointer.DefaultArray(value, defaultValue) + Expect(result).To(Equal(value)) + }) + }) + Context("DefaultPointer", func() { - It("returns a pointer to the the default value if the pointer to the value is nil", func() { + It("returns a pointer to the default value if the pointer to the value is nil", func() { defaultValue := test.RandomString() result := pointer.DefaultPointer(nil, &defaultValue) Expect(result).To(PointTo(Equal(defaultValue))) diff --git a/prometheus/test/prometheus.go b/prometheus/test/prometheus.go new file mode 100644 index 0000000000..15355d651c --- /dev/null +++ b/prometheus/test/prometheus.go @@ -0,0 +1,34 @@ +package test + +import ( + "github.com/prometheus/client_golang/prometheus" + prometheusModel "github.com/prometheus/client_model/go" + + "github.com/tidepool-org/platform/test" +) + +func RandomMetricName() string { + return test.RandomStringFromRangeAndCharset(8, 16, test.CharsetLowercase) +} + +func RandomMetricHelp() string { + return test.RandomString() +} + +func MetricFamilyFromName(name string) *prometheusModel.MetricFamily { + metricFamilies := test.Must(prometheus.DefaultGatherer.Gather()) + for _, metricFamily := range metricFamilies { + if metricFamily.GetName() == name { + return metricFamily + } + } + return nil +} + +func LabelPairsToMap(labelPairs []*prometheusModel.LabelPair) map[string]string { + labels := map[string]string{} + for _, labelPair := range labelPairs { + labels[labelPair.GetName()] = labelPair.GetValue() + } + return labels +} diff --git a/request/condition.go b/request/condition.go index fbdcd08e10..ea4454b52b 100644 --- a/request/condition.go +++ b/request/condition.go @@ -15,6 +15,12 @@ func NewCondition() *Condition { return &Condition{} } +func NewConditionWithRevision(revision *int) *Condition { + return &Condition{ + Revision: revision, + } +} + func (c *Condition) Parse(parser structure.ObjectParser) { c.Revision = parser.Int("revision") } diff --git a/request/inspector.go b/request/inspector.go index c0332ddbf0..d3bd40ebe2 100644 --- a/request/inspector.go +++ b/request/inspector.go @@ -2,10 +2,6 @@ package request import ( "net/http" - "strconv" - - "github.com/prometheus/client_golang/prometheus" - prometheusPromauto "github.com/prometheus/client_golang/prometheus/promauto" ) type ResponseInspector interface { @@ -31,53 +27,3 @@ func NewHeadersInspector() *HeadersInspector { func (h *HeadersInspector) InspectResponse(res *http.Response) { h.Headers = res.Header } - -type PrometheusCodePathResponseInspector struct { - *prometheus.CounterVec - patternMux *http.ServeMux -} - -// When there are only a few discrete paths possible, then no need to simplify via patterns. -// -// For example: /one, /two -func NewPrometheusCodePathResponseInspector(name string, help string) *PrometheusCodePathResponseInspector { - return NewPrometheusCodePathResponseInspectorWithPatterns(name, help) -} - -// Where there are numerous discrete paths possible, then must simplify via patterns to prevent overwhelming Prometheus. -// If no patterns are specified, then all paths are recorded as-is. If one or more patterns are specified, but a path -// does not match one of the patterns, then the path is NOT captured by Prometheus. If you wish to match "all other" paths -// and records those paths as-is, then add the pattern PatternAny at the end of your patterns. -// -// Uses standard Go HTTP pattern matching. See https://go.dev/src/net/http/pattern.go. -// -// For example: /one/{id}, /two/{id} -func NewPrometheusCodePathResponseInspectorWithPatterns(name string, help string, patterns ...string) *PrometheusCodePathResponseInspector { - var patternMux *http.ServeMux - - if len(patterns) > 0 { - patternMux = http.NewServeMux() - for _, pattern := range patterns { - patternMux.HandleFunc(pattern, func(http.ResponseWriter, *http.Request) {}) - } - } - - return &PrometheusCodePathResponseInspector{ - CounterVec: prometheusPromauto.NewCounterVec(prometheus.CounterOpts{Name: name, Help: help}, []string{"code", "path"}), - patternMux: patternMux, - } -} - -func (p *PrometheusCodePathResponseInspector) InspectResponse(res *http.Response) { - path := res.Request.URL.Path - if p.patternMux != nil { - if _, pattern := p.patternMux.Handler(res.Request); pattern == "" { - return - } else if pattern != PatternAny { - path = pattern - } - } - p.With(prometheus.Labels{"code": strconv.Itoa(res.StatusCode), "path": path}).Inc() -} - -const PatternAny = "/" diff --git a/services/tools/dexcom_analyze/dexcom_analyze.go b/services/tools/dexcom_analyze/dexcom_analyze.go index 27b2f4c153..434d7de4d6 100644 --- a/services/tools/dexcom_analyze/dexcom_analyze.go +++ b/services/tools/dexcom_analyze/dexcom_analyze.go @@ -95,13 +95,10 @@ const ( Issue_Task_With_State_Failed_AvailableTime_Present = "task with state failed available time present" Issue_Task_With_State_Failed_DeadlineTime_Present = "task with state failed deadline time present" Issue_Task_With_State_Failed_Error_Missing = "task with state failed error missing" - Issue_Task_With_State_Failed_ExpirationTime_Present = "task with state failed expiration time present" Issue_Task_With_State_Pending_DeadlineTime_Present = "task with state pending deadline time present" - Issue_Task_With_State_Pending_ExpirationTime_Present = "task with state pending expiration time present" Issue_Task_With_State_Running_AvailableTime_Present = "task with state running available time present" Issue_Task_With_State_Running_DeadlineTime_Missing = "task with state running deadline time missing" Issue_Task_With_State_Running_Error_Present = "task with state running error present" - Issue_Task_With_State_Running_ExpirationTime_Present = "task with state running expiration time present" IssueFormat_DataSource_Invalid = "data source invalid ('%s', '%s')" IssueFormat_DataSource_ProviderSession_Mismatch = "data source provider session mismatch ('%s', '%s')" @@ -162,13 +159,10 @@ func Issues() []string { Issue_Task_With_State_Failed_AvailableTime_Present, Issue_Task_With_State_Failed_DeadlineTime_Present, Issue_Task_With_State_Failed_Error_Missing, - Issue_Task_With_State_Failed_ExpirationTime_Present, Issue_Task_With_State_Pending_DeadlineTime_Present, - Issue_Task_With_State_Pending_ExpirationTime_Present, Issue_Task_With_State_Running_AvailableTime_Present, Issue_Task_With_State_Running_DeadlineTime_Missing, Issue_Task_With_State_Running_Error_Present, - Issue_Task_With_State_Running_ExpirationTime_Present, } } @@ -1108,9 +1102,6 @@ func (t *Tool) analyzeTasks() { if record.DeadlineTime != nil { record.AppendIssue(Issue_Task_With_State_Pending_DeadlineTime_Present) } - if record.ExpirationTime != nil { - record.AppendIssue(Issue_Task_With_State_Pending_ExpirationTime_Present) - } case task.TaskStateRunning: if record.AvailableTime != nil { record.AppendIssue(Issue_Task_With_State_Running_AvailableTime_Present) @@ -1121,9 +1112,6 @@ func (t *Tool) analyzeTasks() { if record.Error != nil { record.AppendIssue(Issue_Task_With_State_Running_Error_Present) } - if record.ExpirationTime != nil { - record.AppendIssue(Issue_Task_With_State_Running_ExpirationTime_Present) - } case task.TaskStateFailed: if record.AvailableTime != nil { record.AppendIssue(Issue_Task_With_State_Failed_AvailableTime_Present) @@ -1134,9 +1122,6 @@ func (t *Tool) analyzeTasks() { if record.Error == nil { record.AppendIssue(Issue_Task_With_State_Failed_Error_Missing) } - if record.ExpirationTime != nil { - record.AppendIssue(Issue_Task_With_State_Failed_ExpirationTime_Present) - } case task.TaskStateCompleted: record.AppendIssuef(IssueFormat_Task_State_Invalid, record.State) } @@ -1353,9 +1338,7 @@ func (t *Tool) outputIssue(issue string, marshalables Marshalables, issueMarshal t.outputMongoOperationf("db.tasks.updateMany({state: 'failed', deadlineTime: {$exists: true}}, {$unset: {deadlineTime: true}})") return case Issue_Task_With_State_Failed_Error_Missing: - case Issue_Task_With_State_Failed_ExpirationTime_Present: case Issue_Task_With_State_Pending_DeadlineTime_Present: - case Issue_Task_With_State_Pending_ExpirationTime_Present: case Issue_Task_With_State_Running_AvailableTime_Present: t.outputResolutionHeader("FIXED with BACK-3116. Will keep occurring until deployed. Will need to manually update failed tasks post-deploy.") // NOTE: Uncomment this block to see further details. @@ -1375,7 +1358,6 @@ func (t *Tool) outputIssue(issue string, marshalables Marshalables, issueMarshal t.outputMongoReadOperationsHeader() t.outputMongoTasksAggregation(marshalables.Tasks().IDs()) return - case Issue_Task_With_State_Running_ExpirationTime_Present: default: t.analyzeIssueFormat(issue, marshalables) return diff --git a/store/structured/condition.go b/store/structured/condition.go index cfc7e6cb95..4cb763bd0c 100644 --- a/store/structured/condition.go +++ b/store/structured/condition.go @@ -13,6 +13,12 @@ func NewCondition() *Condition { return &Condition{} } +func NewConditionWithRevision(revision *int) *Condition { + return &Condition{ + Revision: revision, + } +} + func MapCondition(condition *request.Condition) *Condition { if condition == nil { return nil diff --git a/store/structured/mongo/config.go b/store/structured/mongo/config.go index a8ab8ae012..5bb45a1e8b 100644 --- a/store/structured/mongo/config.go +++ b/store/structured/mongo/config.go @@ -85,7 +85,7 @@ func (c *Config) SetDatabaseFromReporter(configReporter platformConfig.Reporter) } // Validate that all parameters are syntactically valid, that all required parameters are present, -// and the the URL constructed from those parameters is parsable by the Mongo driver +// and the URL constructed from those parameters is parsable by the Mongo driver func (c *Config) Validate() error { if len(c.Addresses) == 0 { return errors.New("addresses is missing") diff --git a/store/structured/mongo/result.go b/store/structured/mongo/result.go index a41ea64408..4b03364299 100644 --- a/store/structured/mongo/result.go +++ b/store/structured/mongo/result.go @@ -1,8 +1,13 @@ package mongo import ( + "context" + "time" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/page" ) @@ -91,3 +96,19 @@ func BSONToAny(input any) any { return output } } + +func CloseCursor(ctx context.Context, cursor *mongo.Cursor) { + if cursor == nil { + return + } + if ctx != nil { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + } + if err := cursor.Close(ctx); err != nil { + if lgr := log.LoggerFromContext(ctx); lgr != nil { + lgr.WithError(err).Warn("Unable to close cursor") + } + } +} diff --git a/store/structured/mongo/result_test.go b/store/structured/mongo/result_test.go index ce9183e57d..a7e88c4a74 100644 --- a/store/structured/mongo/result_test.go +++ b/store/structured/mongo/result_test.go @@ -1,32 +1,39 @@ package mongo_test import ( + "context" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" - "github.com/tidepool-org/platform/store/structured/mongo" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" + storeStructuredMongoTest "github.com/tidepool-org/platform/store/structured/mongo/test" ) var _ = Describe("Result", func() { Describe("BSONToMap", func() { It("returns nil for nil input", func() { - Expect(mongo.BSONToMap(nil)).To(BeNil()) + Expect(storeStructuredMongo.BSONToMap(nil)).To(BeNil()) }) It("handles empty bson.M", func() { - Expect(mongo.BSONToMap(bson.M{})).To(Equal(map[string]any{})) + Expect(storeStructuredMongo.BSONToMap(bson.M{})).To(Equal(map[string]any{})) }) It("converts bson.M to map[string]any", func() { - result := mongo.BSONToMap(bson.M{"string": "value", "number": 42}) + result := storeStructuredMongo.BSONToMap(bson.M{"string": "value", "number": 42}) Expect(result).To(Equal(map[string]any{"string": "value", "number": 42})) }) It("handles deeply nested structures", func() { - result := mongo.BSONToMap(bson.M{"zero": bson.A{"nested", "array"}, "one": bson.M{"nested": "object"}}) + result := storeStructuredMongo.BSONToMap(bson.M{"zero": bson.A{"nested", "array"}, "one": bson.M{"nested": "object"}}) Expect(result).To(MatchAllKeys(Keys{ "zero": Equal([]any{"nested", "array"}), "one": Equal(map[string]any{"nested": "object"}), @@ -36,20 +43,20 @@ var _ = Describe("Result", func() { Describe("BSONToArray", func() { It("returns nil for nil input", func() { - Expect(mongo.BSONToArray(nil)).To(BeNil()) + Expect(storeStructuredMongo.BSONToArray(nil)).To(BeNil()) }) It("handles empty bson.A", func() { - Expect(mongo.BSONToArray(bson.A{})).To(Equal([]any{})) + Expect(storeStructuredMongo.BSONToArray(bson.A{})).To(Equal([]any{})) }) It("converts bson.A to []any", func() { - result := mongo.BSONToArray(bson.A{"value", 42}) + result := storeStructuredMongo.BSONToArray(bson.A{"value", 42}) Expect(result).To(Equal([]any{"value", 42})) }) It("handles deeply nested structures", func() { - result := mongo.BSONToArray(bson.A{bson.A{"nested", "array"}, bson.M{"nested": "object"}}) + result := storeStructuredMongo.BSONToArray(bson.A{bson.A{"nested", "array"}, bson.M{"nested": "object"}}) Expect(result).To(MatchAllElementsWithIndex(IndexIdentity, Elements{ "0": Equal([]any{"nested", "array"}), "1": Equal(map[string]any{"nested": "object"}), @@ -59,20 +66,20 @@ var _ = Describe("Result", func() { Describe("BSONToAny", func() { It("returns nil for nil input", func() { - Expect(mongo.BSONToAny(nil)).To(BeNil()) + Expect(storeStructuredMongo.BSONToAny(nil)).To(BeNil()) }) It("returns primitive types unchanged", func() { - Expect(mongo.BSONToAny(true)).To(Equal(true)) - Expect(mongo.BSONToAny(42)).To(Equal(42)) - Expect(mongo.BSONToAny(42.345)).To(Equal(42.345)) - Expect(mongo.BSONToAny("string")).To(Equal("string")) - Expect(mongo.BSONToAny(map[string]string{"string": "value", "number": "42"})).To(Equal(map[string]string{"string": "value", "number": "42"})) - Expect(mongo.BSONToAny([]string{"value", "42"})).To(Equal([]string{"value", "42"})) + Expect(storeStructuredMongo.BSONToAny(true)).To(Equal(true)) + Expect(storeStructuredMongo.BSONToAny(42)).To(Equal(42)) + Expect(storeStructuredMongo.BSONToAny(42.345)).To(Equal(42.345)) + Expect(storeStructuredMongo.BSONToAny("string")).To(Equal("string")) + Expect(storeStructuredMongo.BSONToAny(map[string]string{"string": "value", "number": "42"})).To(Equal(map[string]string{"string": "value", "number": "42"})) + Expect(storeStructuredMongo.BSONToAny([]string{"value", "42"})).To(Equal([]string{"value", "42"})) }) It("converts bson.M", func() { - result := mongo.BSONToAny(bson.M{"zero": bson.A{"nested", "array"}, "one": bson.M{"nested": "object"}}) + result := storeStructuredMongo.BSONToAny(bson.M{"zero": bson.A{"nested", "array"}, "one": bson.M{"nested": "object"}}) Expect(result).To(MatchAllKeys(Keys{ "zero": Equal([]any{"nested", "array"}), "one": Equal(map[string]any{"nested": "object"}), @@ -80,11 +87,52 @@ var _ = Describe("Result", func() { }) It("converts bson.A", func() { - result := mongo.BSONToAny(bson.A{bson.A{"nested", "array"}, bson.M{"nested": "object"}}) + result := storeStructuredMongo.BSONToAny(bson.A{bson.A{"nested", "array"}, bson.M{"nested": "object"}}) Expect(result).To(MatchAllElementsWithIndex(IndexIdentity, Elements{ "0": Equal([]any{"nested", "array"}), "1": Equal(map[string]any{"nested": "object"}), })) }) }) + + Describe("CloseCursor", func() { + var lgr *logTest.Logger + var ctx context.Context + var repository *storeStructuredMongo.Repository + + BeforeEach(func() { + lgr = logTest.NewLogger() + ctx = log.NewContextWithLogger(context.Background(), lgr) + repository = storeStructuredMongoTest.GetSuiteStore().GetRepository(storeStructuredMongoTest.NewCollectionPrefix()) + _, err := repository.InsertMany(ctx, []any{bson.M{"value": 1}, bson.M{"value": 2}}) + Expect(err).ToNot(HaveOccurred()) + }) + + openCursor := func() *mongo.Cursor { + cursor, err := repository.Find(ctx, bson.M{}, options.Find().SetBatchSize(1)) + Expect(err).ToNot(HaveOccurred()) + Expect(cursor.ID()).ToNot(BeZero()) + return cursor + } + + It("does not panic when the cursor is nil", func() { + Expect(func() { storeStructuredMongo.CloseCursor(ctx, nil) }).ToNot(Panic()) + }) + + It("does not panic when both the context and the cursor are nil", func() { + Expect(func() { storeStructuredMongo.CloseCursor(context.Context(nil), nil) }).ToNot(Panic()) + }) + + It("closes the cursor when the context is provided", func() { + cursor := openCursor() + storeStructuredMongo.CloseCursor(ctx, cursor) + Expect(cursor.ID()).To(BeZero()) + }) + + It("closes the cursor when the context is nil", func() { + cursor := openCursor() + storeStructuredMongo.CloseCursor(context.Context(nil), cursor) + Expect(cursor.ID()).To(BeZero()) + }) + }) }) diff --git a/summary/task/migrationrunner.go b/summary/task/migrationrunner.go index ea2a137394..ea4b37f62d 100644 --- a/summary/task/migrationrunner.go +++ b/summary/task/migrationrunner.go @@ -33,10 +33,8 @@ type MigrationRunner struct { func NewDefaultMigrationTaskCreate(summaryType string) *task.TaskCreate { typ := MigrationType + "." + summaryType return &task.TaskCreate{ - Name: pointer.FromAny(typ), - Type: typ, - Priority: 5, - AvailableTime: pointer.FromAny(time.Now().UTC()), + Name: pointer.FromAny(typ), + Type: typ, Data: map[string]any{ ConfigMinInterval: int32(DefaultMigrationAvailableAfterDurationMinimum.Seconds()), ConfigMaxInterval: int32(DefaultMigrationAvailableAfterDurationMaximum.Seconds()), @@ -71,8 +69,8 @@ func (r *MigrationRunner) GetRunnerType() string { return MigrationType + "." + r.summaryType } -func (r *MigrationRunner) GetRunnerDeadline() time.Time { - return time.Now().Add(MigrationTaskDurationMaximum * 3) +func (r *MigrationRunner) GetRunnerDeadline() time.Duration { + return MigrationTaskDurationMaximum * 3 } func (r *MigrationRunner) GetRunnerTimeout() time.Duration { diff --git a/summary/task/migrationrunner_test.go b/summary/task/migrationrunner_test.go index 2904ecc18b..14e88850a9 100644 --- a/summary/task/migrationrunner_test.go +++ b/summary/task/migrationrunner_test.go @@ -57,8 +57,6 @@ var _ = Describe("migrate runner tasks", func() { Expect(t.Name).ToNot(BeNil()) Expect(*t.Name).To(Equal("org.tidepool.summary.migrate.cgm")) Expect(t.Type).To(Equal("org.tidepool.summary.migrate.cgm")) - Expect(t.AvailableTime).ToNot(BeNil()) - Expect(t.AvailableTime.IsZero()).ToNot(BeTrue()) batch, ok := t.Data[ConfigBatch].(int32) Expect(ok).To(BeTrue()) diff --git a/summary/task/updaterunner.go b/summary/task/updaterunner.go index 899208d153..c923282dae 100644 --- a/summary/task/updaterunner.go +++ b/summary/task/updaterunner.go @@ -34,10 +34,8 @@ type UpdateRunner struct { func NewDefaultUpdateTaskCreate(summaryType string) *task.TaskCreate { typ := UpdateType + "." + summaryType return &task.TaskCreate{ - Name: pointer.FromAny(typ), - Type: typ, - Priority: 5, - AvailableTime: pointer.FromAny(time.Now().UTC()), + Name: pointer.FromAny(typ), + Type: typ, Data: map[string]any{ ConfigMinInterval: int32(DefaultUpdateAvailableAfterDurationMinimum.Seconds()), ConfigMaxInterval: int32(DefaultUpdateAvailableAfterDurationMaximum.Seconds()), @@ -69,8 +67,8 @@ func (r *UpdateRunner) GetRunnerType() string { return UpdateType + "." + r.summaryType } -func (r *UpdateRunner) GetRunnerDeadline() time.Time { - return time.Now().Add(UpdateTaskDurationMaximum * 3) +func (r *UpdateRunner) GetRunnerDeadline() time.Duration { + return UpdateTaskDurationMaximum * 3 } func (r *UpdateRunner) GetRunnerTimeout() time.Duration { diff --git a/summary/task/updaterunner_test.go b/summary/task/updaterunner_test.go index 5ea1833d88..64dddb1535 100644 --- a/summary/task/updaterunner_test.go +++ b/summary/task/updaterunner_test.go @@ -57,8 +57,6 @@ var _ = Describe("update runner tasks", func() { Expect(t.Name).ToNot(BeNil()) Expect(*t.Name).To(Equal("org.tidepool.summary.update.cgm")) Expect(t.Type).To(Equal("org.tidepool.summary.update.cgm")) - Expect(t.AvailableTime).ToNot(BeNil()) - Expect(t.AvailableTime.IsZero()).ToNot(BeTrue()) batch, ok := t.Data[ConfigBatch].(int32) Expect(ok).To(BeTrue()) diff --git a/task/client/client.go b/task/client/client.go index 7cc6dd4e2c..6c08793c4a 100644 --- a/task/client/client.go +++ b/task/client/client.go @@ -71,17 +71,22 @@ func (c *Client) CreateTask(ctx context.Context, create *task.TaskCreate) (*task return tsk, nil } -func (c *Client) GetTask(ctx context.Context, id string) (*task.Task, error) { +func (c *Client) GetTask(ctx context.Context, id string, condition *request.Condition) (*task.Task, error) { if ctx == nil { return nil, errors.New("context is missing") } if id == "" { return nil, errors.New("id is missing") } + if condition == nil { + condition = request.NewCondition() + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return nil, errors.Wrap(err, "condition is invalid") + } url := c.client.ConstructURL("v1", "tasks", id) tsk := &task.Task{} - if err := c.client.RequestData(ctx, http.MethodGet, url, nil, nil, tsk); err != nil { + if err := c.client.RequestData(ctx, http.MethodGet, url, []request.RequestMutator{condition}, nil, tsk); err != nil { if request.IsErrorResourceNotFound(err) { return nil, nil } @@ -91,13 +96,18 @@ func (c *Client) GetTask(ctx context.Context, id string) (*task.Task, error) { return tsk, nil } -func (c *Client) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { +func (c *Client) UpdateTask(ctx context.Context, id string, condition *request.Condition, update *task.TaskUpdate) (*task.Task, error) { if ctx == nil { return nil, errors.New("context is missing") } if id == "" { return nil, errors.New("id is missing") } + if condition == nil { + condition = request.NewCondition() + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return nil, errors.Wrap(err, "condition is invalid") + } if update == nil { return nil, errors.New("update is missing") } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(update); err != nil { @@ -106,7 +116,7 @@ func (c *Client) UpdateTask(ctx context.Context, id string, update *task.TaskUpd url := c.client.ConstructURL("v1", "tasks", id) tsk := &task.Task{} - if err := c.client.RequestData(ctx, http.MethodPut, url, nil, update, tsk); err != nil { + if err := c.client.RequestData(ctx, http.MethodPut, url, []request.RequestMutator{condition}, update, tsk); err != nil { if request.IsErrorResourceNotFound(err) { return nil, nil } @@ -116,14 +126,19 @@ func (c *Client) UpdateTask(ctx context.Context, id string, update *task.TaskUpd return tsk, nil } -func (c *Client) DeleteTask(ctx context.Context, id string) error { +func (c *Client) DeleteTask(ctx context.Context, id string, condition *request.Condition) error { if ctx == nil { return errors.New("context is missing") } if id == "" { return errors.New("id is missing") } + if condition == nil { + condition = request.NewCondition() + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return errors.Wrap(err, "condition is invalid") + } url := c.client.ConstructURL("v1", "tasks", id) - return c.client.RequestData(ctx, http.MethodDelete, url, nil, nil, nil) + return c.client.RequestData(ctx, http.MethodDelete, url, []request.RequestMutator{condition}, nil, nil) } diff --git a/task/queue/multi.go b/task/queue/multi.go index 47d2cd294d..bb0deb3aa4 100644 --- a/task/queue/multi.go +++ b/task/queue/multi.go @@ -1,63 +1,76 @@ package queue import ( + "maps" + "sync" + "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/task/store" ) -// MultiQueue creates a queue per registered runner +// MultiQueue runs a queue per runner provided at construction, each processing only tasks of +// that runner's type. Like Queue, it is single-use. The queues map is immutable after +// NewMultiQueue, so no synchronization is required. type MultiQueue struct { queues map[string]Queue - cfg *Config - lgr log.Logger - str store.Store } -func NewMultiQueue(cfg *Config, lgr log.Logger, str store.Store) (Queue, error) { - return &MultiQueue{ - queues: make(map[string]Queue), - cfg: cfg, - lgr: lgr, - str: str, - }, nil -} - -func (m *MultiQueue) RegisterRunner(runner Runner) error { - typ := runner.GetRunnerType() - if _, ok := m.queues[typ]; ok { - return errors.New("runner of the same type is already registered") +func NewMultiQueue(cfg *Config, lgr log.Logger, str store.Store, runners ...Runner) (*MultiQueue, error) { + if cfg == nil { + return nil, errors.New("config is missing") } - - str := m.str.WithTypeFilter(typ) - q, err := New(m.cfg, m.lgr, str) - if err != nil { - return err + if lgr == nil { + return nil, errors.New("logger is missing") } - if err := q.RegisterRunner(runner); err != nil { - return err + if str == nil { + return nil, errors.New("store is missing") } - m.queues[typ] = q - return nil + queues := make(map[string]Queue, len(runners)) + for _, runner := range runners { + if runner == nil { + return nil, errors.New("runner is missing") + } + + typ := runner.GetRunnerType() + if _, ok := queues[typ]; ok { + return nil, errors.New("runner type already registered") + } + + q, err := New(typ, cfg, lgr, str.WithTypeFilter(typ), runner) + if err != nil { + return nil, err + } + + queues[typ] = q + } + + return &MultiQueue{ + queues: queues, + }, nil } func (m *MultiQueue) Start() { for _, q := range m.queues { - q := q q.Start() } } func (m *MultiQueue) Stop() { + // Stop the queues concurrently so total shutdown latency is bounded by a single + // queue's stop timeout rather than the sum across every queue. + var waitGroup sync.WaitGroup for _, q := range m.queues { - q := q - q.Stop() + waitGroup.Go(q.Stop) } + waitGroup.Wait() } +// GetQueues returns a copy of the type-to-queue map so callers cannot mutate the +// internal map. The queue values are shared references, not copies. func (m *MultiQueue) GetQueues() map[string]Queue { - return m.queues + return maps.Clone(m.queues) } var _ Queue = &MultiQueue{} diff --git a/task/queue/multi_test.go b/task/queue/multi_test.go index 62f9c6cc77..5d5c5efb5e 100644 --- a/task/queue/multi_test.go +++ b/task/queue/multi_test.go @@ -28,6 +28,7 @@ var ( var _ = Describe("multi queue", func() { var config *storeStructuredMongo.Config + var queueConfig *queue.Config var lgr log.Logger var str *mongo.Store var multi *queue.MultiQueue @@ -39,37 +40,71 @@ var _ = Describe("multi queue", func() { Expect(err).ToNot(HaveOccurred()) Expect(str).ToNot(BeNil()) lgr = null.NewLogger() - - var q queue.Queue - q, err = queue.NewMultiQueue( - &queue.Config{ - Workers: 10, - Delay: 1, - }, - lgr, - str, - ) - Expect(err).ToNot(HaveOccurred()) - Expect(q).ToNot(BeNil()) - - var ok bool - multi, ok = q.(*queue.MultiQueue) - Expect(ok).To(BeTrue()) + queueConfig = &queue.Config{Workers: 10, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: queue.DelayUnstickDefault, StopWaitTimeout: queue.StopWaitTimeoutDefault} + multi = nil }) AfterEach(func() { + if multi != nil { + multi.Stop() + } Expect(str.Terminate(context.Background())).To(Succeed()) }) - Describe("Register Runner", func() { - It("Creates a new queue for each runner type", func() { - for _, t := range types { - runner := test.NewCountingRunner(t) - Expect(multi.RegisterRunner(runner)).To(Succeed()) - Expect(multi.GetQueues()).To(HaveKey(t)) + Describe("NewMultiQueue", func() { + It("creates a new queue for each runner type", func() { + runners := make([]queue.Runner, 0, len(types)) + for _, typ := range types { + runners = append(runners, test.NewCountingRunner(typ)) } + + var err error + multi, err = queue.NewMultiQueue(queueConfig, lgr, str, runners...) + Expect(err).ToNot(HaveOccurred()) + Expect(multi).ToNot(BeNil()) + queues := multi.GetQueues() Expect(queues).To(HaveLen(len(types))) + for _, typ := range types { + Expect(queues).To(HaveKey(typ)) + } + }) + + It("returns an error when the config is missing", func() { + invalidMulti, err := queue.NewMultiQueue(nil, lgr, str) + Expect(err).To(MatchError("config is missing")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when the logger is missing", func() { + invalidMulti, err := queue.NewMultiQueue(queueConfig, nil, str) + Expect(err).To(MatchError("logger is missing")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when the store is missing", func() { + invalidMulti, err := queue.NewMultiQueue(queueConfig, lgr, nil) + Expect(err).To(MatchError("store is missing")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when a runner is missing", func() { + invalidMulti, err := queue.NewMultiQueue(queueConfig, lgr, str, nil) + Expect(err).To(MatchError("runner is missing")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when two runners have the same type", func() { + invalidMulti, err := queue.NewMultiQueue(queueConfig, lgr, str, test.NewCountingRunner(types[0]), test.NewCountingRunner(types[0])) + Expect(err).To(MatchError("runner type already registered")) + Expect(invalidMulti).To(BeNil()) + }) + + It("returns an error when a runner has invalid durations", func() { + runner := test.NewSleepRunner(types[0], 2*time.Minute, 2*time.Minute, time.Minute, 0) + invalidMulti, err := queue.NewMultiQueue(queueConfig, lgr, str, runner) + Expect(err).To(MatchError("runner deadline is invalid")) + Expect(invalidMulti).To(BeNil()) }) }) @@ -82,20 +117,21 @@ var _ = Describe("multi queue", func() { It("Are partitioned correctly", func() { ctx := log.NewContextWithLogger(context.Background(), lgr) creates := make([]*task.TaskCreate, 0, len(types)*tasksPerType) - runners := make([]*test.CountingRunner, 0, len(types)) + countingRunners := make([]*test.CountingRunner, 0, len(types)) + runners := make([]queue.Runner, 0, len(types)) now := time.Now() // Create tasks and runners for each task type - for _, t := range types { - runner := test.NewCountingRunner(t) + for _, typ := range types { + runner := test.NewCountingRunner(typ) + countingRunners = append(countingRunners, runner) runners = append(runners, runner) - Expect(multi.RegisterRunner(runner)).To(Succeed()) - for i := 0; i < tasksPerType; i++ { - name := fmt.Sprintf("%v:%v", t, i) + for index := 0; index < tasksPerType; index++ { + name := fmt.Sprintf("%v:%v", typ, index) creates = append(creates, &task.TaskCreate{ Name: &name, - Type: t, + Type: typ, AvailableTime: &now, }) } @@ -104,25 +140,15 @@ var _ = Describe("multi queue", func() { // Insert tasks in the database rand.Shuffle(len(creates), func(i, j int) { creates[i], creates[j] = creates[j], creates[i] }) for _, create := range creates { - create := create tsk, err := str.NewTaskRepository().CreateTask(ctx, create) Expect(err).ToNot(HaveOccurred()) Expect(tsk).ToNot(BeNil()) } - // Register runners from all types in the underlying queue - // To make sure they are empty when all work is processed - expectedNoopRunners := make([]*test.CountingRunner, 0) - for typ, q := range multi.GetQueues() { - for _, t := range types { - if typ != t { - runner := test.NewCountingRunner(t) - expectedNoopRunners = append(expectedNoopRunners, runner) - Expect(q.RegisterRunner(runner)).To(Succeed()) - } - } - - } + var err error + multi, err = queue.NewMultiQueue(queueConfig, lgr, str, runners...) + Expect(err).ToNot(HaveOccurred()) + Expect(multi).ToNot(BeNil()) multi.Start() @@ -167,16 +193,11 @@ var _ = Describe("multi queue", func() { expected[typ] = tasksPerType } results := map[string]int{} - for _, runner := range runners { + for _, runner := range countingRunners { results[runner.GetRunnerType()] = runner.GetCount() } Expect(results).To(Equal(expected)) - - for _, runner := range expectedNoopRunners { - // Check extra runners didn't do any work - Expect(runner.GetCount()).To(Equal(0)) - } }) }) }) diff --git a/task/queue/queue.go b/task/queue/queue.go index ebc3a9f80b..f77238a99a 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -2,31 +2,96 @@ package queue import ( "context" - "math/rand" "runtime/debug" "strconv" "sync" "time" - "go.mongodb.org/mongo-driver/mongo" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/tidepool-org/platform/config" + "github.com/tidepool-org/platform/crypto" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/pointer" + storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" "github.com/tidepool-org/platform/task" - "github.com/tidepool-org/platform/task/store" + taskStore "github.com/tidepool-org/platform/task/store" +) + +var ( + // RunnerTimeoutExceededTotal counts task runs that exceeded the runner timeout without + // returning, sorted by type. A non-zero value indicates a runner that does not honor context + // cancellation, which leaves its worker blocked until the process restarts. + RunnerTimeoutExceededTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_runner_timeout_exceeded_total", + Help: "The total number of task runs that exceeded the runner timeout without returning, sorted by type", + }, []string{"type"}) + + // RunDurationSeconds observes how long each task run took, sorted by type. Use it to track + // run latency percentiles and to alert when durations approach the runner timeout. + RunDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tidepool_task_run_duration_seconds", + Help: "The duration of task runs in seconds, sorted by type", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 15), + }, []string{"type"}) + + // RunPanicTotal counts task runs that panicked and were recovered, sorted by type. + RunPanicTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_run_panic_total", + Help: "The total number of task runs that panicked, sorted by type", + }, []string{"type"}) + + // WorkersAvailable reports the number of idle workers per queue. A value pinned at zero + // indicates a saturated queue or workers wedged in non-cooperative runners. + WorkersAvailable = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_task_workers_available", + Help: "The number of idle task queue workers, sorted by queue", + }, []string{"queue"}) + + // WorkersTotal reports the configured number of workers per queue. + WorkersTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_task_workers_total", + Help: "The configured number of task queue workers, sorted by queue", + }, []string{"queue"}) +) + +const ( + WorkersDefault = 5 + DelayDefault = time.Minute + DelayInitialDefault = time.Minute + DelayUnstickDefault = 5 * time.Minute + + // StopWaitTimeoutDefault bounds how long Stop waits for in-flight tasks to observe + // cancellation and exit before abandoning them (they are recovered by the deadline + // and unstick mechanism). Kept under the typical Kubernetes termination grace period + // since we could use it up to twice (once for workers and once for manager) and we + // still want to leave time for the store to flush any pending writes. + StopWaitTimeoutDefault = 10 * time.Second + + DurationJitterFactor = 0.2 + + // TaskDeadlineDefault bounds how long a task is allowed to run before being forcefully + // reset if a runner for the task type is not registered. + TaskDeadlineDefault = time.Minute ) type Config struct { - Workers int - Delay time.Duration + Workers int + Delay time.Duration + DelayInitial time.Duration + DelayUnstick time.Duration + StopWaitTimeout time.Duration } func NewConfig() *Config { return &Config{ - Workers: 1, - Delay: 60 * time.Second, + Workers: WorkersDefault, + Delay: DelayDefault, + DelayInitial: DelayInitialDefault, + DelayUnstick: DelayUnstickDefault, + StopWaitTimeout: StopWaitTimeoutDefault, } } @@ -36,20 +101,39 @@ func (c *Config) Load(configReporter config.Reporter) error { } if workersString, err := configReporter.Get("workers"); err == nil { - var workers int64 - workers, err = strconv.ParseInt(workersString, 10, 0) - if err != nil { + if workers, parseErr := strconv.ParseInt(workersString, 10, 0); parseErr != nil { return errors.New("workers is invalid") + } else { + c.Workers = int(workers) } - c.Workers = int(workers) } if delayString, err := configReporter.Get("delay"); err == nil { - var delay int64 - delay, err = strconv.ParseInt(delayString, 10, 0) - if err != nil { + if delay, parseErr := strconv.ParseInt(delayString, 10, 0); parseErr != nil { return errors.New("delay is invalid") + } else { + c.Delay = time.Duration(delay) * time.Second + } + } + if delayInitialString, err := configReporter.Get("delay_initial"); err == nil { + if delayInitial, parseErr := strconv.ParseInt(delayInitialString, 10, 0); parseErr != nil { + return errors.New("delay initial is invalid") + } else { + c.DelayInitial = time.Duration(delayInitial) * time.Second + } + } + if delayUnstickString, err := configReporter.Get("delay_unstick"); err == nil { + if delayUnstick, parseErr := strconv.ParseInt(delayUnstickString, 10, 0); parseErr != nil { + return errors.New("delay unstick is invalid") + } else { + c.DelayUnstick = time.Duration(delayUnstick) * time.Second + } + } + if stopWaitTimeoutString, err := configReporter.Get("stop_wait_timeout"); err == nil { + if stopWaitTimeout, parseErr := strconv.ParseInt(stopWaitTimeoutString, 10, 0); parseErr != nil { + return errors.New("stop wait timeout is invalid") + } else { + c.StopWaitTimeout = time.Duration(stopWaitTimeout) * time.Second } - c.Delay = time.Duration(delay) * time.Second } return nil @@ -59,62 +143,84 @@ func (c *Config) Validate() error { if c.Workers < 1 { return errors.New("workers is invalid") } - if c.Delay < 0 { + if c.Delay <= 0 { return errors.New("delay is invalid") } - + if c.DelayInitial <= 0 { + return errors.New("delay initial is invalid") + } + if c.DelayUnstick <= 0 { + return errors.New("delay unstick is invalid") + } + if c.StopWaitTimeout <= 0 { + return errors.New("stop wait timeout is invalid") + } return nil } type Runner interface { - // The type of tasks that the runner supports. GetRunnerType() string - // The time after which the task manager will forcefully reset the task back to pending - // and available. This is calculated based upon the current time and a duration significantly - // longer that the task duration maximum. Normally this would only be used on a task that - // is in the running state even though it is not running (likely due to a system crash or interruption). - GetRunnerDeadline() time.Time + // The duration after which the task manager will forcefully reset the task back to pending + // and available. This is a duration significantly longer than the task duration maximum. Normally, + // this would only be used on a task that is in the running state even though it is not running + // (likely due to a system crash or interruption). Must exceed the runner timeout, or a task + // could be unstuck and re-claimed while the original run is still executing. + GetRunnerDeadline() time.Duration // The duration of a task where the task manager will forcefully cancel the task context to interrupt // the task and force completion. This is typically a duration somewhat longer than the task - // duration maximum. + // duration maximum. Must exceed the runner duration maximum. GetRunnerTimeout() time.Duration // The typical duration maximum of the task after which a warning will be displayed. + // Must be positive. GetRunnerDurationMaximum() time.Duration // Execute the specified task within the specified context. The context will be forcefully - // canceled after a duration specified by GetRunnerTimeout. + // canceled after a duration specified by GetRunnerTimeout. Before returning, the runner + // must move the task out of the running state (typically pending via RepeatAvailableAt or + // RepeatAvailableAfter to run again, or completed, or failed); a task left running when + // the runner returns is marked failed, unless the run was interrupted by queue shutdown, + // in which case it is reverted to pending. Run(ctx context.Context, tsk *task.Task) } +// Queue runs tasks via runners provided at construction. A queue is single-use: it may be +// started at most once and stopped at most once, and once stopped it cannot be restarted +// (create a new queue instead). Start after Stop, a second Start, and a second Stop are +// all no-ops. type Queue interface { - RegisterRunner(Runner) error Start() Stop() } +// The queue's fields are all immutable after New, except the lifecycle fields, which are +// guarded by the lifecycle mutex, and workersAvailable, which is owned exclusively by the +// manager goroutine. The workers and manager therefore read the channels and runners map +// freely, without synchronization. type queue struct { + name string + config *Config logger log.Logger - store store.Store - workers int - delay time.Duration + repository taskStore.TaskRepository runners map[string]Runner - workersCancelFunc context.CancelFunc + dispatchChannel chan *task.Task + completionChannel chan *task.Task + lifecycleMutex sync.Mutex + started bool + stopped bool + cancelFunc context.CancelFunc workersWaitGroup sync.WaitGroup - managerCancelFunc context.CancelFunc managerWaitGroup sync.WaitGroup workersAvailable int - dispatchChannel chan *task.Task - completionChannel chan *task.Task - timer *time.Timer - taskRepository store.TaskRepository - iterator *mongo.Cursor } -func New(cfg *Config, lgr log.Logger, str store.Store) (Queue, error) { +func New(name string, cfg *Config, lgr log.Logger, str taskStore.Store, runners ...Runner) (Queue, error) { + if name == "" { + return nil, errors.New("name is missing") + } if cfg == nil { return nil, errors.New("config is missing") } @@ -129,311 +235,465 @@ func New(cfg *Config, lgr log.Logger, str store.Store) (Queue, error) { return nil, errors.Wrap(err, "config is invalid") } - workers := cfg.Workers - delay := cfg.Delay + runnerMap := make(map[string]Runner, len(runners)) + for _, runner := range runners { + if runner == nil { + return nil, errors.New("runner is missing") + } + if _, ok := runnerMap[runner.GetRunnerType()]; ok { + return nil, errors.New("runner type already registered") + } + if err := validateRunner(runner); err != nil { + return nil, err + } + runnerMap[runner.GetRunnerType()] = runner + } return &queue{ - logger: lgr, - store: str, - workers: workers, - delay: delay, - runners: make(map[string]Runner), - dispatchChannel: make(chan *task.Task, workers), - completionChannel: make(chan *task.Task, workers), + name: name, + config: cfg, + logger: lgr.WithField("queue", name), + repository: str.NewTaskRepository(), + runners: runnerMap, + + // NOT buffered so a task is only handed off when a worker is ready to receive it. This ensures a dispatched task is never + // stranded in a buffer during shutdown. + dispatchChannel: make(chan *task.Task), + + // Buffered so that a worker can complete a task and hand it off to the manager even if the manager is busy dispatching other tasks. + completionChannel: make(chan *task.Task, cfg.Workers), }, nil } -func (q *queue) RegisterRunner(runner Runner) error { - if runner == nil { - return errors.New("runner is missing") +func (q *queue) Start() { + q.lifecycleMutex.Lock() + defer q.lifecycleMutex.Unlock() + + if q.started || q.stopped { + return } + q.started = true - q.runners[runner.GetRunnerType()] = runner - return nil + ctx, cancelFunc := context.WithCancel(log.NewContextWithLogger(context.Background(), q.logger)) + q.cancelFunc = cancelFunc + + q.startWorkers(ctx) + q.startManager(ctx) } -func (q *queue) Start() { - backgroundCtx := log.NewContextWithLogger(context.Background(), q.logger) - if q.workersCancelFunc == nil { - ctx, workersCancelFunc := context.WithCancel(backgroundCtx) - q.workersCancelFunc = workersCancelFunc +func (q *queue) Stop() { + // Hold the mutex for the entire stop, including the waits, so a concurrent Start cannot + // observe a partially stopped queue. + q.lifecycleMutex.Lock() + defer q.lifecycleMutex.Unlock() - q.startWorkers(ctx) + if q.stopped { + return } - if q.managerCancelFunc == nil { - ctx, managerCancelFunc := context.WithCancel(backgroundCtx) - q.managerCancelFunc = managerCancelFunc + q.stopped = true - q.startManager(ctx) + // Never started, so nothing to stop; the stopped flag ensures it never will be. + if !q.started { + return } -} -func (q *queue) Stop() { - if q.workersCancelFunc != nil { - q.workersCancelFunc() - q.workersCancelFunc = nil + lgr := q.logger.WithField("stopWaitTimeout", q.config.StopWaitTimeout) + + // Cancel the manager, so it stops dispatching new tasks and begins draining completions + // from the workers, and the workers, to interrupt any in-flight task. + q.cancelFunc() + + // Wait for all workers to exit, but only up to a bounded timeout so a runner that does + // not honor cancellation cannot block shutdown forever. If a worker is still running we + // must NOT close the channels: a stuck worker that later finishes would panic sending on + // a closed completion channel. Instead we leave the goroutines orphaned (reaped at process + // exit); the abandoned task stays running and is recovered by the deadline/unstick mechanism. + if !waitWithTimeout(&q.workersWaitGroup, q.config.StopWaitTimeout) { + lgr.Error("Task queue workers did not stop within timeout; abandoning in-flight tasks") + return } - q.workersWaitGroup.Wait() + // All workers have exited, so completion channel can be closed. close(q.completionChannel) - if q.managerCancelFunc != nil { - q.managerCancelFunc() - q.managerCancelFunc = nil + // Wait for manager to exit. This should be prompt now that the completion channel is + // closed, but bound it too in case a completion write is slow. + if !waitWithTimeout(&q.managerWaitGroup, q.config.StopWaitTimeout) { + lgr.Error("Task queue manager did not stop within timeout") + return } - q.managerWaitGroup.Wait() + // Manager has exited, so no further tasks will be dispatched. Because the + // dispatch channel is not buffered, no dispatched task can be stranded in it; any + // task the manager could not hand off was reverted to pending during dispatch. close(q.dispatchChannel) } func (q *queue) startWorkers(ctx context.Context) { - for q.workersAvailable = 0; q.workersAvailable < q.workers; q.workersAvailable++ { - q.startWorker(ctx) + for q.workersAvailable = 0; q.workersAvailable < q.config.Workers; q.workersAvailable++ { + q.startWorker(log.ContextWithField(ctx, "worker", q.workersAvailable)) } + WorkersAvailable.WithLabelValues(q.name).Set(float64(q.workersAvailable)) + WorkersTotal.WithLabelValues(q.name).Set(float64(q.config.Workers)) } func (q *queue) startWorker(ctx context.Context) { - q.workersWaitGroup.Add(1) - go func() { - defer q.workersWaitGroup.Done() + q.workersWaitGroup.Go(func() { + lgr := log.LoggerFromContext(ctx) + + lgr.Debug("Task queue worker started") for { - select { - case <-ctx.Done(): + if err := q.executeWorker(ctx); err != nil { + lgr.WithError(err).Debug("Task queue worker stopped") return - case tsk := <-q.dispatchChannel: - q.runTask(ctx, tsk) - q.completionChannel <- tsk } } - }() + }) +} + +func (q *queue) executeWorker(ctx context.Context) error { + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } + + select { + case <-ctx.Done(): + return context.Cause(ctx) + case tsk := <-q.dispatchChannel: + if tsk != nil { + q.runTask(ctx, tsk) + q.completionChannel <- tsk + } + } + + return nil } func (q *queue) runTask(ctx context.Context, tsk *task.Task) { - ctx = log.ContextWithField(ctx, "taskId", tsk.ID) + runner, ok := q.runners[tsk.Type] + if !ok { + tsk.AppendError(errors.New("runner not found for task")) + tsk.SetFailed() + return + } + + ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) defer func() { if err := recover(); err != nil { - log.LoggerFromContext(ctx).WithFields(log.Fields{"error": err, "stack": string(debug.Stack())}).Error("Unhandled panic") + lgr.WithFields(log.Fields{"error": err, "stack": string(debug.Stack())}).Error("Unhandled panic while running task") tsk.AppendError(errors.New("unhandled panic")) + RunPanicTotal.WithLabelValues(tsk.Type).Inc() } }() - if runner, ok := q.runners[tsk.Type]; ok { - - // If runner does not respect its own maximum duration, then enforce a context-based timeout. - // This forces the task to cancel via the context. - ctx, cancel := context.WithTimeout(ctx, runner.GetRunnerTimeout()) - defer cancel() - - startTime := time.Now() - - // Run the task via the runner - runner.Run(ctx, tsk) - - if taskDuration := time.Since(startTime); taskDuration > runner.GetRunnerDurationMaximum() { - log.LoggerFromContext(ctx).WithField("taskDuration", taskDuration.Truncate(time.Millisecond).Seconds()).Warn("Task duration exceeds maximum") + startTime := time.Now() + + // If runner does not respect its own maximum duration, then enforce a context-based timeout. + // This forces the task to cancel via the context. + runnerContext, cancel := context.WithTimeoutCause(ctx, runner.GetRunnerTimeout(), errors.New("task runner timeout exceeded")) + defer cancel() + + // Watchdog for a runner that ignores cancellation. Go cannot preempt a goroutine, so if the + // runner blows past its timeout without returning, this worker stays blocked until the + // process restarts (the task itself is recovered by the deadline/unstick mechanism). We + // cannot unblock the worker, but we surface the condition via a log and metric so a + // non-cooperative runner is detectable rather than silent. + runnerWatchdog := time.AfterFunc(runner.GetRunnerTimeout(), func() { + lgr.Error("Task runner exceeded timeout without returning; worker is blocked until it returns") + RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType()).Inc() + }) + defer runnerWatchdog.Stop() + + // Run the task via the runner + runner.Run(runnerContext, tsk) + + // If the runner left the task running, reconcile its state based on why the run ended. + if tsk.State == task.TaskStateRunning { + switch { + case context.Cause(ctx) != nil: + // The parent (worker) context was canceled by shutdown; make the task available + // again for retry rather than treating the interruption as a completion. + tsk.RepeatAvailableAfter(0) + case context.Cause(runnerContext) != nil: + // The runner exceeded its timeout; record the cause so the failure is attributed + // to the timeout rather than the generic missing terminal state error. + tsk.AppendError(context.Cause(runnerContext)) } - } else { - tsk.AppendError(errors.New("runner not found for task")) - tsk.SetFailed() + } + + taskDuration := time.Since(startTime) + RunDurationSeconds.WithLabelValues(runner.GetRunnerType()).Observe(taskDuration.Seconds()) + if taskDuration > runner.GetRunnerDurationMaximum() { + lgr.WithField("taskDuration", taskDuration.Truncate(time.Millisecond).Seconds()).Warn("Task duration exceeds maximum") } } func (q *queue) startManager(ctx context.Context) { - q.managerWaitGroup.Add(1) + q.managerWaitGroup.Go(func() { + lgr := log.LoggerFromContext(ctx) - go func() { - defer q.managerWaitGroup.Done() + lgr.Info("Task queue manager started") - q.startTimer(time.Duration(rand.Int63n(int64(q.delay)) + 1)) - defer q.stopTimer() + // Start at a random future time to help prevent thundering herd problem + select { + case <-ctx.Done(): + lgr.WithError(context.Cause(ctx)).Debug("Task queue manager stopped before dispatching tasks") + return + case <-time.After(randomDuration(q.config.DelayInitial)): + lgr.Debug("Task queue manager initial delay complete") + } - // pick a starting random time in a future cycle to ensure multiple daemons don't do this exactly at the same - // time, it is not an error condition if it does, but could stress the db if the collection gets large - nextUnstickTime := time.Now().Add(time.Duration(rand.Int63n(int64(q.delay * 15)))) + // Start at a random future time to help prevent thundering herd problem + unstickTime := time.Now().Add(randomDuration(q.config.DelayUnstick)) for { - if nextUnstickTime.Before(time.Now()) { - q.unstickTasks(ctx) - nextUnstickTime = time.Now().Add(q.delay * 15) - } + if err := q.executeManager(ctx); err != nil { + lgr.WithError(err).Debug("Task queue manager stopping") - select { - case <-ctx.Done(): // Drain and complete any interrupted tasks + // Complete any remaining tasks concurrently so the total drain time is + // bounded by a single slow completion write rather than the sum across + // all workers' tasks, keeping shutdown within the stop wait timeout. + var completionWaitGroup sync.WaitGroup for tsk := range q.completionChannel { - q.completeTask(ctx, tsk) + if tsk != nil { + completionWaitGroup.Go(func() { q.completeTask(ctx, tsk) }) + } } + completionWaitGroup.Wait() + + lgr.WithError(err).Debug("Task queue manager stopped") return - case tsk := <-q.completionChannel: - if tsk != nil { - q.stopTimer() - q.completeTask(ctx, tsk) - q.startTimer(q.dispatchTasks(ctx)) - } - case <-q.timer.C: - q.startTimer(q.dispatchTasks(ctx)) + } + + if unstickTime.Before(time.Now()) { + q.unstickTasks(ctx) + unstickTime = time.Now().Add(durationWithJitter(q.config.DelayUnstick)) } } - }() + }) +} + +func (q *queue) executeManager(ctx context.Context) error { + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } + + select { + case <-ctx.Done(): + return context.Cause(ctx) + case tsk := <-q.completionChannel: + if tsk != nil { + q.completeTask(ctx, tsk) + q.workersAvailable++ + WorkersAvailable.WithLabelValues(q.name).Set(float64(q.workersAvailable)) + q.dispatchTasks(ctx) + } + case <-time.After(durationWithJitter(q.config.Delay)): + q.dispatchTasks(ctx) + } + + return nil } func (q *queue) unstickTasks(ctx context.Context) { - repository := q.store.NewTaskRepository() - count, err := repository.UnstickTasks(ctx) - if err != nil { - q.logger.WithError(err).Error("Failure in unsticking tasks") + ids, err := q.repository.UnstickTasks(ctx) + if count := len(ids); count > 0 { + log.LoggerFromContext(ctx).WithFields(log.Fields{"count": count, "ids": ids}).Info("Unstuck tasks") } - if count > 0 { - q.logger.WithField("unstickCount", count).Warn("Unstuck tasks") + + // Log error unless context was canceled + if err != nil { + if context.Cause(ctx) == nil { + log.LoggerFromContext(ctx).WithError(err).Error("Unable to unstick tasks") + } } } -func (q *queue) dispatchTasks(ctx context.Context) time.Duration { - defer q.stopPendingIterator(ctx) - for q.workersAvailable > 0 { - iter, err := q.startPendingIterator(ctx) - if err != nil { - q.logger.WithError(err).Error("Failure starting pending iterator") - return q.delay +func (q *queue) dispatchTasks(ctx context.Context) { + if q.workersAvailable < 1 { + return + } + + lgr := log.LoggerFromContext(ctx) + + // Iterate across all pending tasks + cursor, err := q.repository.IteratePending(ctx) + if err != nil { + if context.Cause(ctx) == nil { + lgr.WithError(err).Error("Unable to open task iterator") } + return + } + defer storeStructuredMongo.CloseCursor(ctx, cursor) + // Loop until no more workers available or no more pending tasks + for q.workersAvailable > 0 && cursor.Next(ctx) { tsk := &task.Task{} - if iter.Next(ctx) { - if err := iter.Decode(tsk); err != nil { - q.logger.WithError(err).Error("Failure iterating tasks") - return q.delay - } - q.dispatchTask(ctx, tsk) - } else { - if err := iter.Err(); err != nil { - q.logger.WithError(err).Error("Failure iterating pending tasks") - } - return q.delay + if err = cursor.Decode(tsk); err != nil { + lgr.WithError(err).Error("Unable to decode task") + } else if err = q.dispatchTask(ctx, tsk); err != nil { + lgr.WithError(err).Error("Unable to dispatch task") + return } } - return q.delay + // Log cursor error unless context was canceled + if err := cursor.Err(); err != nil { + if context.Cause(ctx) == nil { + lgr.WithError(err).Error("Unable to iterate tasks") + } + } } -func (q *queue) dispatchTask(ctx context.Context, tsk *task.Task) { - ctx = log.ContextWithField(ctx, "taskId", tsk.ID) +func (q *queue) dispatchTask(ctx context.Context, tsk *task.Task) error { + ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) - repository := q.store.NewTaskRepository() - - tsk.State = task.TaskStateRunning - tsk.AvailableTime = nil - tsk.RunTime = pointer.FromAny(time.Now()) - - // we don't error here if missing, as the task will be failed during runTask + // we don't error here if missing, as the task will be failed during runTask, which persists error to database + var deadline time.Duration if runner, ok := q.runners[tsk.Type]; ok { - tsk.DeadlineTime = pointer.FromAny(runner.GetRunnerDeadline()) + deadline = runner.GetRunnerDeadline() + } else { + deadline = TaskDeadlineDefault } - var err error - tsk, err = repository.UpdateFromState(context.WithoutCancel(ctx), tsk, task.TaskStatePending) + // StartTask completes regardless of context cancellation, so its outcome is definitive: + // a non-nil startedTask means the claim committed with a known state lock. + startedTask, err := q.repository.StartTask(ctx, tsk.ID, tsk.Revision, deadline) if err != nil { - if errors.Is(err, task.AlreadyClaimedTask) { - log.LoggerFromContext(ctx).Warnf("Failure to claim task %s (%s) as it is already in progress or is no longer available.", tsk.Name, tsk.ID) - return + return errors.Wrap(err, "unable to start task") + } else if startedTask == nil { + lgr.Info("Task no longer available to start") + return nil + } + + // Hand the task off to a worker. If the queue is shutting down before a worker can + // receive it, revert the task to pending rather than blocking the manager. The revert + // uses the started task state lock so it reliably matches. + select { + case <-ctx.Done(): + if err := q.repository.StopTask(ctx, startedTask.ID, startedTask.StateLock, task.TaskStatePending, nil, nil); err != nil { + return errors.Wrap(err, "unable to revert task to pending") } - - log.LoggerFromContext(ctx).WithError(err).Error("Failure to update state during dispatch task") - return + case q.dispatchChannel <- startedTask: + q.workersAvailable-- + WorkersAvailable.WithLabelValues(q.name).Set(float64(q.workersAvailable)) } - q.workersAvailable-- - q.dispatchChannel <- tsk + return nil } +// completeTask persists the task's completion. It deliberately does not touch +// workersAvailable (the caller accounts for the freed worker where relevant), so it is safe +// to call concurrently during the manager's shutdown drain. func (q *queue) completeTask(ctx context.Context, tsk *task.Task) { - ctx = log.ContextWithField(ctx, "taskId", tsk.ID) - - q.workersAvailable++ - - repository := q.store.NewTaskRepository() + ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) - q.computeState(tsk) + q.computeState(ctx, tsk) - if tsk.State != task.TaskStatePending { - tsk.AvailableTime = nil - } - tsk.DeadlineTime = nil + var duration *time.Duration if tsk.RunTime != nil { - tsk.Duration = pointer.FromFloat64(time.Since(*tsk.RunTime).Truncate(time.Millisecond).Seconds()) + // Clamp to a zero minimum: the run time round-trips through the database without a + // monotonic reading, so a backwards wall clock step could yield a non-positive elapsed + // time, which StopTask would reject, losing the completion. + duration = pointer.From(max(time.Since(*tsk.RunTime), 0)) } - // Without cancel to ensure task is updated in the database - _, err := repository.UpdateFromState(context.WithoutCancel(ctx), tsk, task.TaskStateRunning) - if err != nil { - log.LoggerFromContext(ctx).WithError(err).Error("Failure to update state during complete task") + if tsk.HasError() { + lgr.Error("Error occurred while running task") } - if tsk.HasError() { - log.LoggerFromContext(ctx).WithError(tsk.Error.Error).Error("Error occurred while running task") + // Data and Error use non-nil wrappers so that a task whose data or error was cleared during + // the run has the corresponding field unset in the database, rather than left stale (a nil + // wrapper means "leave unchanged" to StopTask). + update := &task.TaskUpdate{ + Data: pointer.From(tsk.Data), + AvailableTime: tsk.AvailableTime, + Error: &errors.Serializable{Error: tsk.GetError()}, + } + if err := q.repository.StopTask(ctx, tsk.ID, tsk.StateLock, tsk.State, duration, update); err != nil { + lgr.WithError(err).Error("Unable to complete task") } } -func (q *queue) computeState(tsk *task.Task) { +func (q *queue) computeState(ctx context.Context, tsk *task.Task) { switch tsk.State { case task.TaskStatePending: - if tsk.AvailableTime == nil || time.Now().After(*tsk.AvailableTime) { - // This used to error out here, but was considered too defensive. - tsk.AvailableTime = pointer.FromAny(time.Now()) + now := time.Now().UTC() + if tsk.AvailableTime == nil { + log.LoggerFromContext(ctx).Warn("Available time missing for pending task") + tsk.AvailableTime = pointer.FromTime(now) + } else if tsk.AvailableTime.Before(now) { + if tsk.AvailableTime.Before(now.Add(-time.Minute)) { // Allow some leeway to prevent spurious warnings + log.LoggerFromContext(ctx).Warn("Available time significantly before now for pending task") + } + tsk.AvailableTime = pointer.FromTime(now) } case task.TaskStateRunning: - if tsk.HasError() { - tsk.SetFailed() - } else { - tsk.SetCompleted() + // The runner returned without moving the task out of the running state, violating + // the runner contract; fail the task rather than guessing whether it succeeded. + if !tsk.HasError() { + tsk.AppendError(errors.New("runner failed to set terminal task state")) } + tsk.SetFailed() case task.TaskStateFailed, task.TaskStateCompleted: default: - tsk.AppendError(errors.New("unknown state")) + tsk.AppendError(errors.New("unknown task state")) tsk.SetFailed() } } -func (q *queue) startTimer(delay time.Duration) { - if delay > 0 { - if q.timer == nil { - q.timer = time.NewTimer(delay) - } else { - q.timer.Reset(delay) - } +// validateRunner enforces the runner duration contract, deadline > timeout > duration maximum > 0, +// so that a misconfigured runner fails queue construction rather than causing subtle runtime +// misbehavior (a non-positive deadline prevents the task from ever starting, while a deadline +// that does not exceed the timeout allows the unstick mechanism to reset a task that is still +// running). +func validateRunner(runner Runner) error { + if durationMaximum := runner.GetRunnerDurationMaximum(); durationMaximum <= 0 { + return errors.New("runner duration maximum is invalid") + } else if timeout := runner.GetRunnerTimeout(); timeout <= durationMaximum { + return errors.New("runner timeout is invalid") + } else if runner.GetRunnerDeadline() <= timeout { + return errors.New("runner deadline is invalid") + } else { + return nil } } -func (q *queue) stopTimer() { - if q.timer != nil { - if !q.timer.Stop() { - <-q.timer.C - } +// waitWithTimeout waits for the wait group to complete, returning true if it completed within +// the timeout and false otherwise. On timeout the internal waiter goroutine is left running +// until the wait group eventually completes (or the process exits). +func waitWithTimeout(waitGroup *sync.WaitGroup, timeout time.Duration) bool { + done := make(chan struct{}) + go func() { + waitGroup.Wait() + close(done) + }() + + select { + case <-done: + return true + case <-time.After(timeout): + return false } } -func (q *queue) startPendingIterator(ctx context.Context) (*mongo.Cursor, error) { - if q.taskRepository == nil { - q.taskRepository = q.store.NewTaskRepository() - } - if q.iterator == nil { - if iterator, err := q.taskRepository.IteratePending(ctx); err != nil { - return nil, err - } else { - q.iterator = iterator - } +func randomDuration(duration time.Duration) time.Duration { + if duration <= 0 { + return 0 } - return q.iterator, nil + return time.Duration(crypto.RandomInt64N(int64(duration))) } -func (q *queue) stopPendingIterator(ctx context.Context) { - if q.iterator != nil { - if err := q.iterator.Close(context.WithoutCancel(ctx)); err != nil { - q.logger.WithError(err).Warn("failure closing pending iterator") - } - q.iterator = nil - } - if q.taskRepository != nil { - q.taskRepository = nil +func durationWithJitter(duration time.Duration) time.Duration { + if duration <= 0 { + return 0 } + jitter := time.Duration(float64(duration) * DurationJitterFactor) + return duration + (randomDuration(jitter*2) - jitter) } diff --git a/task/queue/queue_internal_test.go b/task/queue/queue_internal_test.go deleted file mode 100644 index a6c0767cdd..0000000000 --- a/task/queue/queue_internal_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package queue - -import ( - "context" - "fmt" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - - logTest "github.com/tidepool-org/platform/log/test" - "github.com/tidepool-org/platform/task/store" -) - -type failingIteratorStore struct { - store.Store -} - -func (f *failingIteratorStore) NewTaskRepository() store.TaskRepository { - return &failingIteratorRepository{} -} - -type failingIteratorRepository struct { - store.TaskRepository -} - -func (f *failingIteratorRepository) IteratePending(ctx context.Context) ( - *mongo.Cursor, error) { - - return mongo.NewCursorFromDocuments([]interface{}{bson.D{}}, - fmt.Errorf("batch fetch failed"), nil) -} - -var _ = Describe("Queue", func() { - Context("dispatchTasks", func() { - It("logs an error when the pending iterator fails mid-iteration", func() { - logger := logTest.NewLogger() - q := &queue{ - logger: logger, - store: &failingIteratorStore{}, - workersAvailable: 1, - delay: time.Minute, - } - - Expect(q.dispatchTasks(context.Background())).To(Equal(time.Minute)) - logger.AssertError("Failure iterating pending tasks") - }) - }) -}) diff --git a/task/queue/queue_test.go b/task/queue/queue_test.go index ff18fa0800..fe1ac485d5 100644 --- a/task/queue/queue_test.go +++ b/task/queue/queue_test.go @@ -1,8 +1,822 @@ package queue_test import ( + "context" + "sync" + "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/prometheus/client_golang/prometheus/testutil" + "go.mongodb.org/mongo-driver/bson" + + configTest "github.com/tidepool-org/platform/config/test" + "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + "github.com/tidepool-org/platform/pointer" + storeStructuredMongoTest "github.com/tidepool-org/platform/store/structured/mongo/test" + "github.com/tidepool-org/platform/task" + taskQueue "github.com/tidepool-org/platform/task/queue" + taskQueueTest "github.com/tidepool-org/platform/task/queue/test" + taskStore "github.com/tidepool-org/platform/task/store" + taskStoreMongo "github.com/tidepool-org/platform/task/store/mongo" + taskStoreTest "github.com/tidepool-org/platform/task/store/test" + taskTest "github.com/tidepool-org/platform/task/test" + "github.com/tidepool-org/platform/test" ) -var _ = Describe("queue", func() { +var _ = Describe("Queue", func() { + It("WorkersDefault is expected", func() { + Expect(taskQueue.WorkersDefault).To(Equal(5)) + }) + + It("DelayDefault is expected", func() { + Expect(taskQueue.DelayDefault).To(Equal(time.Minute)) + }) + + It("DelayInitialDefault is expected", func() { + Expect(taskQueue.DelayInitialDefault).To(Equal(time.Minute)) + }) + + It("DelayUnstickDefault is expected", func() { + Expect(taskQueue.DelayUnstickDefault).To(Equal(5 * time.Minute)) + }) + + It("StopWaitTimeoutDefault is expected", func() { + Expect(taskQueue.StopWaitTimeoutDefault).To(Equal(10 * time.Second)) + }) + + It("DurationJitterFactor is expected", func() { + Expect(taskQueue.DurationJitterFactor).To(Equal(0.2)) + }) + + It("TaskDeadlineDefault is expected", func() { + Expect(taskQueue.TaskDeadlineDefault).To(Equal(time.Minute)) + }) + + Context("Config", func() { + Context("NewConfig", func() { + It("returns successfully", func() { + cfg := taskQueue.NewConfig() + Expect(cfg).ToNot(BeNil()) + }) + + It("returns default values", func() { + cfg := taskQueue.NewConfig() + Expect(cfg).ToNot(BeNil()) + Expect(cfg.Workers).To(Equal(taskQueue.WorkersDefault)) + Expect(cfg.Delay).To(Equal(taskQueue.DelayDefault)) + Expect(cfg.DelayInitial).To(Equal(taskQueue.DelayInitialDefault)) + Expect(cfg.DelayUnstick).To(Equal(taskQueue.DelayUnstickDefault)) + Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) + }) + }) + + Context("with new config", func() { + var cfg *taskQueue.Config + + BeforeEach(func() { + cfg = taskQueue.NewConfig() + }) + + Context("Load", func() { + var configReporter *configTest.Reporter + + BeforeEach(func() { + configReporter = configTest.NewReporter() + }) + + It("returns an error when the config reporter is missing", func() { + Expect(cfg.Load(nil)).To(MatchError("config reporter is missing")) + }) + + It("returns an error when workers is not parsable", func() { + configReporter.Config["workers"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("workers is invalid")) + }) + + It("returns an error when delay is not parsable", func() { + configReporter.Config["delay"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("delay is invalid")) + }) + + It("returns an error when delay initial is not parsable", func() { + configReporter.Config["delay_initial"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("delay initial is invalid")) + }) + + It("returns an error when delay unstick is not parsable", func() { + configReporter.Config["delay_unstick"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("delay unstick is invalid")) + }) + + It("returns an error when stop wait timeout is not parsable", func() { + configReporter.Config["stop_wait_timeout"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("stop wait timeout is invalid")) + }) + + It("uses existing workers if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.Workers).To(Equal(taskQueue.WorkersDefault)) + }) + + It("uses existing delay if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.Delay).To(Equal(taskQueue.DelayDefault)) + }) + + It("uses existing delay initial if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.DelayInitial).To(Equal(taskQueue.DelayInitialDefault)) + }) + + It("uses existing delay unstick if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.DelayUnstick).To(Equal(taskQueue.DelayUnstickDefault)) + }) + + It("uses existing stop wait timeout if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) + }) + + It("returns successfully and uses values from the config reporter", func() { + configReporter.Config["workers"] = "5" + configReporter.Config["delay"] = "30" + configReporter.Config["delay_initial"] = "45" + configReporter.Config["delay_unstick"] = "60" + configReporter.Config["stop_wait_timeout"] = "15" + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.Workers).To(Equal(5)) + Expect(cfg.Delay).To(Equal(30 * time.Second)) + Expect(cfg.DelayInitial).To(Equal(45 * time.Second)) + Expect(cfg.DelayUnstick).To(Equal(60 * time.Second)) + Expect(cfg.StopWaitTimeout).To(Equal(15 * time.Second)) + }) + }) + + Context("Validate", func() { + It("returns an error when workers is less than 1", func() { + cfg.Workers = 0 + Expect(cfg.Validate()).To(MatchError("workers is invalid")) + }) + + It("returns an error when delay is invalid", func() { + cfg.Delay = 0 + Expect(cfg.Validate()).To(MatchError("delay is invalid")) + }) + + It("returns an error when delay initial is invalid", func() { + cfg.DelayInitial = 0 + Expect(cfg.Validate()).To(MatchError("delay initial is invalid")) + }) + + It("returns an error when delay unstick is invalid", func() { + cfg.DelayUnstick = 0 + Expect(cfg.Validate()).To(MatchError("delay unstick is invalid")) + }) + + It("returns an error when stop wait timeout is invalid", func() { + cfg.StopWaitTimeout = 0 + Expect(cfg.Validate()).To(MatchError("stop wait timeout is invalid")) + }) + + It("returns successfully", func() { + Expect(cfg.Validate()).To(Succeed()) + }) + }) + }) + }) + + Context("New", func() { + var cfg *taskQueue.Config + var lgr *logTest.Logger + var str *taskStoreTest.Store + + BeforeEach(func() { + cfg = taskQueue.NewConfig() + lgr = logTest.NewLogger() + str = taskStoreTest.NewStore() + }) + + It("returns an error when name is missing", func() { + que, err := taskQueue.New("", cfg, lgr, str) + Expect(err).To(MatchError("name is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when config is missing", func() { + que, err := taskQueue.New(taskTest.RandomType(), nil, lgr, str) + Expect(err).To(MatchError("config is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when logger is missing", func() { + que, err := taskQueue.New(taskTest.RandomType(), cfg, nil, str) + Expect(err).To(MatchError("logger is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when store is missing", func() { + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, nil) + Expect(err).To(MatchError("store is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when config is invalid", func() { + cfg.Workers = 0 + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str) + Expect(err).To(MatchError("config is invalid; workers is invalid")) + Expect(que).To(BeNil()) + }) + + It("returns an error when a runner is missing", func() { + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, nil) + Expect(err).To(MatchError("runner is missing")) + Expect(que).To(BeNil()) + }) + + It("returns an error when multiple runners have the same type", func() { + typ := taskTest.RandomType() + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, taskQueueTest.NewCountingRunner(typ), taskQueueTest.NewCountingRunner(typ)) + Expect(err).To(MatchError("runner type already registered")) + Expect(que).To(BeNil()) + }) + + It("returns an error when a runner duration maximum is not positive", func() { + runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 3*time.Minute, 2*time.Minute, 0, 0) + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner) + Expect(err).To(MatchError("runner duration maximum is invalid")) + Expect(que).To(BeNil()) + }) + + It("returns an error when a runner timeout does not exceed its duration maximum", func() { + runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 3*time.Minute, time.Minute, time.Minute, 0) + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner) + Expect(err).To(MatchError("runner timeout is invalid")) + Expect(que).To(BeNil()) + }) + + It("returns an error when a runner deadline does not exceed its timeout", func() { + runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 2*time.Minute, 2*time.Minute, time.Minute, 0) + que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner) + Expect(err).To(MatchError("runner deadline is invalid")) + Expect(que).To(BeNil()) + }) + + It("returns successfully", func() { + str.NewTaskRepositoryOutputs = []taskStore.TaskRepository{nil} + que := test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str)) + Expect(que).ToNot(BeNil()) + }) + + It("returns successfully with multiple runners of different types", func() { + str.NewTaskRepositoryOutputs = []taskStore.TaskRepository{nil} + que := test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, taskQueueTest.NewCountingRunner(taskTest.RandomType()), taskQueueTest.NewCountingRunner(taskTest.RandomType()))) + Expect(que).ToNot(BeNil()) + }) + }) + + Context("with a new queue", func() { + var que taskQueue.Queue + + BeforeEach(func() { + cfg := taskQueue.NewConfig() + lgr := logTest.NewLogger() + str := taskStoreTest.NewStore() + str.NewTaskRepositoryOutputs = []taskStore.TaskRepository{nil} + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, taskQueueTest.NewCountingRunner(taskTest.RandomType()))) + }) + + Context("Start", func() { + It("does nothing when called after Stop, since the queue is single-use", func() { + que.Stop() + Expect(func() { que.Start() }).ToNot(Panic()) + // A no-op Start launches no goroutines, so a subsequent Stop stays a no-op too. + Expect(func() { que.Stop() }).ToNot(Panic()) + }) + + It("is safe to call more than once", func() { + que.Start() + Expect(func() { que.Start() }).ToNot(Panic()) + que.Stop() + }) + }) + + Context("Stop", func() { + It("does not panic when called without a prior call to Start", func() { + Expect(func() { que.Stop() }).ToNot(Panic()) + }) + + It("succeeds if called twice", func() { + Expect(func() { que.Stop() }).ToNot(Panic()) + Expect(func() { que.Stop() }).ToNot(Panic()) + }) + }) + }) + + Context("with a store", func() { + var lgr *logTest.Logger + var ctx context.Context + var str *taskStoreMongo.Store + + BeforeEach(func() { + lgr = logTest.NewLogger() + ctx = log.NewContextWithLogger(context.Background(), lgr) + + cfg := storeStructuredMongoTest.NewConfig() + str = test.Must(taskStoreMongo.NewStore(cfg)) + _ = test.Must(str.GetRepository("tasks").DeleteMany(ctx, bson.M{})) + }) + + AfterEach(func() { + _ = test.Must(str.GetRepository("tasks").DeleteMany(ctx, bson.M{})) + if str != nil { + Expect(str.Terminate(ctx)).To(Succeed()) + } + }) + + Context("with a single queue", func() { + var countingRunner *taskQueueTest.CountingRunner + var panicRunner *taskQueueTest.PanicRunner + var cfg *taskQueue.Config + var que taskQueue.Queue + + BeforeEach(func() { + countingRunner = taskQueueTest.NewCountingRunner(taskTest.RandomType()) + panicRunner = taskQueueTest.NewPanicRunner(taskTest.RandomType()) + + cfg = &taskQueue.Config{Workers: 2, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault} + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, countingRunner, panicRunner)) + }) + + AfterEach(func() { + que.Stop() + lgr.AssertDebug("Task queue worker stopped", log.Fields{"error": errors.NewSerializable(context.Canceled)}) + lgr.AssertDebug("Task queue manager stopped", log.Fields{"error": errors.NewSerializable(context.Canceled)}) + }) + + It("dispatches, runs, and completes a pending task matching a registered runner", func() { + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: countingRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + Expect(countingRunner.GetCount()).To(Equal(1)) + }) + + It("completes a task that the runner updated while it was running", func() { + updatingRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { + // A runner may update its own task mid-run. The update bumps the revision but + // leaves the state lock intact, so the queue's completion must still match. + data := map[string]any{"key": "value"} + updated, err := str.NewTaskRepository().UpdateTask(ctx, tsk.ID, nil, &task.TaskUpdate{Data: &data}) + if err != nil || updated == nil { + tsk.AppendError(errors.New("unable to update task during run")) + return + } + *tsk = *updated // replace the in-memory task with the updated one, per the runner contract + tsk.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, updatingRunner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: updatingRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.Error).To(BeNil()) + Expect(actualTask.Data).To(HaveKeyWithValue("key", "value")) + }) + + It("does not complete a task whose state lock changed while it was running", func() { + stealingRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { + // Simulate the task being unstuck and re-claimed elsewhere by changing the + // state lock out from under this run. The completion must then miss rather + // than falsely complete another run's task. + _, err := str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": tsk.ID}, bson.M{"$set": bson.M{"stateLock": "ffffffffffffffffffffffffffffffff"}}) + if err != nil { + tsk.AppendError(err) + return + } + tsk.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, stealingRunner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: stealingRunner.GetRunnerType()})) + + que.Start() + + // The completion's compare-and-swap misses, which is logged rather than silently swallowed. + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertWarn("Unable to stop task; no running task matched the expected condition") + return true + }, "5s", "100ms").To(BeTrue()) + + // The task was not falsely completed; it remains running (recovered later by unstick). + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateRunning)) + }) + + It("cleans up a task that panics during execution", func() { + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: panicRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + Expect(countingRunner.GetCount()).To(Equal(0)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("unhandled panic")) + + lgr.AssertError("Unhandled panic while running task") + Expect(testutil.ToFloat64(taskQueue.RunPanicTotal.WithLabelValues(panicRunner.GetRunnerType()))).To(Equal(float64(1))) + }) + + It("fails a pending task that does not match any registered runner", func() { + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: taskTest.RandomType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + Expect(countingRunner.GetCount()).To(Equal(0)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("runner not found for task")) + }) + + It("fails a task whose runner leaves it in an unknown state", func() { + unknownStateRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { + tsk.State = "unknown-state" + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, unknownStateRunner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: unknownStateRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("unknown task state")) + }) + + It("warns and sets the available time for a pending task left without one", func() { + missingAvailableTimeRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { + tsk.State = task.TaskStatePending + tsk.AvailableTime = nil + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, missingAvailableTimeRunner)) + + test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: missingAvailableTimeRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertWarn("Available time missing for pending task") + return true + }, "5s", "50ms").To(BeTrue()) + }) + + It("warns when a pending task's available time is significantly in the past", func() { + staleAvailableTimeRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { + tsk.RepeatAvailableAt(time.Now().Add(-2 * time.Minute)) + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, staleAvailableTimeRunner)) + + test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: staleAvailableTimeRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertWarn("Available time significantly before now for pending task") + return true + }, "5s", "50ms").To(BeTrue()) + }) + + It("unsticks and logs a task left running past its deadline", func() { + stuckTask := &task.Task{ + ID: task.NewID(), + Type: countingRunner.GetRunnerType(), + State: task.TaskStateRunning, + DeadlineTime: pointer.FromTime(time.Now().Add(-time.Minute)), + StateLock: pointer.FromString(taskTest.RandomType()), + CreatedTime: time.Now(), + Revision: 1, + } + _, err := str.GetCollection("tasks").InsertOne(ctx, stuckTask) + Expect(err).ToNot(HaveOccurred()) + + unstickConfig := &taskQueue.Config{Workers: 2, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: time.Millisecond, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault} + que = test.Must(taskQueue.New(taskTest.RandomType(), unstickConfig, lgr, str, countingRunner)) + + que.Start() + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertInfo("Unstuck tasks") + return true + }, "5s", "50ms").To(BeTrue()) + + // Once unstuck, the task returns to pending and is dispatched, run, and completed. + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, stuckTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + }) + + It("reverts a task that is still running to pending when the queue is stopped", func() { + hangingRunner := taskQueueTest.NewHangingRunner(taskTest.RandomType(), time.Minute) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, hangingRunner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: hangingRunner.GetRunnerType()})) + + que.Start() + + // Wait until the runner has picked up the task and it is running. + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateRunning)) + + // Stopping cancels the worker context; the hanging runner returns leaving the + // task running, so the queue must revert it to pending for a later retry. + que.Stop() + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + Expect(actualTask.DeadlineTime).To(BeNil()) + }) + + It("cancels the context of a task that exceeds its timeout", func() { + sleepRunner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), time.Minute, 100*time.Millisecond, 50*time.Millisecond, 10*time.Second) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, sleepRunner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: sleepRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "1m", "50ms").To(Equal(task.TaskStateCompleted)) + + Expect(countingRunner.GetCount()).To(Equal(0)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("task runner timeout exceeded")) + }) + + It("fails a task whose runner returns without setting a terminal state", func() { + noopRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), nil) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, noopRunner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: noopRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("runner failed to set terminal task state")) + }) + + It("fails a task that exceeds its timeout without setting its own state", func() { + hangingRunner := taskQueueTest.NewHangingRunner(taskTest.RandomType(), 200*time.Millisecond) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, hangingRunner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: hangingRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("task runner timeout exceeded")) + }) + + It("logs a warning if a task that exceeds its maximum duration", func() { + sleepRunner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 2*time.Minute, time.Minute, time.Millisecond, 10*time.Millisecond) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, sleepRunner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: sleepRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "1m", "50ms").To(Equal(task.TaskStateCompleted)) + + Expect(countingRunner.GetCount()).To(Equal(0)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.Error).To(BeNil()) + + lgr.AssertWarn("Task duration exceeds maximum") + }) + + It("does not race or panic when Stop is called concurrently", func() { + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: countingRunner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + var waitGroup sync.WaitGroup + for range 3 { + waitGroup.Go(func() { + defer GinkgoRecover() + que.Stop() + }) + } + waitGroup.Wait() + }) + }) + + Context("with a blocking runner", func() { + var blockingRunner *taskQueueTest.BlockingRunner + var que taskQueue.Queue + + BeforeEach(func() { + blockingRunner = taskQueueTest.NewBlockingRunner(taskTest.RandomType(), time.Minute) + + cfg := &taskQueue.Config{Workers: 1, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: 250 * time.Millisecond} + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, blockingRunner)) + }) + + It("returns from Stop within the stop timeout when a runner ignores cancellation", func() { + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: blockingRunner.GetRunnerType()})) + + que.Start() + + // Wait until the blocking runner has picked up the task and it is running. + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateRunning)) + + // Stop must return within roughly the stop timeout even though the runner never + // returns; it abandons the in-flight task rather than blocking forever. + stopped := make(chan struct{}) + go func() { + defer GinkgoRecover() + que.Stop() + close(stopped) + }() + Eventually(stopped, "5s").Should(BeClosed()) + + lgr.AssertError("Task queue workers did not stop within timeout; abandoning in-flight tasks") + }) + }) + + Context("with a runner that exceeds its timeout without returning", func() { + var blockingRunner *taskQueueTest.BlockingRunner + var que taskQueue.Queue + + BeforeEach(func() { + // A short duration maximum yields a short runner timeout (3x), so the watchdog fires + // quickly while the runner is still blocked, without an unstick reclaiming the task. + blockingRunner = taskQueueTest.NewBlockingRunner(taskTest.RandomType(), 20*time.Millisecond) + + cfg := &taskQueue.Config{Workers: 1, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: 250 * time.Millisecond} + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, blockingRunner)) + }) + + It("logs and counts the run once its timeout elapses", func() { + runnerType := blockingRunner.GetRunnerType() + taskQueue.RunnerTimeoutExceededTotal.Reset() + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runnerType})) + + que.Start() + + // Wait until the runner has picked up the task and it is running. + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateRunning)) + + // The watchdog fires after the runner timeout, recording the stuck run. + Eventually(func() float64 { + return testutil.ToFloat64(taskQueue.RunnerTimeoutExceededTotal.WithLabelValues(runnerType)) + }, "5s", "50ms").Should(BeNumerically(">=", float64(1))) + lgr.AssertError("Task runner exceeded timeout without returning; worker is blocked until it returns") + + // The runner never returns, so Stop abandons the in-flight worker. + stopped := make(chan struct{}) + go func() { + defer GinkgoRecover() + que.Stop() + close(stopped) + }() + Eventually(stopped, "5s").Should(BeClosed()) + }) + }) + + Context("with multiple queues", func() { + const queueCount = 10 + const workersCount = 10 + const taskCount = 2 * queueCount * workersCount + + var runner *taskQueueTest.RepeatRunner + var ques []taskQueue.Queue + + BeforeEach(func() { + runner = taskQueueTest.NewRepeatRunner(taskTest.RandomType()) + + cfg := &taskQueue.Config{Workers: workersCount, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: time.Millisecond, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault} + ques = make([]taskQueue.Queue, queueCount) + for index := range len(ques) { + ques[index] = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + } + }) + + AfterEach(func() { + for _, que := range ques { + que.Stop() + } + }) + + It("completes all runnings tasks when stopped", func() { + tasks := make(task.Tasks, taskCount) + for index := range len(tasks) { + tasks[index] = test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + } + + for _, que := range ques { + que.Start() + } + + select { + case <-ctx.Done(): + case <-time.After(2 * time.Second): + } + + for _, que := range ques { + que.Stop() + } + + for _, tsk := range tasks { + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, tsk.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + } + }) + + It("eventually a queue attempts to run a task that is already running in another queue", func() { + tsk := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + for _, que := range ques { + que.Start() + } + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertInfo("Task no longer available to start") + return true + }, "10s", "100ms").To(BeTrue()) + + for _, que := range ques { + que.Stop() + } + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, tsk.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + }) + }) + }) }) diff --git a/task/queue/test/runner.go b/task/queue/test/runner.go index 751cb2c23e..dfc4d396a9 100644 --- a/task/queue/test/runner.go +++ b/task/queue/test/runner.go @@ -22,12 +22,13 @@ func NewCountingRunner(typ string) *CountingRunner { mu: &sync.Mutex{}, } } + func (c *CountingRunner) GetRunnerType() string { return c.Type } -func (c *CountingRunner) GetRunnerDeadline() time.Time { - return time.Now().Add(time.Second * 10) +func (c *CountingRunner) GetRunnerDeadline() time.Duration { + return time.Second * 10 } func (c *CountingRunner) GetRunnerTimeout() time.Duration { @@ -53,3 +54,223 @@ func (c *CountingRunner) GetCount() int { } var _ queue.Runner = &CountingRunner{} + +type PanicRunner struct { + Type string +} + +func NewPanicRunner(typ string) *PanicRunner { + return &PanicRunner{ + Type: typ, + } +} + +func (p *PanicRunner) GetRunnerType() string { + return p.Type +} + +func (p *PanicRunner) GetRunnerDeadline() time.Duration { + return p.GetRunnerDurationMaximum() * 5 +} + +func (p *PanicRunner) GetRunnerTimeout() time.Duration { + return p.GetRunnerDurationMaximum() * 3 +} + +func (p *PanicRunner) GetRunnerDurationMaximum() time.Duration { + return time.Second +} + +func (p *PanicRunner) Run(ctx context.Context, tsk *task.Task) { + panic("panic test") +} + +var _ queue.Runner = &PanicRunner{} + +type SleepRunner struct { + Type string + Deadline time.Duration + Timeout time.Duration + Maximum time.Duration + Sleep time.Duration +} + +func NewSleepRunner(typ string, deadline time.Duration, timeout time.Duration, maximum time.Duration, sleep time.Duration) *SleepRunner { + return &SleepRunner{ + Type: typ, + Deadline: deadline, + Timeout: timeout, + Maximum: maximum, + Sleep: sleep, + } +} + +func (s *SleepRunner) GetRunnerType() string { + return s.Type +} + +func (s *SleepRunner) GetRunnerDeadline() time.Duration { + return s.Deadline +} + +func (s *SleepRunner) GetRunnerTimeout() time.Duration { + return s.Timeout +} + +func (s *SleepRunner) GetRunnerDurationMaximum() time.Duration { + return s.Maximum +} + +func (s *SleepRunner) Run(ctx context.Context, tsk *task.Task) { + select { + case <-ctx.Done(): + tsk.AppendError(context.Cause(ctx)) + tsk.SetCompleted() + case <-time.After(s.Sleep): + tsk.SetCompleted() + } +} + +var _ queue.Runner = &SleepRunner{} + +type RepeatRunner struct { + Type string +} + +func NewRepeatRunner(typ string) *RepeatRunner { + return &RepeatRunner{ + Type: typ, + } +} + +func (r *RepeatRunner) GetRunnerType() string { + return r.Type +} + +func (r *RepeatRunner) GetRunnerDeadline() time.Duration { + return r.GetRunnerDurationMaximum() * 5 +} + +func (r *RepeatRunner) GetRunnerTimeout() time.Duration { + return r.GetRunnerDurationMaximum() * 3 +} + +func (r *RepeatRunner) GetRunnerDurationMaximum() time.Duration { + return time.Minute +} + +func (r *RepeatRunner) Run(ctx context.Context, tsk *task.Task) { + tsk.State = task.TaskStatePending +} + +var _ queue.Runner = &RepeatRunner{} + +type HangingRunner struct { + Type string + Timeout time.Duration +} + +func NewHangingRunner(typ string, timeout time.Duration) *HangingRunner { + return &HangingRunner{ + Type: typ, + Timeout: timeout, + } +} + +func (h *HangingRunner) GetRunnerType() string { + return h.Type +} + +func (h *HangingRunner) GetRunnerDeadline() time.Duration { + return h.Timeout * 5 +} + +func (h *HangingRunner) GetRunnerTimeout() time.Duration { + return h.Timeout +} + +func (h *HangingRunner) GetRunnerDurationMaximum() time.Duration { + return h.Timeout / 2 +} + +// Run blocks until its context is canceled and returns without setting a terminal +// state, leaving the task running to exercise shutdown interruption and timeout handling. +func (h *HangingRunner) Run(ctx context.Context, tsk *task.Task) { + <-ctx.Done() +} + +var _ queue.Runner = &HangingRunner{} + +type BlockingRunner struct { + Type string + DurationMaximum time.Duration +} + +func NewBlockingRunner(typ string, durationMaximum time.Duration) *BlockingRunner { + return &BlockingRunner{ + Type: typ, + DurationMaximum: durationMaximum, + } +} + +func (b *BlockingRunner) GetRunnerType() string { + return b.Type +} + +func (b *BlockingRunner) GetRunnerDeadline() time.Duration { + return b.GetRunnerDurationMaximum() * 5 +} + +func (b *BlockingRunner) GetRunnerTimeout() time.Duration { + return b.GetRunnerDurationMaximum() * 3 +} + +func (b *BlockingRunner) GetRunnerDurationMaximum() time.Duration { + return b.DurationMaximum +} + +// Run blocks forever, ignoring context cancellation entirely, to simulate a runner that +// does not honor shutdown. Used to verify Stop returns within its timeout regardless. +func (b *BlockingRunner) Run(ctx context.Context, tsk *task.Task) { + select {} +} + +var _ queue.Runner = &BlockingRunner{} + +type CallbackRunner struct { + Type string + Callback func(ctx context.Context, tsk *task.Task) +} + +func NewCallbackRunner(typ string, callback func(ctx context.Context, tsk *task.Task)) *CallbackRunner { + return &CallbackRunner{ + Type: typ, + Callback: callback, + } +} + +func (c *CallbackRunner) GetRunnerType() string { + return c.Type +} + +func (c *CallbackRunner) GetRunnerDeadline() time.Duration { + return c.GetRunnerDurationMaximum() * 5 +} + +func (c *CallbackRunner) GetRunnerTimeout() time.Duration { + return c.GetRunnerDurationMaximum() * 3 +} + +func (c *CallbackRunner) GetRunnerDurationMaximum() time.Duration { + return time.Minute +} + +// Run invokes the callback, letting a test inject arbitrary behavior (such as updating the +// task while it is running) into the task run. +func (c *CallbackRunner) Run(ctx context.Context, tsk *task.Task) { + if c.Callback != nil { + c.Callback(ctx, tsk) + } +} + +var _ queue.Runner = &CallbackRunner{} diff --git a/task/service/api/v1/v1.go b/task/service/api/v1/v1.go index de7033e74b..df3e09f401 100644 --- a/task/service/api/v1/v1.go +++ b/task/service/api/v1/v1.go @@ -93,7 +93,13 @@ func (r *Router) GetTask(res rest.ResponseWriter, req *rest.Request) { return } - tsk, err := r.TaskClient().GetTask(req.Context(), id) + condition := request.NewCondition() + if err := request.DecodeRequestQuery(req.Request, condition); err != nil { + responder.Error(http.StatusBadRequest, err) + return + } + + tsk, err := r.TaskClient().GetTask(req.Context(), id, condition) if err != nil { responder.InternalServerError(err) return @@ -114,13 +120,19 @@ func (r *Router) UpdateTask(res rest.ResponseWriter, req *rest.Request) { return } + condition := request.NewCondition() + if err := request.DecodeRequestQuery(req.Request, condition); err != nil { + responder.Error(http.StatusBadRequest, err) + return + } + update := task.NewTaskUpdate() if err := request.DecodeRequestBody(req.Request, update); err != nil { responder.Error(http.StatusBadRequest, err) return } - tsk, err := r.TaskClient().UpdateTask(req.Context(), id, update) + tsk, err := r.TaskClient().UpdateTask(req.Context(), id, condition, update) if err != nil { responder.InternalServerError(err) return @@ -141,7 +153,13 @@ func (r *Router) DeleteTask(res rest.ResponseWriter, req *rest.Request) { return } - err := r.TaskClient().DeleteTask(req.Context(), id) + condition := request.NewCondition() + if err := request.DecodeRequestQuery(req.Request, condition); err != nil { + responder.Error(http.StatusBadRequest, err) + return + } + + err := r.TaskClient().DeleteTask(req.Context(), id, condition) if err != nil { responder.InternalServerError(err) return diff --git a/task/service/service/client.go b/task/service/service/client.go index ff9c1ab69d..f212b3701c 100644 --- a/task/service/service/client.go +++ b/task/service/service/client.go @@ -5,6 +5,8 @@ import ( "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/page" + "github.com/tidepool-org/platform/request" + storeStructured "github.com/tidepool-org/platform/store/structured" "github.com/tidepool-org/platform/task" taskStore "github.com/tidepool-org/platform/task/store" ) @@ -33,17 +35,17 @@ func (c *Client) CreateTask(ctx context.Context, create *task.TaskCreate) (*task return repository.CreateTask(ctx, create) } -func (c *Client) GetTask(ctx context.Context, id string) (*task.Task, error) { +func (c *Client) GetTask(ctx context.Context, id string, condition *request.Condition) (*task.Task, error) { repository := c.taskStore.NewTaskRepository() - return repository.GetTask(ctx, id) + return repository.GetTask(ctx, id, storeStructured.MapCondition(condition)) } -func (c *Client) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { +func (c *Client) UpdateTask(ctx context.Context, id string, condition *request.Condition, update *task.TaskUpdate) (*task.Task, error) { repository := c.taskStore.NewTaskRepository() - return repository.UpdateTask(ctx, id, update) + return repository.UpdateTask(ctx, id, storeStructured.MapCondition(condition), update) } -func (c *Client) DeleteTask(ctx context.Context, id string) error { +func (c *Client) DeleteTask(ctx context.Context, id string, condition *request.Condition) error { repository := c.taskStore.NewTaskRepository() - return repository.DeleteTask(ctx, id) + return repository.DeleteTask(ctx, id, storeStructured.MapCondition(condition)) } diff --git a/task/service/service/service.go b/task/service/service/service.go index 6e8d776cf9..da108c1589 100644 --- a/task/service/service/service.go +++ b/task/service/service/service.go @@ -136,7 +136,7 @@ func (s *Service) initializeTaskStore() error { func (s *Service) terminateTaskStore() { if s.taskStore != nil { s.Logger().Debug("Closing task store") - s.taskStore.Terminate(context.Background()) + _ = s.taskStore.Terminate(context.Background()) s.Logger().Debug("Destroying task store") s.taskStore = nil @@ -283,15 +283,6 @@ func (s *Service) initializeTaskQueue() error { return errors.Wrap(err, "unable to load task queue config") } - s.Logger().Debug("Creating task queue") - - taskQueue, err := queue.NewMultiQueue(cfg, s.Logger(), s.TaskStore()) - if err != nil { - return errors.Wrap(err, "unable to create task queue") - } - - s.taskQueue = taskQueue - var runners []queue.Runner if s.dexcomClient != nil { @@ -327,13 +318,15 @@ func (s *Service) initializeTaskQueue() error { } runners = append(runners, ehrSyncRnnr) - for _, r := range runners { - r := r - if err := taskQueue.RegisterRunner(r); err != nil { - return errors.Wrapf(err, "unable to register runner %s", r.GetRunnerType()) - } + s.Logger().Debug("Creating task queue") + + taskQueue, err := queue.NewMultiQueue(cfg, s.Logger(), s.TaskStore(), runners...) + if err != nil { + return errors.Wrap(err, "unable to create task queue") } + s.taskQueue = taskQueue + s.Logger().Debug("Starting task queue") s.taskQueue.Start() diff --git a/task/store/mongo/mongo.go b/task/store/mongo/mongo.go index cd5798cc74..15f1f4e8e7 100644 --- a/task/store/mongo/mongo.go +++ b/task/store/mongo/mongo.go @@ -2,6 +2,7 @@ package mongo import ( "context" + "slices" "time" "github.com/prometheus/client_golang/prometheus" @@ -10,27 +11,43 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "github.com/tidepool-org/platform/ehr/reconcile" + ehrReconcile "github.com/tidepool-org/platform/ehr/reconcile" "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/id" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/page" "github.com/tidepool-org/platform/pointer" + storeStructured "github.com/tidepool-org/platform/store/structured" storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" structureValidator "github.com/tidepool-org/platform/structure/validator" summaryTask "github.com/tidepool-org/platform/summary/task" "github.com/tidepool-org/platform/task" - "github.com/tidepool-org/platform/task/store" + taskStore "github.com/tidepool-org/platform/task/store" ) -var ( - TasksStateTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tidepool_task_tasks_state_total", - Help: "The total number of tasks sorted by state and type", - }, []string{"state", "type"}) +var TasksStateTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_tasks_state_total", + Help: "The total number of tasks sorted by state and type", +}, []string{"state", "type"}) + +// TasksLostCompletionTotal counts task completions dropped because the compare-and-swap in +// StopTask missed (the running claim was concurrently modified, unstuck, or deleted). The task +// is recovered by the deadline/unstick mechanism, but the intended terminal state is lost. The +// type label is populated only when the repository is type-filtered (as it is for each queue in +// a MultiQueue); otherwise it is empty. +var TasksLostCompletionTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_lost_completion_total", + Help: "The total number of task completions dropped because the state-lock compare-and-swap missed, sorted by type", +}, []string{"type"}) + +const ( + MaxTaskCreationDuration = 30 * time.Second + + // TransitionTimeout bounds a task state-transition write (start or stop) that must + // complete regardless of the caller's context being canceled (e.g. during shutdown). + TransitionTimeout = 10 * time.Second ) -const MaxTaskCreationDuration = 30 * time.Second - type Store struct { *storeStructuredMongo.Store typeFilter *string @@ -47,14 +64,14 @@ func NewStore(config *storeStructuredMongo.Config) (*Store, error) { }, nil } -func (s *Store) WithTypeFilter(typeFilter string) store.Store { +func (s *Store) WithTypeFilter(typeFilter string) taskStore.Store { return &Store{ Store: s.Store, typeFilter: &typeFilter, } } -func (s *Store) NewTaskRepository() store.TaskRepository { +func (s *Store) NewTaskRepository() taskStore.TaskRepository { repo := s.TaskRepository() repo.typeFilter = s.typeFilter return repo @@ -91,22 +108,6 @@ func (s *Store) EnsureDefaultTasks() error { return nil } -func (s *Store) EnsureSummaryMigrationTask() error { - ctx, cancel := context.WithTimeout(context.Background(), MaxTaskCreationDuration) - defer cancel() - - repository := s.TaskRepository() - return repository.EnsureSummaryMigrationTask(ctx) -} - -func (s *Store) EnsureEHRReconcileTask() error { - ctx, cancel := context.WithTimeout(context.Background(), MaxTaskCreationDuration) - defer cancel() - - repository := s.TaskRepository() - return repository.EnsureEHRReconcileTask(ctx) -} - type TaskRepository struct { *storeStructuredMongo.Repository typeFilter *string @@ -132,21 +133,11 @@ func (t *TaskRepository) EnsureIndexes() error { SetSparse(true). SetBackground(true), }, - { - Keys: bson.D{{Key: "priority", Value: 1}}, - Options: options.Index(). - SetBackground(true), - }, { Keys: bson.D{{Key: "availableTime", Value: 1}}, Options: options.Index(). SetBackground(true), }, - { - Keys: bson.D{{Key: "expirationTime", Value: 1}}, - Options: options.Index(). - SetBackground(true), - }, { Keys: bson.D{{Key: "state", Value: 1}}, Options: options.Index(). @@ -178,7 +169,7 @@ func (t *TaskRepository) EnsureSummaryMigrationTask(ctx context.Context) error { } func (t *TaskRepository) EnsureEHRReconcileTask(ctx context.Context) error { - create := reconcile.NewTaskCreate() + create := ehrReconcile.NewTaskCreate() return t.ensureTask(ctx, create) } @@ -189,30 +180,14 @@ func (t *TaskRepository) ensureTask(ctx context.Context, create *task.TaskCreate } else if err = structureValidator.New(log.LoggerFromContext(ctx)).Validate(tsk); err != nil { return errors.Wrap(err, "task is invalid") } - if err := t.assertType(t.typeFilter, &tsk.Type); err != nil { - return err - } - - upsert := true - after := options.After - opts := options.FindOneAndUpdateOptions{ - ReturnDocument: &after, - Upsert: &upsert, - } - - res := t.FindOneAndUpdate(ctx, - bson.M{"name": tsk.Name}, - bson.M{"$setOnInsert": tsk}, - &opts, - ) - if res.Err() != nil && !errors.Is(res.Err(), mongo.ErrNoDocuments) { - return errors.Wrap(res.Err(), "unable to create task") + if result, err := t.UpdateOne(ctx, bson.M{"name": tsk.Name}, bson.M{"$setOnInsert": tsk}, options.Update().SetUpsert(true)); err != nil { + return errors.Wrap(err, "unable to create task") + } else if result.UpsertedCount > 0 { + TasksStateTotal.WithLabelValues(task.TaskStatePending, create.Type).Inc() } - TasksStateTotal.WithLabelValues(task.TaskStatePending, create.Type).Inc() - - return res.Err() + return nil } func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) { @@ -233,7 +208,7 @@ func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, return nil, err } - now := time.Now() + now := time.Now().UTC() logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"filter": filter, "pagination": pagination}) tasks := task.Tasks{} @@ -248,15 +223,20 @@ func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, if filter.State != nil { selector["state"] = *filter.State } + if t.typeFilter != nil { + selector["type"] = *t.typeFilter + } opts := storeStructuredMongo.FindWithPagination(pagination). SetSort(bson.M{"createdTime": -1}) cursor, err := t.Find(ctx, selector, opts) - logger.WithFields(log.Fields{"count": len(tasks), "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("ListTasks") if err != nil { + logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("ListTasks") return nil, errors.Wrap(err, "unable to list tasks") } - if err = cursor.All(ctx, &tasks); err != nil { + err = cursor.All(ctx, &tasks) + logger.WithFields(log.Fields{"count": len(tasks), "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("ListTasks") + if err != nil { return nil, errors.Wrap(err, "unable to decode tasks") } @@ -278,15 +258,15 @@ func (t *TaskRepository) CreateTask(ctx context.Context, create *task.TaskCreate } else if err = structureValidator.New(log.LoggerFromContext(ctx)).Validate(tsk); err != nil { return nil, errors.Wrap(err, "task is invalid") } - if err := t.assertType(t.typeFilter, &tsk.Type); err != nil { + if err = t.assertType(t.typeFilter, &tsk.Type); err != nil { return nil, err } - now := time.Now() + now := time.Now().UTC() logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"create": create}) _, err = t.InsertOne(ctx, tsk) - logger.WithFields(log.Fields{"id": tsk.ID, "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("CreateTask") + logger.WithFields(log.Fields{"task": tsk.LogFields(), "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("CreateTask") if err != nil { return nil, errors.Wrap(err, "unable to create task") } @@ -295,25 +275,24 @@ func (t *TaskRepository) CreateTask(ctx context.Context, create *task.TaskCreate return tsk, nil } -func (t *TaskRepository) GetTask(ctx context.Context, id string) (*task.Task, error) { +func (t *TaskRepository) GetTask(ctx context.Context, id string, condition *storeStructured.Condition) (*task.Task, error) { if ctx == nil { return nil, errors.New("context is missing") } if id == "" { return nil, errors.New("id is missing") } - - now := time.Now() - logger := log.LoggerFromContext(ctx).WithField("id", id) - - var task *task.Task - - selector := bson.M{"id": id} - if t.typeFilter != nil { - selector["type"] = t.typeFilter + if condition == nil { + condition = &storeStructured.Condition{} + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return nil, errors.Wrap(err, "condition is invalid") } - err := t.FindOne(ctx, selector).Decode(&task) + now := time.Now().UTC() + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "condition": condition}) + + tsk := &task.Task{} + err := t.FindOne(ctx, t.selector(id, condition)).Decode(tsk) logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("GetTask") if errors.Is(err, mongo.ErrNoDocuments) { @@ -322,72 +301,63 @@ func (t *TaskRepository) GetTask(ctx context.Context, id string) (*task.Task, er return nil, errors.Wrap(err, "unable to get task") } - return task, nil + return tsk, nil } -func (t *TaskRepository) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { +func (t *TaskRepository) UpdateTask(ctx context.Context, id string, condition *storeStructured.Condition, update *task.TaskUpdate) (*task.Task, error) { if ctx == nil { return nil, errors.New("context is missing") } if id == "" { return nil, errors.New("id is missing") } + if condition == nil { + condition = &storeStructured.Condition{} + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return nil, errors.Wrap(err, "condition is invalid") + } if update == nil { return nil, errors.New("update is missing") } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(update); err != nil { return nil, errors.Wrap(err, "update is invalid") } - now := time.Now() - logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "update": update}) + now := time.Now().UTC() + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "condition": condition, "update": update}) - set := bson.M{ - "modifiedTime": now, - } - if update.Priority != nil { - set["priority"] = *update.Priority - } - if update.Data != nil { - set["data"] = *update.Data - } - if update.AvailableTime != nil { - set["availableTime"] = *update.AvailableTime - } - if update.ExpirationTime != nil { - set["expirationTime"] = *update.ExpirationTime - } + set, unset := t.parseUpdate(update) + set["modifiedTime"] = now - selector := bson.M{"id": id} - if t.typeFilter != nil { - selector["type"] = t.typeFilter - } - - changeInfo, err := t.UpdateMany(ctx, selector, t.ConstructUpdate(set, bson.M{})) - logger.WithFields(log.Fields{"changeInfo": changeInfo, "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("UpdateTask") - if err != nil { + updatedTask := &task.Task{} + opts := options.FindOneAndUpdate().SetReturnDocument(options.After) + err := t.FindOneAndUpdate(ctx, t.selector(id, condition), t.ConstructUpdate(set, unset), opts).Decode(updatedTask) + logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("UpdateTask") + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, nil + } else if err != nil { return nil, errors.Wrap(err, "unable to update task") } - return t.GetTask(ctx, id) + return updatedTask, nil } -func (t *TaskRepository) DeleteTask(ctx context.Context, id string) error { +func (t *TaskRepository) DeleteTask(ctx context.Context, id string, condition *storeStructured.Condition) error { if ctx == nil { return errors.New("context is missing") } if id == "" { return errors.New("id is missing") } + if condition == nil { + condition = &storeStructured.Condition{} + } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(condition); err != nil { + return errors.Wrap(err, "condition is invalid") + } - now := time.Now() + now := time.Now().UTC() logger := log.LoggerFromContext(ctx).WithField("id", id) - selector := bson.M{"id": id} - if t.typeFilter != nil { - selector["type"] = t.typeFilter - } - - changeInfo, err := t.DeleteOne(ctx, selector) + changeInfo, err := t.DeleteOne(ctx, t.selector(id, condition)) logger.WithFields(log.Fields{"changeInfo": changeInfo, "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("DeleteTask") if err != nil { return errors.Wrap(err, "unable to delete task") @@ -396,106 +366,260 @@ func (t *TaskRepository) DeleteTask(ctx context.Context, id string) error { return nil } -// TODO: Consider using an "update only specific fields" approach, as above - -func (t *TaskRepository) UpdateFromState(ctx context.Context, tsk *task.Task, state string) (*task.Task, error) { +func (t *TaskRepository) StartTask(ctx context.Context, id string, revision int, deadline time.Duration) (*task.Task, error) { if ctx == nil { return nil, errors.New("context is missing") } - if tsk == nil { - return nil, errors.New("task is missing") + if id == "" { + return nil, errors.New("id is missing") + } else if !task.IsValidID(id) { + return nil, errors.New("id is invalid") + } + if deadline <= 0 { + return nil, errors.New("deadline is invalid") } - now := time.Now() - logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": tsk.ID, "state": state}) + // Add a timeout, but ignore cancel from the parent context so the claim write completes + // and its outcome is known. A write abandoned mid-flight (e.g. on shutdown) can still + // commit in the database, leaving the task running with no reliable way to revert it. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), TransitionTimeout) + defer cancel() - tsk.ModifiedTime = pointer.FromTime(now.Truncate(time.Millisecond)) + now := time.Now().UTC() + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "revision": revision, "deadline": deadline}) - selector := bson.M{ - "id": tsk.ID, - "state": state, - } - if t.typeFilter != nil { - selector["type"] = t.typeFilter + set := bson.M{ + "state": task.TaskStateRunning, + "runTime": now, + "deadlineTime": now.Add(deadline), + "modifiedTime": now, + "stateLock": newStateLock(), } - result, err := t.ReplaceOne(ctx, selector, tsk) - logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("UpdateFromState") - if err != nil { - return nil, errors.Wrap(err, "unable to update from state") + unset := bson.M{ + "duration": 1, } - if result.ModifiedCount != 1 { - return nil, task.AlreadyClaimedTask + + selector := t.selector(id, storeStructured.NewConditionWithRevision(&revision)) + selector["state"] = task.TaskStatePending + + tsk := &task.Task{} + opts := options.FindOneAndUpdate().SetReturnDocument(options.After) + err := t.FindOneAndUpdate(ctx, selector, t.ConstructUpdate(set, unset), opts).Decode(tsk) + logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("StartTask") + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, nil + } else if err != nil { + return nil, errors.Wrap(err, "unable to start task") } - TasksStateTotal.WithLabelValues(tsk.State, tsk.Type).Inc() + TasksStateTotal.WithLabelValues(task.TaskStateRunning, tsk.Type).Inc() return tsk, nil } -func (t *TaskRepository) UnstickTasks(ctx context.Context) (int64, error) { - selector := bson.M{ +// Will only timeout after 10 seconds even if parent context is canceled. +func (t *TaskRepository) StopTask(ctx context.Context, id string, stateLock *string, state string, duration *time.Duration, update *task.TaskUpdate) error { + if ctx == nil { + return errors.New("context is missing") + } + if id == "" { + return errors.New("id is missing") + } else if !task.IsValidID(id) { + return errors.New("id is invalid") + } + if stateLock == nil { + return errors.New("state lock is missing") + } else if *stateLock == "" { + return errors.New("state lock is invalid") + } + if state == "" { + return errors.New("state is missing") + } else if !slices.Contains(task.TaskStates(), state) { + return errors.New("state is invalid") + } + if duration != nil && *duration < 0 { + return errors.New("duration is invalid") + } + if update != nil { + if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(update); err != nil { + return errors.Wrap(err, "update is invalid") + } + } + + // Add a timeout, but ignore cancel from parent context to ensure we stop task even exiting + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), TransitionTimeout) + defer cancel() + + now := time.Now().UTC() + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "stateLock": stateLock, "state": state, "duration": duration, "update": update}) + + set, unset := t.parseUpdate(update) + set["modifiedTime"] = now + set["state"] = state + unset["deadlineTime"] = 1 + unset["stateLock"] = 1 + if duration != nil { + set["duration"] = duration.Truncate(time.Millisecond).Seconds() + } else { + // A nil duration means no run actually happened (e.g. a claimed task whose dispatch + // was reverted during shutdown), so also clear the run time recorded by StartTask, + // keeping the runTime/duration pair describing only the last actual run. + unset["duration"] = 1 + unset["runTime"] = 1 + } + + selector := t.selector(id, nil) + selector["state"] = task.TaskStateRunning + selector["stateLock"] = stateLock + + tsk := &task.Task{} + err := t.FindOneAndUpdate(ctx, selector, t.ConstructUpdate(set, unset)).Decode(tsk) + logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("StopTask") + if errors.Is(err, mongo.ErrNoDocuments) { + // The compare-and-swap missed: no running task matched the expected state lock + // (it was concurrently modified, unstuck, or deleted since it started). + // The state transition is dropped; the deadline and unstick mechanism will + // recover the task if it was left running. This is logged and counted so lost + // completions are observable rather than silently swallowed. + logger.Warn("Unable to stop task; no running task matched the expected condition") + TasksLostCompletionTotal.WithLabelValues(pointer.Default(t.typeFilter, "")).Inc() + return nil + } else if err != nil { + return errors.Wrap(err, "unable to stop task") + } + + TasksStateTotal.WithLabelValues(state, tsk.Type).Inc() + return nil +} + +func (t *TaskRepository) UnstickTasks(ctx context.Context) ([]string, error) { + if ctx == nil { + return nil, errors.New("context is missing") + } + + lgr := log.LoggerFromContext(ctx) + + now := time.Now().UTC() + defer func() { lgr.WithField("duration", time.Since(now)/time.Microsecond).Debug("UnstickTasks") }() + + findSelector := bson.M{ "state": task.TaskStateRunning, - "deadlineTime": bson.M{"$lt": time.Now()}, + "deadlineTime": bson.M{"$lt": now}, } if t.typeFilter != nil { - selector["type"] = t.typeFilter + findSelector["type"] = *t.typeFilter } - update := bson.M{ - "$set": bson.M{"state": task.TaskStatePending}, - "$unset": bson.M{"deadlineTime": ""}, + opts := options.Find().SetSort(bson.M{"deadlineTime": 1}) + cursor, err := t.Find(ctx, findSelector, opts) + if err != nil { + return nil, errors.Wrap(err, "unable to list tasks") } + defer storeStructuredMongo.CloseCursor(ctx, cursor) - result, err := t.UpdateMany(ctx, selector, update) - if err != nil { - return 0, err + var ids []string + for cursor.Next(ctx) { + tsk := &task.Task{} + if err = cursor.Decode(tsk); err != nil { + log.LoggerFromContext(ctx).WithError(err).Error("Unable to decode task") + continue + } + + // The state clause is defensive: a matching deadline time already implies the same + // running claim (every stop clears the deadline time and any re-claim records a + // strictly later one), but including it keeps the invariant local to this update. + updateSelector := bson.M{ + "id": tsk.ID, + "state": task.TaskStateRunning, + "deadlineTime": tsk.DeadlineTime, + } + set := bson.M{ + "state": task.TaskStatePending, + "availableTime": now, + "modifiedTime": now, + } + unset := bson.M{ + "deadlineTime": 1, + "stateLock": 1, + } + if result, updateErr := t.UpdateOne(ctx, updateSelector, t.ConstructUpdate(set, unset)); updateErr != nil { + return ids, updateErr + } else if result.ModifiedCount > 0 { + ids = append(ids, tsk.ID) + } } - return result.ModifiedCount, err + return ids, cursor.Err() } func (t *TaskRepository) IteratePending(ctx context.Context) (*mongo.Cursor, error) { - now := time.Now() + now := time.Now().UTC() selector := bson.M{ "state": task.TaskStatePending, - "$and": []bson.M{ + "$or": []bson.M{ { - "$or": []bson.M{ - { - "availableTime": bson.M{ - "$exists": false, - }, - }, - { - "availableTime": bson.M{ - "$lte": now, - }, - }, + "availableTime": bson.M{ + "$exists": false, }, }, { - "$or": []bson.M{ - { - "expirationTime": bson.M{ - "$exists": false, - }, - }, - { - "expirationTime": bson.M{ - "$gt": now, - }, - }, + "availableTime": bson.M{ + "$lte": now, }, }, }, } if t.typeFilter != nil { - selector["type"] = t.typeFilter + selector["type"] = *t.typeFilter } - opts := options.Find().SetSort(bson.M{"priority": -1}) + opts := options.Find().SetSort(bson.D{{Key: "availableTime", Value: 1}}) return t.Find(ctx, selector, opts) } +func (t *TaskRepository) selector(id string, condition *storeStructured.Condition) bson.M { + selector := bson.M{"id": id} + if condition != nil { + if condition.Revision != nil { + if *condition.Revision == 0 { + selector["revision"] = bson.M{"$in": bson.A{0, nil}} + } else { + selector["revision"] = *condition.Revision + } + } + } + if t.typeFilter != nil { + selector["type"] = *t.typeFilter + } + return selector +} + +func (t *TaskRepository) parseUpdate(update *task.TaskUpdate) (bson.M, bson.M) { + set := bson.M{} + unset := bson.M{} + + if update != nil { + if update.Data != nil { + if *update.Data != nil { + set["data"] = *update.Data + } else { + unset["data"] = true + } + } + if update.AvailableTime != nil { + set["availableTime"] = *update.AvailableTime + } + if update.Error != nil { + if update.Error.Error != nil { + set["error"] = *update.Error + } else { + unset["error"] = true + } + } + } + + return set, unset +} + // assertType return an error if the expected type doesn't match the actual type func (t *TaskRepository) assertType(expected *string, actual *string) error { if expected != nil && actual != nil && *expected != *actual { @@ -503,3 +627,7 @@ func (t *TaskRepository) assertType(expected *string, actual *string) error { } return nil } + +func newStateLock() string { + return id.Must(id.New(16)) +} diff --git a/task/store/mongo/mongo_test.go b/task/store/mongo/mongo_test.go index 756cc33ed0..2b31cdaa5b 100644 --- a/task/store/mongo/mongo_test.go +++ b/task/store/mongo/mongo_test.go @@ -2,7 +2,6 @@ package mongo_test import ( "context" - "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -13,7 +12,7 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" - "github.com/tidepool-org/platform/errors" + "github.com/tidepool-org/platform/ehr/reconcile" "github.com/tidepool-org/platform/log" logTest "github.com/tidepool-org/platform/log/test" "github.com/tidepool-org/platform/pointer" @@ -22,6 +21,8 @@ import ( "github.com/tidepool-org/platform/task" taskStore "github.com/tidepool-org/platform/task/store" taskStoreMongo "github.com/tidepool-org/platform/task/store/mongo" + taskTest "github.com/tidepool-org/platform/task/test" + "github.com/tidepool-org/platform/test" ) var _ = Describe("Mongo", func() { @@ -37,7 +38,7 @@ var _ = Describe("Mongo", func() { AfterEach(func() { if str != nil { - str.Terminate(context.Background()) + _ = str.Terminate(context.Background()) } }) @@ -93,18 +94,10 @@ var _ = Describe("Mongo", func() { "Unique": Equal(true), "Sparse": Equal(true), }), - MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("priority")), - "Background": Equal(true), - }), MatchFields(IgnoreExtras, Fields{ "Key": Equal(storeStructuredMongoTest.MakeKeySlice("availableTime")), "Background": Equal(true), }), - MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("expirationTime")), - "Background": Equal(true), - }), MatchFields(IgnoreExtras, Fields{ "Key": Equal(storeStructuredMongoTest.MakeKeySlice("state")), "Background": Equal(true), @@ -135,12 +128,10 @@ var _ = Describe("Mongo", func() { BeforeEach(func() { var err error tsk, err = task.NewTask(context.Background(), &task.TaskCreate{ - Name: pointer.FromString("test"), - Type: "fetch", - Priority: 0, - Data: nil, - AvailableTime: pointer.FromTime(time.Now()), - ExpirationTime: pointer.FromTime(time.Now().Add(5 * time.Minute)), + Name: pointer.FromString("test"), + Type: "fetch", + Data: nil, + AvailableTime: pointer.FromTime(time.Now()), }) Expect(err).ToNot(HaveOccurred()) tsk.State = task.TaskStateRunning @@ -148,133 +139,145 @@ var _ = Describe("Mongo", func() { Expect(err).ToNot(HaveOccurred()) }) - Context("UpdateFromState", func() { - var updated *task.Task - - const defaultPrometheusOutput = ` - # HELP tidepool_task_tasks_state_total The total number of tasks sorted by state and type - # TYPE tidepool_task_tasks_state_total counter - tidepool_task_tasks_state_total{ state = "", type = "" } 1 - ` - const metricName = "tidepool_task_tasks_state_total" - - BeforeEach(func() { - taskStoreMongo.TasksStateTotal.Reset() - var err error - updated, err = task.NewTask(context.Background(), &task.TaskCreate{ - Name: pointer.FromString("updated"), - Type: "fetch", - Priority: 0, - Data: nil, - AvailableTime: pointer.FromTime(time.Now()), - ExpirationTime: pointer.FromTime(time.Now().Add(5 * time.Minute)), - }) - Expect(err).ToNot(HaveOccurred()) - updated.ID = tsk.ID - }) - + Context("UnstickTasks", func() { It("returns an error when the context is missing", func() { - ctx = nil - result, err := repository.UpdateFromState(ctx, updated, tsk.State) + unstuckTaskIDs, err := repository.UnstickTasks(context.Context(nil)) Expect(err).To(MatchError("context is missing")) - Expect(result).To(BeNil()) + Expect(unstuckTaskIDs).To(BeNil()) }) - It("returns an error when the updated task is missing", func() { - updated = nil - result, err := repository.UpdateFromState(ctx, updated, tsk.State) - Expect(err).To(MatchError("task is missing")) - Expect(result).To(BeNil()) + It("returns no ids when there are no stuck tasks", func() { + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) + Expect(unstuckTaskIDs).To(BeEmpty()) }) - It("successfully fails the task with multiple errors", func() { - updated.State = task.TaskStateFailed - updated.AppendError(errors.New("first error")) - updated.AppendError(errors.New("second error")) - _, err := repository.UpdateFromState(ctx, updated, tsk.State) - Expect(err).ToNot(HaveOccurred()) + It("unsticks a running task with an expired deadline", func() { + stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) - result := task.Task{} - err = collection.FindOne(ctx, bson.M{"id": tsk.ID}).Decode(&result) - Expect(err).ToNot(HaveOccurred()) - Expect(result.State).To(Equal(updated.State)) - Expect(result.Error).To(Equal(updated.Error)) + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) + Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) + + actualStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": stuckTask.ID}).Decode(actualStuckTask)).To(Succeed()) + Expect(actualStuckTask.State).To(Equal(task.TaskStatePending)) + Expect(actualStuckTask.DeadlineTime).To(BeNil()) + Expect(actualStuckTask.AvailableTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) + Expect(actualStuckTask.ModifiedTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) }) - It("returns error if task is updated from the same state multiple times", func() { - updated.State = task.TaskStatePending - _, err := repository.UpdateFromState(ctx, updated, tsk.State) - Expect(err).ToNot(HaveOccurred()) + It("does not unstick a running task with a deadline in the future", func() { + notStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(time.Hour))) + + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) + Expect(unstuckTaskIDs).To(BeEmpty()) - _, err = repository.UpdateFromState(ctx, updated, tsk.State) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError("Task has already been claimed or is now unavailable.")) + actualNotStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": notStuckTask.ID}).Decode(actualNotStuckTask)).To(Succeed()) + Expect(actualNotStuckTask).To(Equal(notStuckTask)) }) - It("records metrics of completed tasks", func() { - updated.State = task.TaskStateCompleted - completedTask, err := repository.UpdateFromState(ctx, updated, tsk.State) + It("does not unstick a task that is not running", func() { + notStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, pointer.FromTime(test.Now().Add(-time.Minute))) - Expect(err).ToNot(HaveOccurred()) + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) + Expect(unstuckTaskIDs).To(BeEmpty()) + + actualNotStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": notStuckTask.ID}).Decode(actualNotStuckTask)).To(Succeed()) + Expect(actualNotStuckTask).To(Equal(notStuckTask)) + }) - prometheusState := strings.ReplaceAll(defaultPrometheusOutput, "", task.TaskStateCompleted) - expectedOutput := strings.ReplaceAll(prometheusState, "", completedTask.Type) + It("unsticks multiple running tasks with expired deadlines, ordered by deadline time", func() { + laterStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + earlierStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Hour))) - prometheusErr := testutil. - CollectAndCompare(taskStoreMongo.TasksStateTotal, strings.NewReader(expectedOutput), metricName) - Expect(prometheusErr).ToNot(HaveOccurred()) + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) + Expect(unstuckTaskIDs).To(Equal([]string{earlierStuckTask.ID, laterStuckTask.ID})) }) - It("records metrics of failed tasks", func() { - updated.State = task.TaskStateFailed - failedTask, err := repository.UpdateFromState(ctx, updated, tsk.State) + It("only unsticks tasks matching the repository type filter", func() { + stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + otherTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + filteredRepository := str.WithTypeFilter(stuckTask.Type).NewTaskRepository() + unstuckTaskIDs, err := filteredRepository.UnstickTasks(ctx) Expect(err).ToNot(HaveOccurred()) + Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) - prometheusState := strings.ReplaceAll(defaultPrometheusOutput, "", task.TaskStateFailed) - expectedOutput := strings.ReplaceAll(prometheusState, "", failedTask.Type) - - prometheusErr := testutil. - CollectAndCompare(taskStoreMongo.TasksStateTotal, strings.NewReader(expectedOutput), metricName) - Expect(prometheusErr).ToNot(HaveOccurred()) + actualOtherTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": otherTask.ID}).Decode(actualOtherTask)).To(Succeed()) + Expect(actualOtherTask).To(Equal(otherTask)) }) + }) + }) - It("records metrics of running tasks", func() { - updated.State = task.TaskStateRunning - runningTask, err := repository.UpdateFromState(ctx, updated, tsk.State) + Context("StopTask", func() { + It("clears the run time and duration when stopping without a duration", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) - Expect(err).ToNot(HaveOccurred()) + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(startedTask.RunTime).ToNot(BeNil()) - prometheusState := strings.ReplaceAll(defaultPrometheusOutput, "", task.TaskStateRunning) - expectedOutput := strings.ReplaceAll(prometheusState, "", runningTask.Type) + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.StateLock, task.TaskStatePending, nil, nil)).To(Succeed()) - prometheusErr := testutil. - CollectAndCompare(taskStoreMongo.TasksStateTotal, strings.NewReader(expectedOutput), metricName) - Expect(prometheusErr).ToNot(HaveOccurred()) - }) + actualTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + Expect(actualTask.RunTime).To(BeNil()) + Expect(actualTask.Duration).To(BeNil()) + Expect(actualTask.StateLock).To(BeNil()) + Expect(actualTask.DeadlineTime).To(BeNil()) + }) - It("records metrics of pending tasks", func() { - tskCreate := &task.TaskCreate{ - Name: pointer.FromString("test"), - Type: "fetch", - Priority: 0, - Data: nil, - AvailableTime: pointer.FromTime(time.Now()), - ExpirationTime: pointer.FromTime(time.Now().Add(5 * time.Minute)), - } - _, err := repository.CreateTask(ctx, tskCreate) - Expect(err).ToNot(HaveOccurred()) + It("retains the run time when stopping with a duration", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) - prometheusState := strings.ReplaceAll(defaultPrometheusOutput, "", task.TaskStatePending) - expectedOutput := strings.ReplaceAll(prometheusState, "", tsk.Type) + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(startedTask.RunTime).ToNot(BeNil()) - prometheusErr := testutil. - CollectAndCompare(taskStoreMongo.TasksStateTotal, strings.NewReader(expectedOutput), metricName) - Expect(prometheusErr).ToNot(HaveOccurred()) - }) + duration := time.Second + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.StateLock, task.TaskStateCompleted, &duration, nil)).To(Succeed()) + actualTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.RunTime).To(PointTo(BeTemporally("~", *startedTask.RunTime, time.Millisecond))) + Expect(actualTask.Duration).To(PointTo(Equal(duration.Seconds()))) + Expect(actualTask.StateLock).To(BeNil()) + Expect(actualTask.DeadlineTime).To(BeNil()) + }) + }) + + Context("EnsureEHRReconcileTask", func() { + BeforeEach(func() { + taskStoreMongo.TasksStateTotal.Reset() + }) + + It("creates the task and increments the pending metric only on the initial insert", func() { + repository := str.TaskRepository() + Expect(repository).ToNot(BeNil()) + + Expect(repository.EnsureEHRReconcileTask(ctx)).To(Succeed()) + Expect(testutil.ToFloat64(taskStoreMongo.TasksStateTotal)).To(Equal(1.0)) + + Expect(repository.EnsureEHRReconcileTask(ctx)).To(Succeed()) + Expect(testutil.ToFloat64(taskStoreMongo.TasksStateTotal)).To(Equal(1.0)) + + count := test.Must(collection.CountDocuments(context.Background(), bson.M{"type": reconcile.Type})) + Expect(count).To(Equal(int64(1))) }) }) }) }) }) + +func insertTaskWithStateAndDeadlineTime(ctx context.Context, collection *mongo.Collection, state string, deadlineTime *time.Time) *task.Task { + tsk := test.Must(task.NewTask(ctx, taskTest.RandomTaskCreate())) + tsk.State = state + tsk.DeadlineTime = deadlineTime + result := test.Must(collection.InsertOne(ctx, tsk)) + Expect(collection.FindOne(ctx, bson.M{"_id": result.InsertedID}).Decode(tsk)).To(Succeed()) + return tsk +} diff --git a/task/store/store.go b/task/store/store.go index 655acca98e..8df4b02174 100644 --- a/task/store/store.go +++ b/task/store/store.go @@ -2,9 +2,12 @@ package store import ( "context" + "time" "go.mongodb.org/mongo-driver/mongo" + "github.com/tidepool-org/platform/page" + storeStructured "github.com/tidepool-org/platform/store/structured" "github.com/tidepool-org/platform/task" ) @@ -15,10 +18,16 @@ type Store interface { } type TaskRepository interface { - task.TaskAccessor + ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) + CreateTask(ctx context.Context, create *task.TaskCreate) (*task.Task, error) + GetTask(ctx context.Context, id string, condition *storeStructured.Condition) (*task.Task, error) + UpdateTask(ctx context.Context, id string, condition *storeStructured.Condition, update *task.TaskUpdate) (*task.Task, error) + DeleteTask(ctx context.Context, id string, condition *storeStructured.Condition) error - UnstickTasks(ctx context.Context) (int64, error) + UnstickTasks(ctx context.Context) ([]string, error) + + StartTask(ctx context.Context, id string, revision int, deadline time.Duration) (*task.Task, error) + StopTask(ctx context.Context, id string, stateLock *string, state string, duration *time.Duration, update *task.TaskUpdate) error - UpdateFromState(ctx context.Context, tsk *task.Task, state string) (*task.Task, error) IteratePending(ctx context.Context) (*mongo.Cursor, error) } diff --git a/task/task.go b/task/task.go index 1c030cf949..f3e97869fa 100644 --- a/task/task.go +++ b/task/task.go @@ -19,15 +19,16 @@ import ( //go:generate mockgen -source=task.go -destination=test/task_mocks.go -package=test -typed type Client interface { - TaskAccessor -} - -type TaskAccessor interface { ListTasks(ctx context.Context, filter *TaskFilter, pagination *page.Pagination) (Tasks, error) CreateTask(ctx context.Context, create *TaskCreate) (*Task, error) - GetTask(ctx context.Context, id string) (*Task, error) - UpdateTask(ctx context.Context, id string, update *TaskUpdate) (*Task, error) - DeleteTask(ctx context.Context, id string) error + GetTask(ctx context.Context, id string, condition *request.Condition) (*Task, error) + // UpdateTask applies the update and returns the resulting task. A runner that updates its + // own running task (for example, to persist progress in Data) must replace its in-memory + // task with the returned task: the update changes fields such as the revision, and the + // queue writes the in-memory task's fields back when it completes the task, so a stale + // in-memory copy would overwrite the update. + UpdateTask(ctx context.Context, id string, condition *request.Condition, update *TaskUpdate) (*Task, error) + DeleteTask(ctx context.Context, id string, condition *request.Condition) error } const ( @@ -83,12 +84,10 @@ func (t *TaskFilter) MutateRequest(req *http.Request) error { } type TaskCreate struct { - Name *string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Priority int `json:"priority,omitempty"` - Data map[string]interface{} `json:"data,omitempty"` - AvailableTime *time.Time `json:"availableTime,omitempty"` - ExpirationTime *time.Time `json:"expirationTime,omitempty"` + Name *string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Data map[string]any `json:"data,omitempty"` + AvailableTime *time.Time `json:"availableTime,omitempty"` } func NewTaskCreate() *TaskCreate { @@ -100,31 +99,21 @@ func (t *TaskCreate) Parse(parser structure.ObjectParser) { if ptr := parser.String("type"); ptr != nil { t.Type = *ptr } - if ptr := parser.Int("priority"); ptr != nil { - t.Priority = *ptr - } if ptr := parser.Object("data"); ptr != nil { t.Data = *ptr } t.AvailableTime = parser.Time("availableTime", time.RFC3339Nano) - t.ExpirationTime = parser.Time("expirationTime", time.RFC3339Nano) } func (t *TaskCreate) Validate(validator structure.Validator) { validator.String("name", t.Name).NotEmpty() validator.String("type", &t.Type).NotEmpty() - expirationTimeValidator := validator.Time("expirationTime", t.ExpirationTime) - expirationTimeValidator.AfterNow(time.Second) - if t.AvailableTime != nil { - expirationTimeValidator.After(*t.AvailableTime) - } } type TaskUpdate struct { - Priority *int `json:"priority,omitempty" bson:"priority,omitempty"` - Data *map[string]interface{} `json:"data,omitempty" bson:"data,omitempty"` - AvailableTime *time.Time `json:"availableTime,omitempty" bson:"availableTime,omitempty"` - ExpirationTime *time.Time `json:"expirationTime,omitempty" bson:"expirationTime,omitempty"` + Data *map[string]any `json:"data,omitempty" bson:"data,omitempty"` + AvailableTime *time.Time `json:"availableTime,omitempty" bson:"availableTime,omitempty"` + Error *errors.Serializable `json:"error,omitempty" bson:"error,omitempty"` } func NewTaskUpdate() *TaskUpdate { @@ -132,22 +121,28 @@ func NewTaskUpdate() *TaskUpdate { } func (t *TaskUpdate) Parse(parser structure.ObjectParser) { - t.Priority = parser.Int("priority") t.Data = parser.Object("data") t.AvailableTime = parser.Time("availableTime", time.RFC3339Nano) - t.ExpirationTime = parser.Time("expirationTime", time.RFC3339Nano) + if parser.ReferenceExists("error") { + t.Error = &errors.Serializable{} + t.Error.Parse("error", parser) + } } func (t *TaskUpdate) Validate(validator structure.Validator) { - expirationTimeValidator := validator.Time("expirationTime", t.ExpirationTime) - expirationTimeValidator.AfterNow(time.Second) - if t.AvailableTime != nil { - expirationTimeValidator.After(*t.AvailableTime) + if t.Error != nil { + t.Error.Validate(validator.WithReference("error")) + } +} + +func (t *TaskUpdate) Normalize(normalizer structure.Normalizer) { + if t.Error != nil { + t.Error.Normalize(normalizer.WithReference("error")) } } func (t *TaskUpdate) IsEmpty() bool { - return t.Priority == nil && t.Data == nil && t.AvailableTime == nil && t.ExpirationTime == nil + return t.Data == nil && t.AvailableTime == nil && t.Error == nil } func NewID() string { @@ -178,20 +173,24 @@ func ErrorValueStringAsIDNotValid(value string) error { var idExpression = regexp.MustCompile("^[0-9a-f]{32}$") type Task struct { - ID string `json:"id,omitempty" bson:"id,omitempty"` - Name *string `json:"name,omitempty" bson:"name,omitempty"` - Type string `json:"type,omitempty" bson:"type,omitempty"` - Priority int `json:"priority,omitempty" bson:"priority,omitempty"` - Data map[string]interface{} `json:"data,omitempty" bson:"data,omitempty"` - AvailableTime *time.Time `json:"availableTime,omitempty" bson:"availableTime,omitempty"` - DeadlineTime *time.Time `json:"deadlineTime,omitempty" bson:"deadlineTime,omitempty"` - ExpirationTime *time.Time `json:"expirationTime,omitempty" bson:"expirationTime,omitempty"` - State string `json:"state,omitempty" bson:"state,omitempty"` - Error *errors.Serializable `json:"error,omitempty" bson:"error,omitempty"` - RunTime *time.Time `json:"runTime,omitempty" bson:"runTime,omitempty"` - Duration *float64 `json:"duration,omitempty" bson:"duration,omitempty"` - CreatedTime time.Time `json:"createdTime,omitempty" bson:"createdTime,omitempty"` - ModifiedTime *time.Time `json:"modifiedTime,omitempty" bson:"modifiedTime,omitempty"` + ID string `json:"id" bson:"id"` + Name *string `json:"name,omitempty" bson:"name,omitempty"` + Type string `json:"type" bson:"type"` + Data map[string]any `json:"data,omitempty" bson:"data,omitempty"` + AvailableTime *time.Time `json:"availableTime,omitempty" bson:"availableTime,omitempty"` + DeadlineTime *time.Time `json:"deadlineTime,omitempty" bson:"deadlineTime,omitempty"` + State string `json:"state" bson:"state"` + Error *errors.Serializable `json:"error,omitempty" bson:"error,omitempty"` + RunTime *time.Time `json:"runTime,omitempty" bson:"runTime,omitempty"` + Duration *float64 `json:"duration,omitempty" bson:"duration,omitempty"` + CreatedTime time.Time `json:"createdTime" bson:"createdTime"` + ModifiedTime *time.Time `json:"modifiedTime,omitempty" bson:"modifiedTime,omitempty"` + Revision int `json:"revision" bson:"revision"` + + // Database only + + // Use to enforce only one state transition at a time. This is a unique value that changes on every update. + StateLock *string `json:"-" bson:"stateLock,omitempty"` } func NewTask(ctx context.Context, create *TaskCreate) (*Task, error) { @@ -201,16 +200,22 @@ func NewTask(ctx context.Context, create *TaskCreate) (*Task, error) { return nil, errors.Wrap(err, "create is invalid") } + now := time.Now().UTC() + + availableTime := create.AvailableTime + if availableTime == nil || availableTime.Before(now) { + availableTime = pointer.From(now) + } + return &Task{ - ID: NewID(), - Name: create.Name, - Type: create.Type, - Priority: create.Priority, - Data: create.Data, - AvailableTime: create.AvailableTime, - ExpirationTime: create.ExpirationTime, - State: TaskStatePending, - CreatedTime: time.Now(), + ID: NewID(), + Name: create.Name, + Type: create.Type, + Data: create.Data, + AvailableTime: availableTime, + State: TaskStatePending, + CreatedTime: now, + Revision: 1, }, nil } @@ -222,14 +227,10 @@ func (t *Task) Parse(parser structure.ObjectParser) { if ptr := parser.String("type"); ptr != nil { t.Type = *ptr } - if ptr := parser.Int("priority"); ptr != nil { - t.Priority = *ptr - } if ptr := parser.Object("data"); ptr != nil { t.Data = *ptr } t.AvailableTime = parser.Time("availableTime", time.RFC3339Nano) - t.ExpirationTime = parser.Time("expirationTime", time.RFC3339Nano) if ptr := parser.String("state"); ptr != nil { t.State = *ptr } @@ -243,17 +244,15 @@ func (t *Task) Parse(parser structure.ObjectParser) { t.CreatedTime = *ptr } t.ModifiedTime = parser.Time("modifiedTime", time.RFC3339Nano) + if ptr := parser.Int("revision"); ptr != nil { + t.Revision = *ptr + } } func (t *Task) Validate(validator structure.Validator) { validator.String("id", &t.ID).Using(IDValidator) validator.String("name", t.Name).NotEmpty() validator.String("type", &t.Type).NotEmpty() - expirationTimeValidator := validator.Time("expirationTime", t.ExpirationTime) - expirationTimeValidator.AfterNow(time.Second) - if t.AvailableTime != nil { - expirationTimeValidator.After(*t.AvailableTime) - } validator.String("state", &t.State).OneOf(TaskStates()...) if t.Error != nil { t.Error.Validate(validator.WithReference("error")) @@ -262,6 +261,7 @@ func (t *Task) Validate(validator structure.Validator) { validator.Float64("duration", t.Duration).GreaterThanOrEqualTo(0) validator.Time("createdTime", &t.CreatedTime).NotZero().BeforeNow(time.Second) validator.Time("modifiedTime", t.ModifiedTime).After(t.CreatedTime).BeforeNow(time.Second) + validator.Int("revision", &t.Revision).GreaterThanOrEqualTo(0) } func (t *Task) Normalize(normalizer structure.Normalizer) { @@ -327,6 +327,16 @@ func (t *Task) ClearError() { t.Error = nil } +func (t *Task) LogFields() log.Fields { + return log.Fields{ + "id": t.ID, + "type": t.Type, + "state": t.State, + "revision": t.Revision, + "stateLock": t.StateLock, + } +} + type Tasks []*Task func (t Tasks) Sanitize(details request.AuthDetails) error { @@ -337,5 +347,3 @@ func (t Tasks) Sanitize(details request.AuthDetails) error { } return nil } - -var AlreadyClaimedTask = errors.New("Task has already been claimed or is now unavailable.") diff --git a/task/task_test.go b/task/task_test.go index 8021a84d89..4ff3ca0df2 100644 --- a/task/task_test.go +++ b/task/task_test.go @@ -1,13 +1,22 @@ package task_test import ( + "context" + "time" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + "github.com/tidepool-org/platform/errors" errorsTest "github.com/tidepool-org/platform/errors/test" + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + "github.com/tidepool-org/platform/pointer" structureTest "github.com/tidepool-org/platform/structure/test" structureValidator "github.com/tidepool-org/platform/structure/validator" "github.com/tidepool-org/platform/task" + taskTest "github.com/tidepool-org/platform/task/test" "github.com/tidepool-org/platform/test" ) @@ -48,4 +57,78 @@ var _ = Describe("Task", func() { Entry("is ErrorValueStringAsIDNotValid with non-empty string", task.ErrorValueStringAsIDNotValid("0123456789abcdef0123456789abcdef"), "value-not-valid", "value is not valid", `value "0123456789abcdef0123456789abcdef" is not valid as task id`), ) }) + + Context("Task", func() { + Context("NewTask", func() { + var ctx context.Context + var create *task.TaskCreate + + BeforeEach(func() { + ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) + create = taskTest.RandomTaskCreate(test.AllowOptionals()) + }) + + It("returns an error when the create is missing", func() { + result, err := task.NewTask(ctx, nil) + errorsTest.ExpectEqual(err, errors.New("create is missing")) + Expect(result).To(BeNil()) + }) + + It("returns an error when the create is invalid", func() { + create.Type = "" + result, err := task.NewTask(ctx, create) + errorsTest.ExpectEqual(err, errors.New("create is invalid")) + Expect(result).To(BeNil()) + }) + + It("returns a new pending task using the current time as the available time when the create does not specify one", func() { + create.AvailableTime = nil + + result := test.Must(task.NewTask(ctx, create)) + Expect(result).To(PointTo(MatchAllFields(Fields{ + "ID": MatchRegexp("^[0-9a-f]{32}$"), + "Name": Equal(create.Name), + "Type": Equal(create.Type), + "Data": Equal(create.Data), + "AvailableTime": PointTo(BeTemporally("~", time.Now(), time.Second)), + "DeadlineTime": BeNil(), + "State": Equal(task.TaskStatePending), + "Error": BeNil(), + "RunTime": BeNil(), + "Duration": BeNil(), + "CreatedTime": BeTemporally("~", time.Now(), time.Second), + "ModifiedTime": BeNil(), + "Revision": Equal(1), + "StateLock": BeNil(), + }))) + }) + + It("returns a new pending task using the current time as the available time when the create specifies one in the past", func() { + create.AvailableTime = pointer.From(test.RandomTimeBeforeNow()) + + result := test.Must(task.NewTask(ctx, create)) + Expect(result.AvailableTime).To(PointTo(BeTemporally("~", time.Now(), time.Second))) + }) + + It("returns a new pending task using the specified available time when it is in the future", func() { + create.AvailableTime = pointer.From(test.RandomTimeAfterNow()) + + result := test.Must(task.NewTask(ctx, create)) + Expect(result.AvailableTime).To(PointTo(Equal(*create.AvailableTime))) + }) + }) + + Context("LogFields", func() { + It("returns the id, type, and state as log fields", func() { + tsk := taskTest.RandomTask() + Expect(tsk.LogFields()).To(Equal(log.Fields{ + "id": tsk.ID, + "type": tsk.Type, + "state": tsk.State, + "revision": tsk.Revision, + "stateLock": tsk.StateLock, + })) + }) + }) + }) }) diff --git a/task/test/client.go b/task/test/client.go index f160c98d5f..03cf75a221 100644 --- a/task/test/client.go +++ b/task/test/client.go @@ -1,15 +1,150 @@ package test +import ( + "context" + + "github.com/onsi/gomega" + + "github.com/tidepool-org/platform/page" + request "github.com/tidepool-org/platform/request" + "github.com/tidepool-org/platform/task" +) + +type ListTasksInput struct { + Context context.Context //nolint:containedctx // Test only + Filter *task.TaskFilter + Pagination *page.Pagination +} + +type ListTasksOutput struct { + Tasks task.Tasks + Error error +} + +type CreateTaskInput struct { + Context context.Context //nolint:containedctx // Test only + Create *task.TaskCreate +} + +type CreateTaskOutput struct { + Task *task.Task + Error error +} + +type GetTaskInput struct { + Context context.Context //nolint:containedctx // Test only + ID string + Condition *request.Condition +} + +type GetTaskOutput struct { + Task *task.Task + Error error +} + +type UpdateTaskInput struct { + Context context.Context //nolint:containedctx // Test only + ID string + Condition *request.Condition + Update *task.TaskUpdate +} + +type UpdateTaskOutput struct { + Task *task.Task + Error error +} + +type DeleteTaskInput struct { + Context context.Context //nolint:containedctx // Test only + ID string + Condition *request.Condition +} + type Client struct { - *TaskAccessor + ListTasksInvocations int + ListTasksInputs []ListTasksInput + ListTasksOutputs []ListTasksOutput + CreateTaskInvocations int + CreateTaskInputs []CreateTaskInput + CreateTaskOutputs []CreateTaskOutput + GetTaskInvocations int + GetTaskInputs []GetTaskInput + GetTaskOutputs []GetTaskOutput + UpdateTaskInvocations int + UpdateTaskInputs []UpdateTaskInput + UpdateTaskOutputs []UpdateTaskOutput + DeleteTaskInvocations int + DeleteTaskInputs []DeleteTaskInput + DeleteTaskOutputs []error } func NewClient() *Client { - return &Client{ - TaskAccessor: NewTaskAccessor(), - } + return &Client{} +} + +func (t *Client) ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) { + t.ListTasksInvocations++ + + t.ListTasksInputs = append(t.ListTasksInputs, ListTasksInput{Context: ctx, Filter: filter, Pagination: pagination}) + + gomega.Expect(t.ListTasksOutputs).ToNot(gomega.BeEmpty()) + + output := t.ListTasksOutputs[0] + t.ListTasksOutputs = t.ListTasksOutputs[1:] + return output.Tasks, output.Error +} + +func (t *Client) CreateTask(ctx context.Context, create *task.TaskCreate) (*task.Task, error) { + t.CreateTaskInvocations++ + + t.CreateTaskInputs = append(t.CreateTaskInputs, CreateTaskInput{Context: ctx, Create: create}) + + gomega.Expect(t.CreateTaskOutputs).ToNot(gomega.BeEmpty()) + + output := t.CreateTaskOutputs[0] + t.CreateTaskOutputs = t.CreateTaskOutputs[1:] + return output.Task, output.Error +} + +func (t *Client) GetTask(ctx context.Context, id string, condition *request.Condition) (*task.Task, error) { + t.GetTaskInvocations++ + + t.GetTaskInputs = append(t.GetTaskInputs, GetTaskInput{Context: ctx, ID: id, Condition: condition}) + + gomega.Expect(t.GetTaskOutputs).ToNot(gomega.BeEmpty()) + + output := t.GetTaskOutputs[0] + t.GetTaskOutputs = t.GetTaskOutputs[1:] + return output.Task, output.Error +} + +func (t *Client) UpdateTask(ctx context.Context, id string, condition *request.Condition, update *task.TaskUpdate) (*task.Task, error) { + t.UpdateTaskInvocations++ + + t.UpdateTaskInputs = append(t.UpdateTaskInputs, UpdateTaskInput{Context: ctx, ID: id, Condition: condition, Update: update}) + + gomega.Expect(t.UpdateTaskOutputs).ToNot(gomega.BeEmpty()) + + output := t.UpdateTaskOutputs[0] + t.UpdateTaskOutputs = t.UpdateTaskOutputs[1:] + return output.Task, output.Error +} + +func (t *Client) DeleteTask(ctx context.Context, id string, condition *request.Condition) error { + t.DeleteTaskInvocations++ + + t.DeleteTaskInputs = append(t.DeleteTaskInputs, DeleteTaskInput{Context: ctx, ID: id, Condition: condition}) + + gomega.Expect(t.DeleteTaskOutputs).ToNot(gomega.BeEmpty()) + + output := t.DeleteTaskOutputs[0] + t.DeleteTaskOutputs = t.DeleteTaskOutputs[1:] + return output } -func (c *Client) Expectations() { - c.TaskAccessor.Expectations() +func (t *Client) Expectations() { + gomega.Expect(t.ListTasksOutputs).To(gomega.BeEmpty()) + gomega.Expect(t.CreateTaskOutputs).To(gomega.BeEmpty()) + gomega.Expect(t.GetTaskOutputs).To(gomega.BeEmpty()) + gomega.Expect(t.UpdateTaskOutputs).To(gomega.BeEmpty()) } diff --git a/task/test/task.go b/task/test/task.go new file mode 100644 index 0000000000..3bce102f0f --- /dev/null +++ b/task/test/task.go @@ -0,0 +1,92 @@ +package test + +import ( + "time" + + errorsTest "github.com/tidepool-org/platform/errors/test" + metadataTest "github.com/tidepool-org/platform/metadata/test" + "github.com/tidepool-org/platform/pointer" + "github.com/tidepool-org/platform/task" + "github.com/tidepool-org/platform/test" +) + +func RandomID() string { + return task.NewID() +} + +func RandomName() string { + return test.RandomString() +} + +func RandomType() string { + return test.RandomString() +} + +func RandomData(options ...test.Option) map[string]any { + return metadataTest.RandomOptionalMetadataMap(options...) +} + +func RandomState() string { + return test.RandomStringFromArray(task.TaskStates()) +} + +func RandomTaskCreate(options ...test.Option) *task.TaskCreate { + now := test.Now() + availableTime := test.RandomTimeAfter(now) + return &task.TaskCreate{ + Name: test.RandomOptional(RandomName, options...), + Type: RandomType(), + Data: RandomData(options...), + AvailableTime: test.RandomOptional(test.Constant(availableTime), options...), + } +} + +func RandomTask(options ...test.Option) *task.Task { + now := test.Now() + + tsk := &task.Task{ + ID: RandomID(), + Name: test.RandomOptional(RandomName, options...), + Type: RandomType(), + Data: RandomData(options...), + State: RandomState(), + CreatedTime: test.RandomTimeBefore(now), + } + + switch tsk.State { + case task.TaskStatePending: + tsk.AvailableTime = pointer.From(test.RandomTimeAfterNow()) + tsk.DeadlineTime = nil + tsk.ModifiedTime = test.RandomOptional(func() time.Time { return test.RandomTimeFromRange(tsk.CreatedTime, now) }, options...) + if tsk.ModifiedTime != nil { + tsk.Error = test.RandomOptionalPointer(errorsTest.RandomSerializable, options...) + tsk.RunTime = tsk.ModifiedTime + tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + } + case task.TaskStateRunning: + tsk.AvailableTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, now)) + tsk.DeadlineTime = pointer.From(test.RandomTimeAfterNow()) + tsk.ModifiedTime = pointer.From(test.RandomTimeFromRange(*tsk.AvailableTime, now)) + if test.RandomBool() { + tsk.Error = test.RandomOptionalPointer(errorsTest.RandomSerializable, options...) + tsk.RunTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, *tsk.AvailableTime)) + tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + } + case task.TaskStateFailed: + tsk.AvailableTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, now)) + tsk.DeadlineTime = nil + tsk.ModifiedTime = pointer.From(test.RandomTimeFromRange(*tsk.AvailableTime, now)) + tsk.Error = errorsTest.RandomSerializable() + tsk.RunTime = tsk.ModifiedTime + tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + case task.TaskStateCompleted: + tsk.AvailableTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, now)) + tsk.DeadlineTime = nil + tsk.ModifiedTime = pointer.From(test.RandomTimeFromRange(*tsk.AvailableTime, now)) + tsk.Error = test.RandomOptionalPointer(errorsTest.RandomSerializable, options...) + tsk.RunTime = tsk.ModifiedTime + tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + } + + return tsk +} diff --git a/task/test/task_accessor.go b/task/test/task_accessor.go deleted file mode 100644 index f71b10aa9f..0000000000 --- a/task/test/task_accessor.go +++ /dev/null @@ -1,146 +0,0 @@ -package test - -import ( - "context" - - "github.com/onsi/gomega" - - "github.com/tidepool-org/platform/page" - "github.com/tidepool-org/platform/task" -) - -type ListTasksInput struct { - Context context.Context - Filter *task.TaskFilter - Pagination *page.Pagination -} - -type ListTasksOutput struct { - Tasks task.Tasks - Error error -} - -type CreateTaskInput struct { - Context context.Context - Create *task.TaskCreate -} - -type CreateTaskOutput struct { - Task *task.Task - Error error -} - -type GetTaskInput struct { - Context context.Context - ID string -} - -type GetTaskOutput struct { - Task *task.Task - Error error -} - -type UpdateTaskInput struct { - Context context.Context - ID string - Update *task.TaskUpdate -} - -type UpdateTaskOutput struct { - Task *task.Task - Error error -} - -type DeleteTaskInput struct { - Context context.Context - ID string -} - -type TaskAccessor struct { - ListTasksInvocations int - ListTasksInputs []ListTasksInput - ListTasksOutputs []ListTasksOutput - CreateTaskInvocations int - CreateTaskInputs []CreateTaskInput - CreateTaskOutputs []CreateTaskOutput - GetTaskInvocations int - GetTaskInputs []GetTaskInput - GetTaskOutputs []GetTaskOutput - UpdateTaskInvocations int - UpdateTaskInputs []UpdateTaskInput - UpdateTaskOutputs []UpdateTaskOutput - DeleteTaskInvocations int - DeleteTaskInputs []DeleteTaskInput - DeleteTaskOutputs []error -} - -func NewTaskAccessor() *TaskAccessor { - return &TaskAccessor{} -} - -func (t *TaskAccessor) ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) { - t.ListTasksInvocations++ - - t.ListTasksInputs = append(t.ListTasksInputs, ListTasksInput{Context: ctx, Filter: filter, Pagination: pagination}) - - gomega.Expect(t.ListTasksOutputs).ToNot(gomega.BeEmpty()) - - output := t.ListTasksOutputs[0] - t.ListTasksOutputs = t.ListTasksOutputs[1:] - return output.Tasks, output.Error -} - -func (t *TaskAccessor) CreateTask(ctx context.Context, create *task.TaskCreate) (*task.Task, error) { - t.CreateTaskInvocations++ - - t.CreateTaskInputs = append(t.CreateTaskInputs, CreateTaskInput{Context: ctx, Create: create}) - - gomega.Expect(t.CreateTaskOutputs).ToNot(gomega.BeEmpty()) - - output := t.CreateTaskOutputs[0] - t.CreateTaskOutputs = t.CreateTaskOutputs[1:] - return output.Task, output.Error -} - -func (t *TaskAccessor) GetTask(ctx context.Context, id string) (*task.Task, error) { - t.GetTaskInvocations++ - - t.GetTaskInputs = append(t.GetTaskInputs, GetTaskInput{Context: ctx, ID: id}) - - gomega.Expect(t.GetTaskOutputs).ToNot(gomega.BeEmpty()) - - output := t.GetTaskOutputs[0] - t.GetTaskOutputs = t.GetTaskOutputs[1:] - return output.Task, output.Error -} - -func (t *TaskAccessor) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { - t.UpdateTaskInvocations++ - - t.UpdateTaskInputs = append(t.UpdateTaskInputs, UpdateTaskInput{Context: ctx, ID: id, Update: update}) - - gomega.Expect(t.UpdateTaskOutputs).ToNot(gomega.BeEmpty()) - - output := t.UpdateTaskOutputs[0] - t.UpdateTaskOutputs = t.UpdateTaskOutputs[1:] - return output.Task, output.Error -} - -func (t *TaskAccessor) DeleteTask(ctx context.Context, id string) error { - t.DeleteTaskInvocations++ - - t.DeleteTaskInputs = append(t.DeleteTaskInputs, DeleteTaskInput{Context: ctx, ID: id}) - - gomega.Expect(t.DeleteTaskOutputs).ToNot(gomega.BeEmpty()) - - output := t.DeleteTaskOutputs[0] - t.DeleteTaskOutputs = t.DeleteTaskOutputs[1:] - return output -} - -func (t *TaskAccessor) Expectations() { - gomega.Expect(t.ListTasksOutputs).To(gomega.BeEmpty()) - gomega.Expect(t.CreateTaskOutputs).To(gomega.BeEmpty()) - gomega.Expect(t.GetTaskOutputs).To(gomega.BeEmpty()) - gomega.Expect(t.UpdateTaskOutputs).To(gomega.BeEmpty()) -} diff --git a/task/test/task_mocks.go b/task/test/task_mocks.go index c68f2e35b0..d4401f7706 100644 --- a/task/test/task_mocks.go +++ b/task/test/task_mocks.go @@ -16,6 +16,7 @@ import ( gomock "go.uber.org/mock/gomock" page "github.com/tidepool-org/platform/page" + request "github.com/tidepool-org/platform/request" task "github.com/tidepool-org/platform/task" ) @@ -83,17 +84,17 @@ func (c *MockClientCreateTaskCall) DoAndReturn(f func(context.Context, *task.Tas } // DeleteTask mocks base method. -func (m *MockClient) DeleteTask(ctx context.Context, id string) error { +func (m *MockClient) DeleteTask(ctx context.Context, id string, condition *request.Condition) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteTask", ctx, id) + ret := m.ctrl.Call(m, "DeleteTask", ctx, id, condition) ret0, _ := ret[0].(error) return ret0 } // DeleteTask indicates an expected call of DeleteTask. -func (mr *MockClientMockRecorder) DeleteTask(ctx, id any) *MockClientDeleteTaskCall { +func (mr *MockClientMockRecorder) DeleteTask(ctx, id, condition any) *MockClientDeleteTaskCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTask", reflect.TypeOf((*MockClient)(nil).DeleteTask), ctx, id) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTask", reflect.TypeOf((*MockClient)(nil).DeleteTask), ctx, id, condition) return &MockClientDeleteTaskCall{Call: call} } @@ -109,30 +110,30 @@ func (c *MockClientDeleteTaskCall) Return(arg0 error) *MockClientDeleteTaskCall } // Do rewrite *gomock.Call.Do -func (c *MockClientDeleteTaskCall) Do(f func(context.Context, string) error) *MockClientDeleteTaskCall { +func (c *MockClientDeleteTaskCall) Do(f func(context.Context, string, *request.Condition) error) *MockClientDeleteTaskCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientDeleteTaskCall) DoAndReturn(f func(context.Context, string) error) *MockClientDeleteTaskCall { +func (c *MockClientDeleteTaskCall) DoAndReturn(f func(context.Context, string, *request.Condition) error) *MockClientDeleteTaskCall { c.Call = c.Call.DoAndReturn(f) return c } // GetTask mocks base method. -func (m *MockClient) GetTask(ctx context.Context, id string) (*task.Task, error) { +func (m *MockClient) GetTask(ctx context.Context, id string, condition *request.Condition) (*task.Task, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTask", ctx, id) + ret := m.ctrl.Call(m, "GetTask", ctx, id, condition) ret0, _ := ret[0].(*task.Task) ret1, _ := ret[1].(error) return ret0, ret1 } // GetTask indicates an expected call of GetTask. -func (mr *MockClientMockRecorder) GetTask(ctx, id any) *MockClientGetTaskCall { +func (mr *MockClientMockRecorder) GetTask(ctx, id, condition any) *MockClientGetTaskCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTask", reflect.TypeOf((*MockClient)(nil).GetTask), ctx, id) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTask", reflect.TypeOf((*MockClient)(nil).GetTask), ctx, id, condition) return &MockClientGetTaskCall{Call: call} } @@ -148,13 +149,13 @@ func (c *MockClientGetTaskCall) Return(arg0 *task.Task, arg1 error) *MockClientG } // Do rewrite *gomock.Call.Do -func (c *MockClientGetTaskCall) Do(f func(context.Context, string) (*task.Task, error)) *MockClientGetTaskCall { +func (c *MockClientGetTaskCall) Do(f func(context.Context, string, *request.Condition) (*task.Task, error)) *MockClientGetTaskCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientGetTaskCall) DoAndReturn(f func(context.Context, string) (*task.Task, error)) *MockClientGetTaskCall { +func (c *MockClientGetTaskCall) DoAndReturn(f func(context.Context, string, *request.Condition) (*task.Task, error)) *MockClientGetTaskCall { c.Call = c.Call.DoAndReturn(f) return c } @@ -199,18 +200,18 @@ func (c *MockClientListTasksCall) DoAndReturn(f func(context.Context, *task.Task } // UpdateTask mocks base method. -func (m *MockClient) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { +func (m *MockClient) UpdateTask(ctx context.Context, id string, condition *request.Condition, update *task.TaskUpdate) (*task.Task, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateTask", ctx, id, update) + ret := m.ctrl.Call(m, "UpdateTask", ctx, id, condition, update) ret0, _ := ret[0].(*task.Task) ret1, _ := ret[1].(error) return ret0, ret1 } // UpdateTask indicates an expected call of UpdateTask. -func (mr *MockClientMockRecorder) UpdateTask(ctx, id, update any) *MockClientUpdateTaskCall { +func (mr *MockClientMockRecorder) UpdateTask(ctx, id, condition, update any) *MockClientUpdateTaskCall { mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTask", reflect.TypeOf((*MockClient)(nil).UpdateTask), ctx, id, update) + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTask", reflect.TypeOf((*MockClient)(nil).UpdateTask), ctx, id, condition, update) return &MockClientUpdateTaskCall{Call: call} } @@ -226,231 +227,13 @@ func (c *MockClientUpdateTaskCall) Return(arg0 *task.Task, arg1 error) *MockClie } // Do rewrite *gomock.Call.Do -func (c *MockClientUpdateTaskCall) Do(f func(context.Context, string, *task.TaskUpdate) (*task.Task, error)) *MockClientUpdateTaskCall { +func (c *MockClientUpdateTaskCall) Do(f func(context.Context, string, *request.Condition, *task.TaskUpdate) (*task.Task, error)) *MockClientUpdateTaskCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockClientUpdateTaskCall) DoAndReturn(f func(context.Context, string, *task.TaskUpdate) (*task.Task, error)) *MockClientUpdateTaskCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// MockTaskAccessor is a mock of TaskAccessor interface. -type MockTaskAccessor struct { - ctrl *gomock.Controller - recorder *MockTaskAccessorMockRecorder - isgomock struct{} -} - -// MockTaskAccessorMockRecorder is the mock recorder for MockTaskAccessor. -type MockTaskAccessorMockRecorder struct { - mock *MockTaskAccessor -} - -// NewMockTaskAccessor creates a new mock instance. -func NewMockTaskAccessor(ctrl *gomock.Controller) *MockTaskAccessor { - mock := &MockTaskAccessor{ctrl: ctrl} - mock.recorder = &MockTaskAccessorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockTaskAccessor) EXPECT() *MockTaskAccessorMockRecorder { - return m.recorder -} - -// CreateTask mocks base method. -func (m *MockTaskAccessor) CreateTask(ctx context.Context, create *task.TaskCreate) (*task.Task, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTask", ctx, create) - ret0, _ := ret[0].(*task.Task) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CreateTask indicates an expected call of CreateTask. -func (mr *MockTaskAccessorMockRecorder) CreateTask(ctx, create any) *MockTaskAccessorCreateTaskCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTask", reflect.TypeOf((*MockTaskAccessor)(nil).CreateTask), ctx, create) - return &MockTaskAccessorCreateTaskCall{Call: call} -} - -// MockTaskAccessorCreateTaskCall wrap *gomock.Call -type MockTaskAccessorCreateTaskCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorCreateTaskCall) Return(arg0 *task.Task, arg1 error) *MockTaskAccessorCreateTaskCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorCreateTaskCall) Do(f func(context.Context, *task.TaskCreate) (*task.Task, error)) *MockTaskAccessorCreateTaskCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorCreateTaskCall) DoAndReturn(f func(context.Context, *task.TaskCreate) (*task.Task, error)) *MockTaskAccessorCreateTaskCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// DeleteTask mocks base method. -func (m *MockTaskAccessor) DeleteTask(ctx context.Context, id string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteTask", ctx, id) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteTask indicates an expected call of DeleteTask. -func (mr *MockTaskAccessorMockRecorder) DeleteTask(ctx, id any) *MockTaskAccessorDeleteTaskCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTask", reflect.TypeOf((*MockTaskAccessor)(nil).DeleteTask), ctx, id) - return &MockTaskAccessorDeleteTaskCall{Call: call} -} - -// MockTaskAccessorDeleteTaskCall wrap *gomock.Call -type MockTaskAccessorDeleteTaskCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorDeleteTaskCall) Return(arg0 error) *MockTaskAccessorDeleteTaskCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorDeleteTaskCall) Do(f func(context.Context, string) error) *MockTaskAccessorDeleteTaskCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorDeleteTaskCall) DoAndReturn(f func(context.Context, string) error) *MockTaskAccessorDeleteTaskCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// GetTask mocks base method. -func (m *MockTaskAccessor) GetTask(ctx context.Context, id string) (*task.Task, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTask", ctx, id) - ret0, _ := ret[0].(*task.Task) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTask indicates an expected call of GetTask. -func (mr *MockTaskAccessorMockRecorder) GetTask(ctx, id any) *MockTaskAccessorGetTaskCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTask", reflect.TypeOf((*MockTaskAccessor)(nil).GetTask), ctx, id) - return &MockTaskAccessorGetTaskCall{Call: call} -} - -// MockTaskAccessorGetTaskCall wrap *gomock.Call -type MockTaskAccessorGetTaskCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorGetTaskCall) Return(arg0 *task.Task, arg1 error) *MockTaskAccessorGetTaskCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorGetTaskCall) Do(f func(context.Context, string) (*task.Task, error)) *MockTaskAccessorGetTaskCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorGetTaskCall) DoAndReturn(f func(context.Context, string) (*task.Task, error)) *MockTaskAccessorGetTaskCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// ListTasks mocks base method. -func (m *MockTaskAccessor) ListTasks(ctx context.Context, filter *task.TaskFilter, pagination *page.Pagination) (task.Tasks, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListTasks", ctx, filter, pagination) - ret0, _ := ret[0].(task.Tasks) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ListTasks indicates an expected call of ListTasks. -func (mr *MockTaskAccessorMockRecorder) ListTasks(ctx, filter, pagination any) *MockTaskAccessorListTasksCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTasks", reflect.TypeOf((*MockTaskAccessor)(nil).ListTasks), ctx, filter, pagination) - return &MockTaskAccessorListTasksCall{Call: call} -} - -// MockTaskAccessorListTasksCall wrap *gomock.Call -type MockTaskAccessorListTasksCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorListTasksCall) Return(arg0 task.Tasks, arg1 error) *MockTaskAccessorListTasksCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorListTasksCall) Do(f func(context.Context, *task.TaskFilter, *page.Pagination) (task.Tasks, error)) *MockTaskAccessorListTasksCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorListTasksCall) DoAndReturn(f func(context.Context, *task.TaskFilter, *page.Pagination) (task.Tasks, error)) *MockTaskAccessorListTasksCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// UpdateTask mocks base method. -func (m *MockTaskAccessor) UpdateTask(ctx context.Context, id string, update *task.TaskUpdate) (*task.Task, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateTask", ctx, id, update) - ret0, _ := ret[0].(*task.Task) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// UpdateTask indicates an expected call of UpdateTask. -func (mr *MockTaskAccessorMockRecorder) UpdateTask(ctx, id, update any) *MockTaskAccessorUpdateTaskCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTask", reflect.TypeOf((*MockTaskAccessor)(nil).UpdateTask), ctx, id, update) - return &MockTaskAccessorUpdateTaskCall{Call: call} -} - -// MockTaskAccessorUpdateTaskCall wrap *gomock.Call -type MockTaskAccessorUpdateTaskCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *MockTaskAccessorUpdateTaskCall) Return(arg0 *task.Task, arg1 error) *MockTaskAccessorUpdateTaskCall { - c.Call = c.Call.Return(arg0, arg1) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *MockTaskAccessorUpdateTaskCall) Do(f func(context.Context, string, *task.TaskUpdate) (*task.Task, error)) *MockTaskAccessorUpdateTaskCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *MockTaskAccessorUpdateTaskCall) DoAndReturn(f func(context.Context, string, *task.TaskUpdate) (*task.Task, error)) *MockTaskAccessorUpdateTaskCall { +func (c *MockClientUpdateTaskCall) DoAndReturn(f func(context.Context, string, *request.Condition, *task.TaskUpdate) (*task.Task, error)) *MockClientUpdateTaskCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/test/http/http.go b/test/http/http.go index 0603530ef2..303cd41f10 100644 --- a/test/http/http.go +++ b/test/http/http.go @@ -248,3 +248,18 @@ func (r *ResponseWriter) Flush() { } r.ResponseRecorder.Flush() } + +type RoundTripper struct { + Request *http.Request + Response *http.Response + Error error +} + +func NewRoundTripper() *RoundTripper { + return &RoundTripper{} +} + +func (t *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + t.Request = req + return t.Response, t.Error +} diff --git a/test/time.go b/test/time.go index c3070cf4d6..02db1cbed2 100644 --- a/test/time.go +++ b/test/time.go @@ -9,6 +9,10 @@ import ( gomegaTypes "github.com/onsi/gomega/types" ) +func Now() time.Time { + return now +} + func PastFarTime() time.Time { return now.AddDate(-30, 0, 0) } diff --git a/twiist/provider/provider.go b/twiist/provider/provider.go index 20594721de..19d53f509e 100644 --- a/twiist/provider/provider.go +++ b/twiist/provider/provider.go @@ -2,12 +2,14 @@ package provider import ( "context" + "net/http" "time" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/tidepool-org/platform/auth" providerSession "github.com/tidepool-org/platform/auth/providersession" + "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/config" "github.com/tidepool-org/platform/data" dataDeduplicatorDeduplicator "github.com/tidepool-org/platform/data/deduplicator/deduplicator" @@ -65,7 +67,18 @@ func New(providerDependencies ProviderDependencies) (*Provider, error) { return nil, errors.Wrap(err, "unable to create provider config") } - prvdr, err := oauthProvider.New(twiist.ProviderName, cfg, providerDependencies.JWKS) + // Attach prometheus round tripper to default client transport + prometheusRequestMetricsRoundTripper.WithRoundTripper(http.DefaultClient.Transport) + + // Create http client + httpClient := &http.Client{ + Transport: prometheusRequestMetricsRoundTripper, + CheckRedirect: http.DefaultClient.CheckRedirect, + Jar: http.DefaultClient.Jar, + Timeout: 2 * time.Minute, + } + + prvdr, err := oauthProvider.New(twiist.ProviderName, cfg, httpClient, providerDependencies.JWKS) if err != nil { return nil, err } @@ -279,3 +292,5 @@ func NewDataSetCreate() *data.DataSetCreate { TimeProcessing: pointer.FromString(data.TimeProcessingNone), } } + +var prometheusRequestMetricsRoundTripper = client.NewPrometheusRequestMetricsRoundTripper("tidepool_twiist_api", "Tidepool twiist API") From d30d9d013131a75f2b57254dbe9349dcdd603f13 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Mon, 20 Jul 2026 09:33:05 -0700 Subject: [PATCH 02/20] Update Abbott plugin --- private/plugin/abbott | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private/plugin/abbott b/private/plugin/abbott index 806e28a7da..c0fcb7b1d9 160000 --- a/private/plugin/abbott +++ b/private/plugin/abbott @@ -1 +1 @@ -Subproject commit 806e28a7dae5dd2878f99c332d59094741690b8b +Subproject commit c0fcb7b1d9f5f8468af91511e64f56628f49c110 From 4df7d84c8d3d91157845a263fd36e6e8d770213d Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Mon, 20 Jul 2026 10:40:31 -0700 Subject: [PATCH 03/20] Make task deadline to database only - Make task deadline time database only - Add grace period to runner watchdog to remove spurious metrics - Minor updates --- plugin/abbott/go.mod | 1 + plugin/abbott/go.sum | 3 ++ .../tools/dexcom_analyze/dexcom_analyze.go | 32 -------------- task/client/client.go | 2 +- task/queue/multi_test.go | 2 +- task/queue/queue.go | 44 ++++++++++++++----- task/queue/queue_test.go | 39 ++++++++++++---- task/store/mongo/mongo.go | 2 +- task/store/mongo/mongo_test.go | 2 +- task/store/test/task_session.go | 17 ------- task/task.go | 5 +-- task/test/client.go | 1 + task/test/task.go | 8 ++-- 13 files changed, 78 insertions(+), 80 deletions(-) delete mode 100644 task/store/test/task_session.go diff --git a/plugin/abbott/go.mod b/plugin/abbott/go.mod index 988e47d6c3..a90d315439 100644 --- a/plugin/abbott/go.mod +++ b/plugin/abbott/go.mod @@ -20,6 +20,7 @@ require ( github.com/golang/mock v1.6.0 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gowebpki/jcs v1.0.1 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/klauspost/compress v1.18.5 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect diff --git a/plugin/abbott/go.sum b/plugin/abbott/go.sum index 688b299209..a15e308884 100644 --- a/plugin/abbott/go.sum +++ b/plugin/abbott/go.sum @@ -37,6 +37,8 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU= +github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= @@ -80,6 +82,7 @@ github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKk github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= diff --git a/services/tools/dexcom_analyze/dexcom_analyze.go b/services/tools/dexcom_analyze/dexcom_analyze.go index 434d7de4d6..f8c06d002a 100644 --- a/services/tools/dexcom_analyze/dexcom_analyze.go +++ b/services/tools/dexcom_analyze/dexcom_analyze.go @@ -10,7 +10,6 @@ import ( "sort" "strconv" "strings" - "time" "github.com/urfave/cli" "golang.org/x/exp/maps" @@ -93,11 +92,8 @@ const ( Issue_Task_With_DeviceHashes_And_DataSource_LastImportTime_Missing = "task with device hashes and data source last import time missing" Issue_Task_With_DeviceHashes_And_DataSource_LatestDataTime_Missing = "task with device hashes and data source latest data time missing" Issue_Task_With_State_Failed_AvailableTime_Present = "task with state failed available time present" - Issue_Task_With_State_Failed_DeadlineTime_Present = "task with state failed deadline time present" Issue_Task_With_State_Failed_Error_Missing = "task with state failed error missing" - Issue_Task_With_State_Pending_DeadlineTime_Present = "task with state pending deadline time present" Issue_Task_With_State_Running_AvailableTime_Present = "task with state running available time present" - Issue_Task_With_State_Running_DeadlineTime_Missing = "task with state running deadline time missing" Issue_Task_With_State_Running_Error_Present = "task with state running error present" IssueFormat_DataSource_Invalid = "data source invalid ('%s', '%s')" @@ -157,11 +153,8 @@ func Issues() []string { Issue_Task_With_DeviceHashes_And_DataSource_LastImportTime_Missing, Issue_Task_With_DeviceHashes_And_DataSource_LatestDataTime_Missing, Issue_Task_With_State_Failed_AvailableTime_Present, - Issue_Task_With_State_Failed_DeadlineTime_Present, Issue_Task_With_State_Failed_Error_Missing, - Issue_Task_With_State_Pending_DeadlineTime_Present, Issue_Task_With_State_Running_AvailableTime_Present, - Issue_Task_With_State_Running_DeadlineTime_Missing, Issue_Task_With_State_Running_Error_Present, } } @@ -1099,16 +1092,10 @@ func (t *Tool) analyzeTasks() { switch record.State { case task.TaskStatePending: - if record.DeadlineTime != nil { - record.AppendIssue(Issue_Task_With_State_Pending_DeadlineTime_Present) - } case task.TaskStateRunning: if record.AvailableTime != nil { record.AppendIssue(Issue_Task_With_State_Running_AvailableTime_Present) } - if record.DeadlineTime == nil { - record.AppendIssue(Issue_Task_With_State_Running_DeadlineTime_Missing) - } if record.Error != nil { record.AppendIssue(Issue_Task_With_State_Running_Error_Present) } @@ -1116,9 +1103,6 @@ func (t *Tool) analyzeTasks() { if record.AvailableTime != nil { record.AppendIssue(Issue_Task_With_State_Failed_AvailableTime_Present) } - if record.DeadlineTime != nil { - record.AppendIssue(Issue_Task_With_State_Failed_DeadlineTime_Present) - } if record.Error == nil { record.AppendIssue(Issue_Task_With_State_Failed_Error_Missing) } @@ -1330,29 +1314,13 @@ func (t *Tool) outputIssue(issue string, marshalables Marshalables, issueMarshal t.outputMongoWriteOperationsHeader() t.outputMongoOperationf("db.tasks.updateMany({state: 'failed', availableTime: {$exists: true}}, {$unset: {availableTime: true}})") return - case Issue_Task_With_State_Failed_DeadlineTime_Present: - t.outputResolutionHeader("FIXED with BACK-3116. Will keep occurring until deployed. Will need to manually update failed tasks post-deploy.") - t.outputMongoReadOperationsHeader() - t.outputMongoOperationf("db.tasks.countDocuments({state: 'failed', deadlineTime: {$exists: true}})") - t.outputMongoWriteOperationsHeader() - t.outputMongoOperationf("db.tasks.updateMany({state: 'failed', deadlineTime: {$exists: true}}, {$unset: {deadlineTime: true}})") - return case Issue_Task_With_State_Failed_Error_Missing: - case Issue_Task_With_State_Pending_DeadlineTime_Present: case Issue_Task_With_State_Running_AvailableTime_Present: t.outputResolutionHeader("FIXED with BACK-3116. Will keep occurring until deployed. Will need to manually update failed tasks post-deploy.") // NOTE: Uncomment this block to see further details. // t.outputMongoReadOperationsHeader() // t.outputMongoTasksAggregation(marshalables.Tasks().IDs()) return - case Issue_Task_With_State_Running_DeadlineTime_Missing: - t.outputResolutionHeader("Examine each to determine why it is missing.") - t.outputMongoReadOperationsHeader() - t.outputMongoTasksAggregation(marshalables.Tasks().IDs()) - t.outputMongoOperationf("db.tasks.find({id: {$in: [%s]}})", mongoIDs(marshalables.Tasks().IDs())) - t.outputMongoWriteOperationsHeader() - t.outputMongoOperationf("db.tasks.updateMany({id: {$in: [%s]}}, {$set: {deadlineTime: ISODate('%s')}})", mongoIDs(marshalables.Tasks().IDs()), time.Now().Format(time.RFC3339)) - return case Issue_Task_With_State_Running_Error_Present: t.outputResolutionHeader("Examine each to determine why it is present.") t.outputMongoReadOperationsHeader() diff --git a/task/client/client.go b/task/client/client.go index 6c08793c4a..5ffc67c78f 100644 --- a/task/client/client.go +++ b/task/client/client.go @@ -59,7 +59,7 @@ func (c *Client) CreateTask(ctx context.Context, create *task.TaskCreate) (*task if create == nil { return nil, errors.New("create is missing") } else if err := structureValidator.New(log.LoggerFromContext(ctx)).Validate(create); err != nil { - return nil, errors.New("create is invalid") + return nil, errors.Wrap(err, "create is invalid") } url := c.client.ConstructURL("v1", "tasks") diff --git a/task/queue/multi_test.go b/task/queue/multi_test.go index 5d5c5efb5e..5c4a36ec82 100644 --- a/task/queue/multi_test.go +++ b/task/queue/multi_test.go @@ -40,7 +40,7 @@ var _ = Describe("multi queue", func() { Expect(err).ToNot(HaveOccurred()) Expect(str).ToNot(BeNil()) lgr = null.NewLogger() - queueConfig = &queue.Config{Workers: 10, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: queue.DelayUnstickDefault, StopWaitTimeout: queue.StopWaitTimeoutDefault} + queueConfig = &queue.Config{Workers: 10, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: queue.DelayUnstickDefault, StopWaitTimeout: queue.StopWaitTimeoutDefault, RunnerWatchdogGracePeriod: queue.RunnerWatchdogGracePeriodDefault} multi = nil }) diff --git a/task/queue/queue.go b/task/queue/queue.go index f77238a99a..209166e988 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -75,23 +75,31 @@ const ( // TaskDeadlineDefault bounds how long a task is allowed to run before being forcefully // reset if a runner for the task type is not registered. TaskDeadlineDefault = time.Minute + + // RunnerWatchdogGracePeriodDefault is the extra time beyond the runner timeout that the + // watchdog waits before reporting a runner as blocked. The runner context is still canceled + // at the runner timeout; the grace period only gives a cooperative runner time to observe + // that cancellation and return before the watchdog reports it as non-cooperative. + RunnerWatchdogGracePeriodDefault = 5 * time.Second ) type Config struct { - Workers int - Delay time.Duration - DelayInitial time.Duration - DelayUnstick time.Duration - StopWaitTimeout time.Duration + Workers int + Delay time.Duration + DelayInitial time.Duration + DelayUnstick time.Duration + StopWaitTimeout time.Duration + RunnerWatchdogGracePeriod time.Duration } func NewConfig() *Config { return &Config{ - Workers: WorkersDefault, - Delay: DelayDefault, - DelayInitial: DelayInitialDefault, - DelayUnstick: DelayUnstickDefault, - StopWaitTimeout: StopWaitTimeoutDefault, + Workers: WorkersDefault, + Delay: DelayDefault, + DelayInitial: DelayInitialDefault, + DelayUnstick: DelayUnstickDefault, + StopWaitTimeout: StopWaitTimeoutDefault, + RunnerWatchdogGracePeriod: RunnerWatchdogGracePeriodDefault, } } @@ -135,6 +143,13 @@ func (c *Config) Load(configReporter config.Reporter) error { c.StopWaitTimeout = time.Duration(stopWaitTimeout) * time.Second } } + if runnerWatchdogGracePeriodString, err := configReporter.Get("runner_watchdog_grace_period"); err == nil { + if runnerWatchdogGracePeriod, parseErr := strconv.ParseInt(runnerWatchdogGracePeriodString, 10, 0); parseErr != nil { + return errors.New("runner watchdog grace period is invalid") + } else { + c.RunnerWatchdogGracePeriod = time.Duration(runnerWatchdogGracePeriod) * time.Second + } + } return nil } @@ -155,6 +170,9 @@ func (c *Config) Validate() error { if c.StopWaitTimeout <= 0 { return errors.New("stop wait timeout is invalid") } + if c.RunnerWatchdogGracePeriod <= 0 { + return errors.New("runner watchdog grace period is invalid") + } return nil } @@ -401,8 +419,10 @@ func (q *queue) runTask(ctx context.Context, tsk *task.Task) { // runner blows past its timeout without returning, this worker stays blocked until the // process restarts (the task itself is recovered by the deadline/unstick mechanism). We // cannot unblock the worker, but we surface the condition via a log and metric so a - // non-cooperative runner is detectable rather than silent. - runnerWatchdog := time.AfterFunc(runner.GetRunnerTimeout(), func() { + // non-cooperative runner is detectable rather than silent. The grace period keeps a + // cooperative runner that returns promptly after the timeout cancellation from being + // falsely reported. + runnerWatchdog := time.AfterFunc(runner.GetRunnerTimeout()+q.config.RunnerWatchdogGracePeriod, func() { lgr.Error("Task runner exceeded timeout without returning; worker is blocked until it returns") RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType()).Inc() }) diff --git a/task/queue/queue_test.go b/task/queue/queue_test.go index fe1ac485d5..1acb9e38c0 100644 --- a/task/queue/queue_test.go +++ b/task/queue/queue_test.go @@ -48,6 +48,10 @@ var _ = Describe("Queue", func() { Expect(taskQueue.StopWaitTimeoutDefault).To(Equal(10 * time.Second)) }) + It("RunnerWatchdogGracePeriodDefault is expected", func() { + Expect(taskQueue.RunnerWatchdogGracePeriodDefault).To(Equal(5 * time.Second)) + }) + It("DurationJitterFactor is expected", func() { Expect(taskQueue.DurationJitterFactor).To(Equal(0.2)) }) @@ -71,6 +75,7 @@ var _ = Describe("Queue", func() { Expect(cfg.DelayInitial).To(Equal(taskQueue.DelayInitialDefault)) Expect(cfg.DelayUnstick).To(Equal(taskQueue.DelayUnstickDefault)) Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) + Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(taskQueue.RunnerWatchdogGracePeriodDefault)) }) }) @@ -117,6 +122,11 @@ var _ = Describe("Queue", func() { Expect(cfg.Load(configReporter)).To(MatchError("stop wait timeout is invalid")) }) + It("returns an error when runner watchdog grace period is not parsable", func() { + configReporter.Config["runner_watchdog_grace_period"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("runner watchdog grace period is invalid")) + }) + It("uses existing workers if not set", func() { Expect(cfg.Load(configReporter)).To(Succeed()) Expect(cfg.Workers).To(Equal(taskQueue.WorkersDefault)) @@ -142,18 +152,25 @@ var _ = Describe("Queue", func() { Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) }) + It("uses existing runner watchdog grace period if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(taskQueue.RunnerWatchdogGracePeriodDefault)) + }) + It("returns successfully and uses values from the config reporter", func() { configReporter.Config["workers"] = "5" configReporter.Config["delay"] = "30" configReporter.Config["delay_initial"] = "45" configReporter.Config["delay_unstick"] = "60" configReporter.Config["stop_wait_timeout"] = "15" + configReporter.Config["runner_watchdog_grace_period"] = "20" Expect(cfg.Load(configReporter)).To(Succeed()) Expect(cfg.Workers).To(Equal(5)) Expect(cfg.Delay).To(Equal(30 * time.Second)) Expect(cfg.DelayInitial).To(Equal(45 * time.Second)) Expect(cfg.DelayUnstick).To(Equal(60 * time.Second)) Expect(cfg.StopWaitTimeout).To(Equal(15 * time.Second)) + Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(20 * time.Second)) }) }) @@ -183,6 +200,11 @@ var _ = Describe("Queue", func() { Expect(cfg.Validate()).To(MatchError("stop wait timeout is invalid")) }) + It("returns an error when runner watchdog grace period is invalid", func() { + cfg.RunnerWatchdogGracePeriod = 0 + Expect(cfg.Validate()).To(MatchError("runner watchdog grace period is invalid")) + }) + It("returns successfully", func() { Expect(cfg.Validate()).To(Succeed()) }) @@ -348,7 +370,7 @@ var _ = Describe("Queue", func() { countingRunner = taskQueueTest.NewCountingRunner(taskTest.RandomType()) panicRunner = taskQueueTest.NewPanicRunner(taskTest.RandomType()) - cfg = &taskQueue.Config{Workers: 2, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault} + cfg = &taskQueue.Config{Workers: 2, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault} que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, countingRunner, panicRunner)) }) @@ -526,15 +548,15 @@ var _ = Describe("Queue", func() { ID: task.NewID(), Type: countingRunner.GetRunnerType(), State: task.TaskStateRunning, - DeadlineTime: pointer.FromTime(time.Now().Add(-time.Minute)), StateLock: pointer.FromString(taskTest.RandomType()), CreatedTime: time.Now(), Revision: 1, + DeadlineTime: pointer.FromTime(time.Now().Add(-time.Minute)), } _, err := str.GetCollection("tasks").InsertOne(ctx, stuckTask) Expect(err).ToNot(HaveOccurred()) - unstickConfig := &taskQueue.Config{Workers: 2, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: time.Millisecond, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault} + unstickConfig := &taskQueue.Config{Workers: 2, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: time.Millisecond, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault} que = test.Must(taskQueue.New(taskTest.RandomType(), unstickConfig, lgr, str, countingRunner)) que.Start() @@ -677,7 +699,7 @@ var _ = Describe("Queue", func() { BeforeEach(func() { blockingRunner = taskQueueTest.NewBlockingRunner(taskTest.RandomType(), time.Minute) - cfg := &taskQueue.Config{Workers: 1, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: 250 * time.Millisecond} + cfg := &taskQueue.Config{Workers: 1, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: 250 * time.Millisecond, RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault} que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, blockingRunner)) }) @@ -710,11 +732,12 @@ var _ = Describe("Queue", func() { var que taskQueue.Queue BeforeEach(func() { - // A short duration maximum yields a short runner timeout (3x), so the watchdog fires - // quickly while the runner is still blocked, without an unstick reclaiming the task. + // A short duration maximum yields a short runner timeout (3x), and a short grace + // period keeps the watchdog prompt, so the watchdog fires quickly while the runner + // is still blocked, without an unstick reclaiming the task. blockingRunner = taskQueueTest.NewBlockingRunner(taskTest.RandomType(), 20*time.Millisecond) - cfg := &taskQueue.Config{Workers: 1, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: 250 * time.Millisecond} + cfg := &taskQueue.Config{Workers: 1, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: 250 * time.Millisecond, RunnerWatchdogGracePeriod: 50 * time.Millisecond} que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, blockingRunner)) }) @@ -759,7 +782,7 @@ var _ = Describe("Queue", func() { BeforeEach(func() { runner = taskQueueTest.NewRepeatRunner(taskTest.RandomType()) - cfg := &taskQueue.Config{Workers: workersCount, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: time.Millisecond, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault} + cfg := &taskQueue.Config{Workers: workersCount, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: time.Millisecond, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault} ques = make([]taskQueue.Queue, queueCount) for index := range len(ques) { ques[index] = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) diff --git a/task/store/mongo/mongo.go b/task/store/mongo/mongo.go index 15f1f4e8e7..29e5777e8c 100644 --- a/task/store/mongo/mongo.go +++ b/task/store/mongo/mongo.go @@ -114,7 +114,7 @@ type TaskRepository struct { } func (t *TaskRepository) EnsureIndexes() error { - // Repositories operation only a subset of the tasks shouldn't invoke this method + // Repositories operating on a subset of the tasks shouldn't invoke this method if t.typeFilter != nil { return errors.New("calling EnsureIndexes() on a partitioned repository is not allowed") } diff --git a/task/store/mongo/mongo_test.go b/task/store/mongo/mongo_test.go index 2b31cdaa5b..aaca5973b8 100644 --- a/task/store/mongo/mongo_test.go +++ b/task/store/mongo/mongo_test.go @@ -160,9 +160,9 @@ var _ = Describe("Mongo", func() { actualStuckTask := &task.Task{} Expect(collection.FindOne(ctx, bson.M{"id": stuckTask.ID}).Decode(actualStuckTask)).To(Succeed()) Expect(actualStuckTask.State).To(Equal(task.TaskStatePending)) - Expect(actualStuckTask.DeadlineTime).To(BeNil()) Expect(actualStuckTask.AvailableTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) Expect(actualStuckTask.ModifiedTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) + Expect(actualStuckTask.DeadlineTime).To(BeNil()) }) It("does not unstick a running task with a deadline in the future", func() { diff --git a/task/store/test/task_session.go b/task/store/test/task_session.go deleted file mode 100644 index 8cb7a8794a..0000000000 --- a/task/store/test/task_session.go +++ /dev/null @@ -1,17 +0,0 @@ -package test - -import "github.com/tidepool-org/platform/test" - -type TasksSession struct { - *test.Closer -} - -func NewTasksSession() *TasksSession { - return &TasksSession{ - Closer: test.NewCloser(), - } -} - -func (t *TasksSession) AssertOutputsEmpty() { - t.Closer.AssertOutputsEmpty() -} diff --git a/task/task.go b/task/task.go index f3e97869fa..6410b13a8c 100644 --- a/task/task.go +++ b/task/task.go @@ -178,7 +178,6 @@ type Task struct { Type string `json:"type" bson:"type"` Data map[string]any `json:"data,omitempty" bson:"data,omitempty"` AvailableTime *time.Time `json:"availableTime,omitempty" bson:"availableTime,omitempty"` - DeadlineTime *time.Time `json:"deadlineTime,omitempty" bson:"deadlineTime,omitempty"` State string `json:"state" bson:"state"` Error *errors.Serializable `json:"error,omitempty" bson:"error,omitempty"` RunTime *time.Time `json:"runTime,omitempty" bson:"runTime,omitempty"` @@ -190,7 +189,8 @@ type Task struct { // Database only // Use to enforce only one state transition at a time. This is a unique value that changes on every update. - StateLock *string `json:"-" bson:"stateLock,omitempty"` + StateLock *string `json:"-" bson:"stateLock,omitempty"` + DeadlineTime *time.Time `json:"-" bson:"deadlineTime,omitempty"` } func NewTask(ctx context.Context, create *TaskCreate) (*Task, error) { @@ -280,7 +280,6 @@ func (t *Task) Sanitize(details request.AuthDetails) error { func (t *Task) RepeatAvailableAt(availableTime time.Time) { t.State = TaskStatePending t.AvailableTime = pointer.FromTime(availableTime) - t.DeadlineTime = nil } func (t *Task) RepeatAvailableAfter(availableDuration time.Duration) { diff --git a/task/test/client.go b/task/test/client.go index 03cf75a221..d458cf353a 100644 --- a/task/test/client.go +++ b/task/test/client.go @@ -147,4 +147,5 @@ func (t *Client) Expectations() { gomega.Expect(t.CreateTaskOutputs).To(gomega.BeEmpty()) gomega.Expect(t.GetTaskOutputs).To(gomega.BeEmpty()) gomega.Expect(t.UpdateTaskOutputs).To(gomega.BeEmpty()) + gomega.Expect(t.DeleteTaskOutputs).To(gomega.BeEmpty()) } diff --git a/task/test/task.go b/task/test/task.go index 3bce102f0f..f0b11b88b4 100644 --- a/task/test/task.go +++ b/task/test/task.go @@ -56,36 +56,36 @@ func RandomTask(options ...test.Option) *task.Task { switch tsk.State { case task.TaskStatePending: tsk.AvailableTime = pointer.From(test.RandomTimeAfterNow()) - tsk.DeadlineTime = nil tsk.ModifiedTime = test.RandomOptional(func() time.Time { return test.RandomTimeFromRange(tsk.CreatedTime, now) }, options...) if tsk.ModifiedTime != nil { tsk.Error = test.RandomOptionalPointer(errorsTest.RandomSerializable, options...) tsk.RunTime = tsk.ModifiedTime tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) } + tsk.DeadlineTime = nil case task.TaskStateRunning: tsk.AvailableTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, now)) - tsk.DeadlineTime = pointer.From(test.RandomTimeAfterNow()) tsk.ModifiedTime = pointer.From(test.RandomTimeFromRange(*tsk.AvailableTime, now)) if test.RandomBool() { tsk.Error = test.RandomOptionalPointer(errorsTest.RandomSerializable, options...) tsk.RunTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, *tsk.AvailableTime)) tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) } + tsk.DeadlineTime = pointer.From(test.RandomTimeAfterNow()) case task.TaskStateFailed: tsk.AvailableTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, now)) - tsk.DeadlineTime = nil tsk.ModifiedTime = pointer.From(test.RandomTimeFromRange(*tsk.AvailableTime, now)) tsk.Error = errorsTest.RandomSerializable() tsk.RunTime = tsk.ModifiedTime tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + tsk.DeadlineTime = nil case task.TaskStateCompleted: tsk.AvailableTime = pointer.From(test.RandomTimeFromRange(tsk.CreatedTime, now)) - tsk.DeadlineTime = nil tsk.ModifiedTime = pointer.From(test.RandomTimeFromRange(*tsk.AvailableTime, now)) tsk.Error = test.RandomOptionalPointer(errorsTest.RandomSerializable, options...) tsk.RunTime = tsk.ModifiedTime tsk.Duration = pointer.From(test.RandomFloat64FromRange(0, 10)) + tsk.DeadlineTime = nil } return tsk From 9d0e3b15d502e07a550e86f2c3b8d238145cba78 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Mon, 20 Jul 2026 14:23:25 -0700 Subject: [PATCH 04/20] Minor updates for Dexcom code --- dexcom/alert.go | 2 +- dexcom/calibration.go | 2 +- dexcom/data_range.go | 4 ++-- dexcom/data_range_test.go | 33 +++++++++++++++++++++++++++++++++ dexcom/egv.go | 2 +- dexcom/event.go | 2 +- dexcom/moment.go | 4 ++-- dexcom/moment_test.go | 14 +++++++++----- 8 files changed, 50 insertions(+), 13 deletions(-) diff --git a/dexcom/alert.go b/dexcom/alert.go index 4cfa6ff4b9..5295d42734 100644 --- a/dexcom/alert.go +++ b/dexcom/alert.go @@ -174,7 +174,7 @@ func (a *Alert) Validate(validator structure.Validator) { logger.Warnf("AlertState is '%s'", *a.AlertState) } if a.TransmitterID != nil && *a.TransmitterID == "" { - logger.Warnf("TransmitterID is empty", *a.TransmitterID) + logger.Warn("TransmitterID is empty") } } diff --git a/dexcom/calibration.go b/dexcom/calibration.go index b2efaee055..a5cc1ecb01 100644 --- a/dexcom/calibration.go +++ b/dexcom/calibration.go @@ -161,7 +161,7 @@ func (c *Calibration) Validate(validator structure.Validator) { logger.Warnf("Unit is '%s'", *c.Unit) } if c.TransmitterID != nil && *c.TransmitterID == "" { - logger.Warnf("TransmitterID is empty", *c.TransmitterID) + logger.Warn("TransmitterID is empty") } if c.DisplayDevice != nil && *c.DisplayDevice == DeviceDisplayDeviceUnknown { logger.Warnf("DisplayDevice is '%s'", *c.DisplayDevice) diff --git a/dexcom/data_range.go b/dexcom/data_range.go index 3064e53144..87c119c6bf 100644 --- a/dexcom/data_range.go +++ b/dexcom/data_range.go @@ -79,8 +79,8 @@ func (d *DataRangesResponse) DataRange() *DataRange { endMoments = append(endMoments, d.Events.End) } - startMoments = startMoments.Compact() - endMoments = endMoments.Compact() + startMoments = startMoments.CompactBySystemTimeRaw() + endMoments = endMoments.CompactBySystemTimeRaw() if len(startMoments) == 0 || len(endMoments) == 0 { return nil } diff --git a/dexcom/data_range_test.go b/dexcom/data_range_test.go index d033ecc4a8..5d2e013c24 100644 --- a/dexcom/data_range_test.go +++ b/dexcom/data_range_test.go @@ -242,6 +242,39 @@ var _ = Describe("DataRange", func() { } Expect(datum.DataRange()).To(Equal(expectedDataRange)) }) + + It("returns data range excluding moments without a system time", func() { + datum := dexcomTest.RandomDataRangesResponse() + datum.Calibrations.Start = dexcomTest.RandomMomentFromTime(time.Unix(1730000000, 0)) + datum.Calibrations.End = dexcomTest.RandomMomentFromTime(time.Unix(1790000000, 0)) + datum.EGVs.Start = dexcomTest.RandomMomentFromTime(time.Unix(1710000000, 0)) + datum.EGVs.End = dexcomTest.RandomMomentFromTime(time.Unix(1780000000, 0)) + datum.Events.Start = dexcomTest.RandomMomentFromTime(time.Unix(1750000000, 0)) + datum.Events.End = dexcomTest.RandomMomentFromTime(time.Unix(1770000000, 0)) + datum.EGVs.Start.SystemTime = nil + datum.Calibrations.End.SystemTime = nil + expectedDataRange := &dexcom.DataRange{ + Start: datum.Calibrations.Start, + End: datum.EGVs.End, + } + Expect(datum.DataRange()).To(Equal(expectedDataRange)) + }) + + It("returns nil if no start moments have a system time", func() { + datum := dexcomTest.RandomDataRangesResponse() + datum.Calibrations.Start.SystemTime = nil + datum.EGVs.Start.SystemTime = nil + datum.Events.Start.SystemTime = nil + Expect(datum.DataRange()).To(BeNil()) + }) + + It("returns nil if no end moments have a system time", func() { + datum := dexcomTest.RandomDataRangesResponse() + datum.Calibrations.End.SystemTime = nil + datum.EGVs.End.SystemTime = nil + datum.Events.End.SystemTime = nil + Expect(datum.DataRange()).To(BeNil()) + }) }) }) diff --git a/dexcom/egv.go b/dexcom/egv.go index a9a32560eb..ae14a7d423 100644 --- a/dexcom/egv.go +++ b/dexcom/egv.go @@ -241,7 +241,7 @@ func (e *EGV) Validate(validator structure.Validator) { logger.Warnf("Trend is '%s'", *e.Trend) } if e.TransmitterID != nil && *e.TransmitterID == "" { - logger.Warnf("TransmitterID is empty", *e.TransmitterID) + logger.Warn("TransmitterID is empty") } if e.TransmitterTicks != nil && *e.TransmitterTicks == EGVTransmitterTickMinimum { logger.Warnf("TransmitterTicks is %d", *e.TransmitterTicks) diff --git a/dexcom/event.go b/dexcom/event.go index 6747bb0529..5aca4e92b4 100644 --- a/dexcom/event.go +++ b/dexcom/event.go @@ -287,7 +287,7 @@ func (e *Event) Validate(validator structure.Validator) { logger.Warnf("EventType is '%s'", *e.EventType) } if e.TransmitterID != nil && *e.TransmitterID == "" { - logger.Warnf("TransmitterID is empty", *e.TransmitterID) + logger.Warn("TransmitterID is empty") } if e.DisplayDevice != nil && *e.DisplayDevice == DeviceDisplayDeviceUnknown { logger.Warnf("DisplayDevice is '%s'", *e.DisplayDevice) diff --git a/dexcom/moment.go b/dexcom/moment.go index f45b53ee85..e1cb1dc0b3 100644 --- a/dexcom/moment.go +++ b/dexcom/moment.go @@ -50,10 +50,10 @@ func (m *Moment) SystemTimeRaw() *time.Time { type Moments []*Moment -func (m Moments) Compact() Moments { +func (m Moments) CompactBySystemTimeRaw() Moments { var moments Moments for _, moment := range m { - if moment != nil { + if systemTimeRaw(moment) != nil { moments = append(moments, moment) } } diff --git a/dexcom/moment_test.go b/dexcom/moment_test.go index 2b7a222a96..1c3f0fbee5 100644 --- a/dexcom/moment_test.go +++ b/dexcom/moment_test.go @@ -187,13 +187,17 @@ var _ = Describe("Moment", func() { }) }) - Context("Moment", func() { - Context("Compact", func() { - It("removes nil moments from array", func() { + Context("Moments", func() { + Context("CompactBySystemTimeRaw", func() { + It("removes nil moments and moments without a system time from array", func() { moment1 := dexcomTest.RandomMomentFromRange(test.PastFarTime(), test.FutureFarTime()) moment2 := dexcomTest.RandomMomentFromRange(test.PastFarTime(), test.FutureFarTime()) - moments := dexcom.Moments{nil, moment1, nil, moment2, nil} - Expect(moments.Compact()).To(Equal(dexcom.Moments{moment1, moment2})) + moment3 := dexcomTest.RandomMomentFromRange(test.PastFarTime(), test.FutureFarTime()) + moment3.SystemTime = nil + moment4 := dexcomTest.RandomMomentFromRange(test.PastFarTime(), test.FutureFarTime()) + moment4.SystemTime.Time = time.Time{} + moments := dexcom.Moments{nil, moment1, moment3, moment2, moment4, nil} + Expect(moments.CompactBySystemTimeRaw()).To(Equal(dexcom.Moments{moment1, moment2})) }) }) }) From 50dda41d65ee610bea37fdf596f740064ce08c9a Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Mon, 20 Jul 2026 15:26:39 -0700 Subject: [PATCH 05/20] Clear task available time appropriately - Clear task available time appropriately - Add and update tests --- task/queue/queue.go | 1 + task/queue/queue_internal_test.go | 83 +++++++++++++++++++++++++++++++ task/store/mongo/mongo.go | 3 +- task/store/mongo/mongo_test.go | 5 ++ 4 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 task/queue/queue_internal_test.go diff --git a/task/queue/queue.go b/task/queue/queue.go index 209166e988..1ce042e42c 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -662,6 +662,7 @@ func (q *queue) computeState(ctx context.Context, tsk *task.Task) { } tsk.SetFailed() case task.TaskStateFailed, task.TaskStateCompleted: + tsk.AvailableTime = nil default: tsk.AppendError(errors.New("unknown task state")) tsk.SetFailed() diff --git a/task/queue/queue_internal_test.go b/task/queue/queue_internal_test.go new file mode 100644 index 0000000000..e59ebadffe --- /dev/null +++ b/task/queue/queue_internal_test.go @@ -0,0 +1,83 @@ +package queue + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "go.mongodb.org/mongo-driver/mongo" + + "github.com/tidepool-org/platform/log" + logTest "github.com/tidepool-org/platform/log/test" + "github.com/tidepool-org/platform/pointer" + "github.com/tidepool-org/platform/task" + taskStore "github.com/tidepool-org/platform/task/store" + taskTest "github.com/tidepool-org/platform/task/test" + "github.com/tidepool-org/platform/test" +) + +var _ = Describe("Queue", func() { + Context("dispatchTasks", func() { + It("logs an error when the pending iterator cannot be opened", func() { + lgr := logTest.NewLogger() + ctx := log.NewContextWithLogger(context.Background(), lgr) + cfg := NewConfig() + que := &queue{ + name: taskTest.RandomType(), + config: cfg, + logger: lgr, + repository: &failureOpeningIteratorRepository{}, + workersAvailable: 1, + } + que.dispatchTasks(ctx) + lgr.AssertError("Unable to open task iterator") + }) + + It("logs an error when the pending iterator fails mid-iteration", func() { + lgr := logTest.NewLogger() + ctx := log.NewContextWithLogger(context.Background(), lgr) + cfg := NewConfig() + que := &queue{ + name: taskTest.RandomType(), + config: cfg, + logger: lgr, + repository: &failureIteratingIteratorRepository{}, + workersAvailable: 1, + } + que.dispatchTasks(ctx) + lgr.AssertError("Unable to iterate tasks") + }) + }) + + Context("computeState", func() { + It("clears the available time for a completed task", func() { + completedTask := &task.Task{State: task.TaskStateCompleted, AvailableTime: pointer.FromTime(test.RandomTimeBeforeNow())} + (&queue{}).computeState(context.Background(), completedTask) + Expect(completedTask.AvailableTime).To(BeNil()) + }) + + It("clears the available time for a failed task", func() { + failedTask := &task.Task{State: task.TaskStateFailed, AvailableTime: pointer.FromTime(test.RandomTimeBeforeNow())} + (&queue{}).computeState(context.Background(), failedTask) + Expect(failedTask.AvailableTime).To(BeNil()) + }) + }) +}) + +type failureOpeningIteratorRepository struct { + taskStore.TaskRepository +} + +func (f *failureOpeningIteratorRepository) IteratePending(ctx context.Context) (*mongo.Cursor, error) { + return nil, fmt.Errorf("failure opening iterator") +} + +type failureIteratingIteratorRepository struct { + taskStore.TaskRepository +} + +func (f *failureIteratingIteratorRepository) IteratePending(ctx context.Context) (*mongo.Cursor, error) { + return mongo.NewCursorFromDocuments([]any{}, fmt.Errorf("failure iterating iterator"), nil) +} diff --git a/task/store/mongo/mongo.go b/task/store/mongo/mongo.go index 29e5777e8c..1edf9464c4 100644 --- a/task/store/mongo/mongo.go +++ b/task/store/mongo/mongo.go @@ -396,7 +396,8 @@ func (t *TaskRepository) StartTask(ctx context.Context, id string, revision int, "stateLock": newStateLock(), } unset := bson.M{ - "duration": 1, + "availableTime": 1, + "duration": 1, } selector := t.selector(id, storeStructured.NewConditionWithRevision(&revision)) diff --git a/task/store/mongo/mongo_test.go b/task/store/mongo/mongo_test.go index aaca5973b8..600f8854ca 100644 --- a/task/store/mongo/mongo_test.go +++ b/task/store/mongo/mongo_test.go @@ -214,10 +214,12 @@ var _ = Describe("Mongo", func() { Context("StopTask", func() { It("clears the run time and duration when stopping without a duration", func() { pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + Expect(pendingTask.AvailableTime).ToNot(BeNil()) startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) Expect(startedTask).ToNot(BeNil()) Expect(startedTask.RunTime).ToNot(BeNil()) + Expect(startedTask.AvailableTime).To(BeNil()) Expect(repository.StopTask(ctx, startedTask.ID, startedTask.StateLock, task.TaskStatePending, nil, nil)).To(Succeed()) @@ -228,10 +230,12 @@ var _ = Describe("Mongo", func() { Expect(actualTask.Duration).To(BeNil()) Expect(actualTask.StateLock).To(BeNil()) Expect(actualTask.DeadlineTime).To(BeNil()) + Expect(actualTask.AvailableTime).To(BeNil()) }) It("retains the run time when stopping with a duration", func() { pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + Expect(pendingTask.AvailableTime).ToNot(BeNil()) startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) Expect(startedTask).ToNot(BeNil()) @@ -247,6 +251,7 @@ var _ = Describe("Mongo", func() { Expect(actualTask.Duration).To(PointTo(Equal(duration.Seconds()))) Expect(actualTask.StateLock).To(BeNil()) Expect(actualTask.DeadlineTime).To(BeNil()) + Expect(actualTask.AvailableTime).To(BeNil()) }) }) From d6f346b498fa32a63949050c2e30ea96c5d59911 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Mon, 20 Jul 2026 15:50:54 -0700 Subject: [PATCH 06/20] Add metrics routes --- auth/service/api/v1/metrics.go | 21 +++++++++++++++++++++ auth/service/api/v1/router.go | 1 + data/service/api/v1/metrics.go | 24 ++++++++++++++++++++++++ data/service/api/v1/v1.go | 1 + 4 files changed, 47 insertions(+) create mode 100644 auth/service/api/v1/metrics.go create mode 100644 data/service/api/v1/metrics.go diff --git a/auth/service/api/v1/metrics.go b/auth/service/api/v1/metrics.go new file mode 100644 index 0000000000..4f8f9382ee --- /dev/null +++ b/auth/service/api/v1/metrics.go @@ -0,0 +1,21 @@ +package v1 + +import ( + "net/http" + + "github.com/ant0ine/go-json-rest/rest" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func (r *Router) MetricsRoutes() []*rest.Route { + return []*rest.Route{ + rest.Get("/v1/metrics", r.PrometheusMetrics), + } +} + +func (r *Router) PrometheusMetrics(res rest.ResponseWriter, req *rest.Request) { + // The default go-json-rest middleware gzips the content + promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{DisableCompression: true}). + ServeHTTP(res.(http.ResponseWriter), req.Request) +} diff --git a/auth/service/api/v1/router.go b/auth/service/api/v1/router.go index e62e7b2e40..e19e3c4d09 100644 --- a/auth/service/api/v1/router.go +++ b/auth/service/api/v1/router.go @@ -23,6 +23,7 @@ func NewRouter(svc service.Service) (*Router, error) { func (r *Router) Routes() []*rest.Route { routes := [][]*rest.Route{ + r.MetricsRoutes(), r.OAuthRoutes(), r.ProviderSessionsRoutes(), r.RestrictedTokensRoutes(), diff --git a/data/service/api/v1/metrics.go b/data/service/api/v1/metrics.go new file mode 100644 index 0000000000..48f99df33e --- /dev/null +++ b/data/service/api/v1/metrics.go @@ -0,0 +1,24 @@ +package v1 + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + dataService "github.com/tidepool-org/platform/data/service" +) + +func MetricsRoutes() []dataService.Route { + return []dataService.Route{ + dataService.Get("/v1/metrics", PrometheusMetrics), + } +} +func PrometheusMetrics(dataServiceContext dataService.Context) { + res := dataServiceContext.Response() + req := dataServiceContext.Request() + + // The default go-json-rest middleware gzips the content + promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{DisableCompression: true}). + ServeHTTP(res.(http.ResponseWriter), req.Request) +} diff --git a/data/service/api/v1/v1.go b/data/service/api/v1/v1.go index 3707db8104..387e43b7a6 100644 --- a/data/service/api/v1/v1.go +++ b/data/service/api/v1/v1.go @@ -30,6 +30,7 @@ func Routes() []service.Route { service.Post("/v1/partners/twiist/data/:tidepoolLinkId", NewTwiistDataCreateHandler(DataSetsDataCreate), api.RequireAuth), } + routes = append(routes, MetricsRoutes()...) routes = append(routes, DataSetsRoutes()...) routes = append(routes, SourcesRoutes()...) routes = append(routes, SummaryRoutes()...) From 8b901377670cc49002901b74b20b6417a824a308 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Mon, 20 Jul 2026 18:18:03 -0700 Subject: [PATCH 07/20] Address PR comments --- data/service/api/v1/metrics.go | 1 + dexcom/provider/provider.go | 3 --- dexcom/provider/provider_test.go | 2 +- oura/provider/provider.go | 3 --- pointer/default_test.go | 4 ++-- private/plugin/abbott | 2 +- twiist/provider/provider.go | 3 --- 7 files changed, 5 insertions(+), 13 deletions(-) diff --git a/data/service/api/v1/metrics.go b/data/service/api/v1/metrics.go index 48f99df33e..13328e14d7 100644 --- a/data/service/api/v1/metrics.go +++ b/data/service/api/v1/metrics.go @@ -14,6 +14,7 @@ func MetricsRoutes() []dataService.Route { dataService.Get("/v1/metrics", PrometheusMetrics), } } + func PrometheusMetrics(dataServiceContext dataService.Context) { res := dataServiceContext.Response() req := dataServiceContext.Request() diff --git a/dexcom/provider/provider.go b/dexcom/provider/provider.go index edc8bdb16e..e4b98d4716 100644 --- a/dexcom/provider/provider.go +++ b/dexcom/provider/provider.go @@ -48,9 +48,6 @@ func New(configReporter config.Reporter, dataSourceClient dataSource.Client, tas return nil, errors.Wrap(err, "unable to create provider config") } - // Attach prometheus round tripper to default client transport - prometheusRequestMetricsRoundTripper.WithRoundTripper(http.DefaultClient.Transport) - // Create http client httpClient := &http.Client{ Transport: prometheusRequestMetricsRoundTripper, diff --git a/dexcom/provider/provider_test.go b/dexcom/provider/provider_test.go index 4a72702be5..93744dd3eb 100644 --- a/dexcom/provider/provider_test.go +++ b/dexcom/provider/provider_test.go @@ -17,7 +17,7 @@ import ( ) var _ = Describe("Provider", func() { - It("PathPatternAny is expected", func() { + It("RequestTimeHeaderName is expected", func() { Expect(dexcomProvider.RequestTimeHeaderName).To(Equal("request-time")) }) diff --git a/oura/provider/provider.go b/oura/provider/provider.go index aeb352f39d..2e1920a97d 100644 --- a/oura/provider/provider.go +++ b/oura/provider/provider.go @@ -67,9 +67,6 @@ func New(dependencies Dependencies) (*Provider, error) { return nil, errors.Wrap(err, "dependencies is invalid") } - // Attach prometheus round tripper to default client transport - prometheusRequestMetricsRoundTripper.WithRoundTripper(http.DefaultClient.Transport) - // Create http client httpClient := &http.Client{ Transport: prometheusRequestMetricsRoundTripper, diff --git a/pointer/default_test.go b/pointer/default_test.go index daf51348d0..4fdf874f8c 100644 --- a/pointer/default_test.go +++ b/pointer/default_test.go @@ -26,13 +26,13 @@ var _ = Describe("Default", func() { }) Context("DefaultArray", func() { - It("returns the default value if the pointer to the value is nil", func() { + It("returns the default value if the value is nil", func() { defaultValue := test.RandomStringArray() result := pointer.DefaultArray(nil, defaultValue) Expect(result).To(Equal(defaultValue)) }) - It("returns the dereferenced pointer to the value if the pointer to the value is not nil", func() { + It("returns the value if it is not nil", func() { value := test.RandomStringArray() defaultValue := test.RandomStringArray() result := pointer.DefaultArray(value, defaultValue) diff --git a/private/plugin/abbott b/private/plugin/abbott index c0fcb7b1d9..cd4a03ffc7 160000 --- a/private/plugin/abbott +++ b/private/plugin/abbott @@ -1 +1 @@ -Subproject commit c0fcb7b1d9f5f8468af91511e64f56628f49c110 +Subproject commit cd4a03ffc7812e92d6c7c9a2b2e54aa220ce1371 diff --git a/twiist/provider/provider.go b/twiist/provider/provider.go index 19d53f509e..477b58307e 100644 --- a/twiist/provider/provider.go +++ b/twiist/provider/provider.go @@ -67,9 +67,6 @@ func New(providerDependencies ProviderDependencies) (*Provider, error) { return nil, errors.Wrap(err, "unable to create provider config") } - // Attach prometheus round tripper to default client transport - prometheusRequestMetricsRoundTripper.WithRoundTripper(http.DefaultClient.Transport) - // Create http client httpClient := &http.Client{ Transport: prometheusRequestMetricsRoundTripper, From 56e3ea0cb5cc753ea9c5299b3490e0df5ed5db2f Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Tue, 21 Jul 2026 14:43:16 -0700 Subject: [PATCH 08/20] Correctly wire up round tripper in OAuth clients - Correctly wire up round tripper in OAuth clients - Remove unnecessary and outdated mocks - Update tests --- client/client.go | 18 +- client/config.go | 15 + client/prometheus.go | 2 +- dexcom/client/client.go | 65 ++++- dexcom/client/client_test.go | 467 ++++++++++++++++-------------- dexcom/fetch/runner.go | 18 +- dexcom/fetch/runner_test.go | 70 ++--- dexcom/provider/provider.go | 58 +--- dexcom/provider/provider_test.go | 113 -------- ehr/sync/runner.go | 3 +- errors/errors.go | 61 ++-- errors/errors_test.go | 5 +- oauth/client/client.go | 23 +- oauth/client/client_test.go | 106 +++---- oauth/provider/client/client.go | 4 +- oauth/provider/provider.go | 10 +- oauth/test/token_source.go | 113 -------- oauth/test/token_source_source.go | 54 ---- oura/client/client_test.go | 2 +- oura/provider/provider.go | 7 +- private/plugin/abbott | 2 +- summary/task/migrationrunner.go | 6 +- summary/task/updaterunner.go | 6 +- task/queue/multi.go | 8 +- task/queue/queue.go | 236 +++++++-------- task/queue/queue_internal_test.go | 8 +- task/queue/queue_test.go | 40 ++- task/queue/runner.go | 104 +++++++ task/service/service/service.go | 2 +- task/store/mongo/mongo.go | 143 +++++---- task/store/mongo/mongo_test.go | 30 +- task/task.go | 5 + twiist/provider/provider.go | 14 +- 33 files changed, 888 insertions(+), 930 deletions(-) delete mode 100644 dexcom/provider/provider_test.go delete mode 100644 oauth/test/token_source.go delete mode 100644 oauth/test/token_source_source.go create mode 100644 task/queue/runner.go diff --git a/client/client.go b/client/client.go index 635495844e..208fae5e38 100644 --- a/client/client.go +++ b/client/client.go @@ -31,8 +31,7 @@ type ErrorResponseParser interface { } type Client struct { - address string - userAgent string + config Config errorResponseParser ErrorResponseParser } @@ -48,14 +47,13 @@ func NewWithErrorParser(cfg *Config, errorResponseParser ErrorResponseParser) (* } return &Client{ - address: cfg.Address, - userAgent: cfg.UserAgent, + config: *cfg, errorResponseParser: errorResponseParser, }, nil } func (c *Client) ConstructURL(paths ...string) string { - return ConstructURL(c.address, paths...) + return ConstructURL(c.config.Address, paths...) } func (c *Client) AppendURLQuery(urlString string, query map[string]string) string { @@ -87,6 +85,12 @@ func (c *Client) RequestStreamWithHTTPClient(ctx context.Context, method string, return nil, err } + if c.config.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, c.config.Timeout) + defer cancel() + } + res, err := httpClient.Do(req) if err != nil { return nil, errors.Wrapf(err, "unable to perform request to %s %s", method, url) @@ -127,8 +131,8 @@ func (c *Client) createRequest(ctx context.Context, method string, url string, m return nil, errors.New("url is missing") } - if c.userAgent != "" { - mutators = append(mutators, request.NewHeaderMutator("User-Agent", c.userAgent)) + if c.config.UserAgent != "" { + mutators = append(mutators, request.NewHeaderMutator("User-Agent", c.config.UserAgent)) } var body io.Reader diff --git a/client/config.go b/client/config.go index 2d2d811a00..9a094aeecf 100644 --- a/client/config.go +++ b/client/config.go @@ -2,6 +2,8 @@ package client import ( "net/url" + "strconv" + "time" "github.com/kelseyhightower/envconfig" @@ -23,6 +25,9 @@ type Config struct { // // More info: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent UserAgent string `envconfig:"TIDEPOOL_USER_AGENT"` + + // Timeout specifies the maximum amount of time a request can take. Zero means no timeout. + Timeout time.Duration } func NewConfig() *Config { @@ -36,6 +41,13 @@ func (c *Config) Load(loader ConfigLoader) error { func (c *Config) LoadFromConfigReporter(reporter config.Reporter) error { c.Address = reporter.GetWithDefault("address", c.Address) c.UserAgent = reporter.GetWithDefault("user_agent", c.UserAgent) + if timeoutString, err := reporter.Get("timeout"); err == nil { + if timeout, parseErr := strconv.ParseInt(timeoutString, 10, 0); parseErr != nil { + return errors.New("timeout is invalid") + } else { + c.Timeout = time.Duration(timeout) * time.Second + } + } return nil } @@ -45,6 +57,9 @@ func (c *Config) Validate() error { } else if _, err := url.Parse(c.Address); err != nil { return errors.New("address is invalid") } + if c.Timeout < 0 { + return errors.New("timeout is invalid") + } return nil } diff --git a/client/prometheus.go b/client/prometheus.go index 45d579d35b..79275da865 100644 --- a/client/prometheus.go +++ b/client/prometheus.go @@ -116,7 +116,7 @@ func NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(n func (p *PrometheusRequestMetricsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { start := time.Now() - res, err := p.ResolvedRoundTripper().RoundTrip(req) + res, err := p.PrometheusRequestRoundTripper.RoundTrip(req) duration := time.Since(start) if labels := p.Labels(req, res); labels != nil { diff --git a/dexcom/client/client.go b/dexcom/client/client.go index 29a219d8a3..e026fb1256 100644 --- a/dexcom/client/client.go +++ b/dexcom/client/client.go @@ -2,8 +2,13 @@ package client import ( "context" + "fmt" + "net/http" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/dexcom" "github.com/tidepool-org/platform/errors" @@ -17,12 +22,29 @@ type Client struct { } func New(cfg *client.Config, tknSrcSrc oauth.TokenSourceSource) (*Client, error) { + if cfg == nil { + return nil, errors.New("config is missing") + } else if err := cfg.Validate(); err != nil { + return nil, errors.Wrap(err, "config is invalid") + } + + if cfg.Timeout == 0 { + cfg.Timeout = 1 * time.Minute + } + + httpClient := &http.Client{ + Transport: prometheusRequestMetricsRoundTripper, + CheckRedirect: http.DefaultClient.CheckRedirect, + Jar: http.DefaultClient.Jar, + Timeout: http.DefaultClient.Timeout, + } + baseClient, err := client.New(cfg) if err != nil { return nil, err } - clnt, err := oauthClient.NewWithClient(baseClient, tknSrcSrc) + clnt, err := oauthClient.NewWithClient(baseClient, httpClient, tknSrcSrc) if err != nil { return nil, err } @@ -118,4 +140,45 @@ func (c *Client) sendDexcomRequest(ctx context.Context, method string, url strin }) } +// Some Dexcom API responses include a "request-time" header with, supposedly, the internal duration of the request. +// This could be useful for debugging connection issues if compared against the calculated request duration. +// See client/prometheus.go for details on how the request duration is calculated and recorded. + +const RequestTimeHeaderName = "request-time" + +type PrometheusRequestMetricsRoundTripper struct { + *client.PrometheusRequestMetricsRoundTripper + requestTimeHistogramVec *prometheus.HistogramVec +} + +func NewPrometheusRequestMetricsRoundTripper(name string, help string) *PrometheusRequestMetricsRoundTripper { + return &PrometheusRequestMetricsRoundTripper{ + PrometheusRequestMetricsRoundTripper: client.NewPrometheusRequestMetricsRoundTripper(name, help), + requestTimeHistogramVec: promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: fmt.Sprintf("%s_request_time_seconds", name), + Help: fmt.Sprintf("%s request time (seconds)", help), + Buckets: client.DurationBucketsDefault, + }, + client.PrometheusLabelNames(), + ), + } +} + +func (p *PrometheusRequestMetricsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + res, err := p.PrometheusRequestMetricsRoundTripper.RoundTrip(req) + + if res != nil { + if labels := p.Labels(req, res); labels != nil { + if requestTime, parseErr := time.ParseDuration(res.Header.Get(RequestTimeHeaderName)); parseErr == nil { + p.requestTimeHistogramVec.With(*labels).Observe(requestTime.Seconds()) + } + } + } + + return res, err +} + const requestDurationMaximum = 60 * time.Second + +var prometheusRequestMetricsRoundTripper = NewPrometheusRequestMetricsRoundTripper("tidepool_dexcom_api", "Tidepool Dexcom API") diff --git a/dexcom/client/client_test.go b/dexcom/client/client_test.go index bfffb34488..bebb889638 100644 --- a/dexcom/client/client_test.go +++ b/dexcom/client/client_test.go @@ -4,12 +4,15 @@ import ( "context" "fmt" "net/http" + "strconv" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/ghttp" + "go.uber.org/mock/gomock" + "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/dexcom" dexcomClient "github.com/tidepool-org/platform/dexcom/client" @@ -20,6 +23,7 @@ import ( logTest "github.com/tidepool-org/platform/log/test" oauthTest "github.com/tidepool-org/platform/oauth/test" "github.com/tidepool-org/platform/pointer" + prometheusTest "github.com/tidepool-org/platform/prometheus/test" "github.com/tidepool-org/platform/test" testHttp "github.com/tidepool-org/platform/test/http" ) @@ -27,17 +31,15 @@ import ( var _ = Describe("Client", func() { var userAgent string var config *client.Config - var tokenSourceSource *oauthTest.TokenSourceSource + var mockController *gomock.Controller + var mockTokenSourceSource *oauthTest.MockTokenSourceSource BeforeEach(func() { userAgent = testHttp.NewUserAgent() config = client.NewConfig() config.UserAgent = userAgent - tokenSourceSource = oauthTest.NewTokenSourceSource() - }) - - AfterEach(func() { - tokenSourceSource.AssertOutputsEmpty() + mockController = gomock.NewController(GinkgoT()) + mockTokenSourceSource = oauthTest.NewMockTokenSourceSource(mockController) }) Context("New", func() { @@ -46,14 +48,14 @@ var _ = Describe("Client", func() { }) It("returns an error when config is missing", func() { - clnt, err := dexcomClient.New(nil, tokenSourceSource) + clnt, err := dexcomClient.New(nil, mockTokenSourceSource) Expect(err).To(MatchError("config is missing")) Expect(clnt).To(BeNil()) }) It("returns an error when config is invalid", func() { config.Address = "" - clnt, err := dexcomClient.New(config, tokenSourceSource) + clnt, err := dexcomClient.New(config, mockTokenSourceSource) Expect(err).To(MatchError("config is invalid; address is missing")) Expect(clnt).To(BeNil()) }) @@ -65,7 +67,7 @@ var _ = Describe("Client", func() { }) It("returns successfully", func() { - Expect(dexcomClient.New(config, tokenSourceSource)).ToNot(BeNil()) + Expect(dexcomClient.New(config, mockTokenSourceSource)).ToNot(BeNil()) }) }) @@ -73,20 +75,20 @@ var _ = Describe("Client", func() { var server *Server var responseHeaders http.Header var ctx context.Context - var tokenSource *oauthTest.TokenSource + var mockTokenSource *oauthTest.MockTokenSource var clnt *dexcomClient.Client BeforeEach(func() { server = NewServer() responseHeaders = http.Header{"Content-Type": []string{"application/json; charset=utf-8"}} ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) - tokenSource = oauthTest.NewTokenSource() + mockTokenSource = oauthTest.NewMockTokenSource(mockController) }) JustBeforeEach(func() { config.Address = server.URL() var err error - clnt, err = dexcomClient.New(config, tokenSourceSource) + clnt, err = dexcomClient.New(config, mockTokenSourceSource) Expect(err).ToNot(HaveOccurred()) Expect(clnt).ToNot(BeNil()) }) @@ -95,7 +97,6 @@ var _ = Describe("Client", func() { if server != nil { server.Close() } - tokenSource.AssertOutputsEmpty() }) Context("GetDataRange", func() { @@ -104,6 +105,8 @@ var _ = Describe("Client", func() { var responseDataRangesResponse *dexcom.DataRangesResponse BeforeEach(func() { + lastSyncTime = nil + requestQuery = "" responseDataRangesResponse = dexcomTest.RandomDataRangesResponse() }) @@ -114,23 +117,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + dataRangeResponse, err := clnt.GetDataRange(context.Context(nil), lastSyncTime, mockTokenSource) + Expect(err).To(MatchError("unable to get data range; context is missing")) + Expect(dataRangeResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get data range; %s", responseErr))) Expect(dataRangeResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError(`unable to get data range; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(dataRangeResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -139,26 +147,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) - Expect(err).To(MatchError("unable to get data range; context is missing")) - Expect(dataRangeResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get data range; unable to perform request to .*: connect: connection refused")) Expect(dataRangeResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -175,7 +173,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; bad request")) Expect(dataRangeResponse).To(BeNil()) }) @@ -194,7 +192,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; authentication token is not authorized for requested action")) Expect(dataRangeResponse).To(BeNil()) }) @@ -213,7 +211,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; resource not found")) Expect(dataRangeResponse).To(BeNil()) }) @@ -232,7 +230,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get data range; unexpected response status code 500 from")) Expect(dataRangeResponse).To(BeNil()) @@ -252,7 +250,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; json is malformed")) Expect(dataRangeResponse).To(BeNil()) }) @@ -271,7 +269,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(dataRangeResponse).To(Equal(responseDataRangesResponse)) }) @@ -285,8 +283,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -295,8 +291,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request without last sync time", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -305,9 +299,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/dataRange", requestQuery), @@ -319,8 +313,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -339,7 +331,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, tokenSource) + dataRangeResponse, err := clnt.GetDataRange(ctx, lastSyncTime, mockTokenSource) Expect(err).To(MatchError("unable to get data range; authentication token is invalid")) Expect(dataRangeResponse).To(BeNil()) }) @@ -373,23 +365,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + alertsResponse, err := clnt.GetAlerts(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get alerts; context is missing")) + Expect(alertsResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get alerts; %s", responseErr))) Expect(alertsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get alerts; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(alertsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -398,26 +395,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get alerts; context is missing")) - Expect(alertsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get alerts; unable to perform request to .*: connect: connection refused")) Expect(alertsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -434,7 +421,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; bad request")) Expect(alertsResponse).To(BeNil()) }) @@ -453,7 +440,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; authentication token is not authorized for requested action")) Expect(alertsResponse).To(BeNil()) }) @@ -472,7 +459,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; resource not found")) Expect(alertsResponse).To(BeNil()) }) @@ -491,7 +478,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get alerts; unexpected response status code 500 from")) Expect(alertsResponse).To(BeNil()) @@ -511,7 +498,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; json is malformed")) Expect(alertsResponse).To(BeNil()) }) @@ -530,7 +517,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(alertsResponse).To(Equal(responseAlertsResponse)) }) @@ -539,8 +526,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) requestAssertions() @@ -548,9 +533,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/alerts", requestQuery), @@ -562,8 +547,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -582,7 +565,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, tokenSource) + alertsResponse, err := clnt.GetAlerts(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get alerts; authentication token is invalid")) Expect(alertsResponse).To(BeNil()) }) @@ -605,23 +588,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + calibrationsResponse, err := clnt.GetCalibrations(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get calibrations; context is missing")) + Expect(calibrationsResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get calibrations; %s", responseErr))) Expect(calibrationsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get calibrations; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(calibrationsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -630,26 +618,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get calibrations; context is missing")) - Expect(calibrationsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get calibrations; unable to perform request to .*: connect: connection refused")) Expect(calibrationsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -666,7 +644,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; bad request")) Expect(calibrationsResponse).To(BeNil()) }) @@ -685,7 +663,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; authentication token is not authorized for requested action")) Expect(calibrationsResponse).To(BeNil()) }) @@ -704,7 +682,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; resource not found")) Expect(calibrationsResponse).To(BeNil()) }) @@ -723,7 +701,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get calibrations; unexpected response status code 500 from")) Expect(calibrationsResponse).To(BeNil()) @@ -743,7 +721,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; json is malformed")) Expect(calibrationsResponse).To(BeNil()) }) @@ -762,7 +740,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(calibrationsResponse).To(Equal(responseCalibrationsResponse)) }) @@ -771,8 +749,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -781,9 +757,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/calibrations", requestQuery), @@ -795,8 +771,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -815,7 +789,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, tokenSource) + calibrationsResponse, err := clnt.GetCalibrations(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get calibrations; authentication token is invalid")) Expect(calibrationsResponse).To(BeNil()) }) @@ -838,23 +812,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + devicesResponse, err := clnt.GetDevices(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get devices; context is missing")) + Expect(devicesResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get devices; %s", responseErr))) Expect(devicesResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get devices; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(devicesResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -863,26 +842,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get devices; context is missing")) - Expect(devicesResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get devices; unable to perform request to .*: connect: connection refused")) Expect(devicesResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -899,7 +868,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; bad request")) Expect(devicesResponse).To(BeNil()) }) @@ -918,7 +887,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; authentication token is not authorized for requested action")) Expect(devicesResponse).To(BeNil()) }) @@ -937,7 +906,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; resource not found")) Expect(devicesResponse).To(BeNil()) }) @@ -956,7 +925,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get devices; unexpected response status code 500 from")) Expect(devicesResponse).To(BeNil()) @@ -976,7 +945,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; json is malformed")) Expect(devicesResponse).To(BeNil()) }) @@ -995,7 +964,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(devicesResponse).To(Equal(responseDevicesResponse)) }) @@ -1004,8 +973,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -1014,9 +981,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/devices", requestQuery), @@ -1028,8 +995,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -1048,7 +1013,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, tokenSource) + devicesResponse, err := clnt.GetDevices(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get devices; authentication token is invalid")) Expect(devicesResponse).To(BeNil()) }) @@ -1071,23 +1036,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + egvsResponse, err := clnt.GetEGVs(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get egvs; context is missing")) + Expect(egvsResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get egvs; %s", responseErr))) Expect(egvsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get egvs; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(egvsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -1096,26 +1066,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get egvs; context is missing")) - Expect(egvsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get egvs; unable to perform request to .*: connect: connection refused")) Expect(egvsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -1132,7 +1092,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; bad request")) Expect(egvsResponse).To(BeNil()) }) @@ -1151,7 +1111,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; authentication token is not authorized for requested action")) Expect(egvsResponse).To(BeNil()) }) @@ -1170,7 +1130,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; resource not found")) Expect(egvsResponse).To(BeNil()) }) @@ -1189,7 +1149,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get egvs; unexpected response status code 500 from")) Expect(egvsResponse).To(BeNil()) @@ -1209,7 +1169,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; json is malformed")) Expect(egvsResponse).To(BeNil()) }) @@ -1228,7 +1188,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(egvsResponse).To(Equal(responseEGVsResponse)) }) @@ -1237,8 +1197,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -1247,9 +1205,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/egvs", requestQuery), @@ -1261,8 +1219,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -1281,7 +1237,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, tokenSource) + egvsResponse, err := clnt.GetEGVs(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get egvs; authentication token is invalid")) Expect(egvsResponse).To(BeNil()) }) @@ -1304,23 +1260,28 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(BeEmpty()) }) + It("returns error when context is missing", func() { + eventsResponse, err := clnt.GetEvents(context.Context(nil), startTime, endTime, mockTokenSource) + Expect(err).To(MatchError("unable to get events; context is missing")) + Expect(eventsResponse).To(BeNil()) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf("unable to get events; %s", responseErr))) Expect(eventsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when token source returns that indicates an oauth token failure", func() { responseErr := errors.New(`oauth2: "invalid_grant"`) - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError(`unable to get events; oauth2: "invalid_grant"; authentication token is invalid`)) Expect(eventsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -1329,26 +1290,16 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) - Expect(err).To(MatchError("unable to get events; context is missing")) - Expect(eventsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err.Error()).To(MatchRegexp("unable to get events; unable to perform request to .*: connect: connection refused")) Expect(eventsResponse).To(BeNil()) - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) }) requestAssertions := func() { @@ -1365,7 +1316,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; bad request")) Expect(eventsResponse).To(BeNil()) }) @@ -1384,7 +1335,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; authentication token is not authorized for requested action")) Expect(eventsResponse).To(BeNil()) }) @@ -1403,7 +1354,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; resource not found")) Expect(eventsResponse).To(BeNil()) }) @@ -1422,7 +1373,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(MatchRegexp("unable to get events; unexpected response status code 500 from")) Expect(eventsResponse).To(BeNil()) @@ -1442,7 +1393,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; json is malformed")) Expect(eventsResponse).To(BeNil()) }) @@ -1461,7 +1412,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).ToNot(HaveOccurred()) Expect(eventsResponse).To(Equal(responseEventsResponse)) }) @@ -1470,8 +1421,6 @@ var _ = Describe("Client", func() { When("the server responds directly to the one request", func() { AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(0)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -1480,9 +1429,9 @@ var _ = Describe("Client", func() { When("the server responds with unauthorized, the token is expired and the request retried", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest("GET", "/v3/users/self/events", requestQuery), @@ -1494,8 +1443,6 @@ var _ = Describe("Client", func() { }) AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal([]oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}})) - Expect(tokenSource.ExpireTokenInvocations).To(Equal(1)) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -1514,7 +1461,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, tokenSource) + eventsResponse, err := clnt.GetEvents(ctx, startTime, endTime, mockTokenSource) Expect(err).To(MatchError("unable to get events; authentication token is invalid")) Expect(eventsResponse).To(BeNil()) }) @@ -1524,4 +1471,98 @@ var _ = Describe("Client", func() { }) }) }) + + It("RequestTimeHeaderName is expected", func() { + Expect(dexcomClient.RequestTimeHeaderName).To(Equal("request-time")) + }) + + Context("PrometheusRequestMetricsRoundTripper", func() { + Context("NewPrometheusRequestMetricsRoundTripper", func() { + It("returns successfully", func() { + roundTripper := dexcomClient.NewPrometheusRequestMetricsRoundTripper(prometheusTest.RandomMetricName(), prometheusTest.RandomMetricHelp()) + Expect(roundTripper).ToNot(BeNil()) + Expect(roundTripper.PrometheusRequestMetricsRoundTripper).ToNot(BeNil()) + }) + }) + + Context("RoundTrip", func() { + var testRoundTripper *testHttp.RoundTripper + var name string + var roundTripper *dexcomClient.PrometheusRequestMetricsRoundTripper + var request *http.Request + + BeforeEach(func() { + testRoundTripper = testHttp.NewRoundTripper() + name = prometheusTest.RandomMetricName() + roundTripper = dexcomClient.NewPrometheusRequestMetricsRoundTripper(name, prometheusTest.RandomMetricHelp()) + roundTripper.WithRoundTripper(testRoundTripper) + request = testHttp.NewRequest() + }) + + It("returns the response from the resolved round tripper", func() { + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} + + result := test.Must(roundTripper.RoundTrip(request)) + Expect(result).To(BeIdenticalTo(testRoundTripper.Response)) + Expect(testRoundTripper.Request).To(BeIdenticalTo(request)) + }) + + It("returns the error from the resolved round tripper", func() { + testErr := errorsTest.RandomError() + testRoundTripper.Error = testErr + + result, err := roundTripper.RoundTrip(request) + Expect(err).To(Equal(testErr)) + Expect(result).To(BeNil()) + }) + + It("does not record a request time metric when the resolved round tripper returns an error", func() { + testRoundTripper.Error = errorsTest.RandomError() + + _, _ = roundTripper.RoundTrip(request) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) + }) + + It("records a request time metric when the response has a valid request-time header", func() { + statusCode := testHttp.NewStatusCode() + requestTime := time.Duration(test.RandomIntFromRange(1, 60*1000)) * time.Millisecond + header := http.Header{} + header.Set(dexcomClient.RequestTimeHeaderName, requestTime.String()) + testRoundTripper.Response = &http.Response{StatusCode: statusCode, Header: header} + + _ = test.Must(roundTripper.RoundTrip(request)) + + family := prometheusTest.MetricFamilyFromName(name + "_request_time_seconds") + Expect(family).ToNot(BeNil()) + Expect(family.GetMetric()).To(HaveLen(1)) + metric := family.GetMetric()[0] + Expect(metric.GetHistogram().GetSampleCount()).To(Equal(uint64(1))) + Expect(metric.GetHistogram().GetSampleSum()).To(Equal(requestTime.Seconds())) + Expect(prometheusTest.LabelPairsToMap(metric.GetLabel())).To(Equal(map[string]string{ + client.PrometheusLabelNameMethod: request.Method, + client.PrometheusLabelNamePath: request.URL.Path, + client.PrometheusLabelNameStatus: strconv.Itoa(statusCode), + })) + }) + + It("does not record a request time metric when the response does not have a request-time header", func() { + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode(), Header: http.Header{}} + + _ = test.Must(roundTripper.RoundTrip(request)) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) + }) + + It("does not record a request time metric when the request-time header is not a valid duration", func() { + header := http.Header{} + header.Set(dexcomClient.RequestTimeHeaderName, test.RandomString()) + testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode(), Header: header} + + _ = test.Must(roundTripper.RoundTrip(request)) + + Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) + }) + }) + }) }) diff --git a/dexcom/fetch/runner.go b/dexcom/fetch/runner.go index ba559ad6f3..28b0f4406b 100644 --- a/dexcom/fetch/runner.go +++ b/dexcom/fetch/runner.go @@ -411,13 +411,14 @@ func (t *TaskRunner) fetchSinceLatestDataTime() error { // If past deadline (based upon runner maximum duration), then bail if time.Now().After(t.deadline) { - return t.rescheduleTaskWithResourceError(context.DeadlineExceeded) + t.rescheduleTaskNow() + return nil } startTime = startTime.AddDate(0, 0, DataRangeDaysMaximum) } - return t.updateDataSourceWithLastImportTime() + return nil } func (t *TaskRunner) fetchDataRange() (*DataRange, error) { @@ -777,8 +778,8 @@ func (t *TaskRunner) afterLatestDataTime(latestDataTime *time.Time) bool { return latestDataTime != nil && (t.dataSource.LatestDataTime == nil || latestDataTime.After(*t.dataSource.LatestDataTime)) } -// Handle potential dexcom client error. Update provider session with latest token. -// If error, then retry or reschedule. Otherwise, reset retry count. +// Handle potential dexcom client error. If error, then retry or reschedule. +// Otherwise, reset retry count. func (t *TaskRunner) handleDexcomClientError(err error) error { if err != nil { return t.retryOrRescheduleTaskWithDexcomClientError(err) @@ -821,6 +822,10 @@ func (t *TaskRunner) rescheduleTaskWithError(err error) error { return err } +func (t *TaskRunner) rescheduleTaskNow() { + t.task.RepeatAvailableAfter(0) +} + func (t *TaskRunner) rescheduleTask() { t.task.RepeatAvailableAfter(availableAfterDuration()) } @@ -832,8 +837,7 @@ func (t *TaskRunner) failTaskWithInvalidStateError(err error) error { // Fail task immediately and permanently. Do not reschedule. For situations where any future attempt is // also guaranteed to fail. For example, when the task data is missing information. Should not normally happen. func (t *TaskRunner) failTaskWithError(err error) error { - t.task.AppendError(err) - t.task.SetFailed() + t.task.SetFailedWithError(err) return err } @@ -844,7 +848,7 @@ func (t *TaskRunner) incrementTaskRetryCount() int { retryCount = int(value) + 1 } } - t.task.Data[dexcom.DataKeyRetryCount] = retryCount + t.task.Data[dexcom.DataKeyRetryCount] = int32(retryCount) return retryCount } diff --git a/dexcom/fetch/runner_test.go b/dexcom/fetch/runner_test.go index 5d0b55bb20..70c632800c 100644 --- a/dexcom/fetch/runner_test.go +++ b/dexcom/fetch/runner_test.go @@ -207,7 +207,7 @@ var _ = Describe("Runner", func() { } assertTaskRetryCount := func(retryCount int) { - Expect(tsk.Data[dexcom.DataKeyRetryCount]).To(Equal(retryCount)) + Expect(tsk.Data[dexcom.DataKeyRetryCount]).To(Equal(int32(retryCount))) } assertTaskRetryCountNotPresent := func() { @@ -258,7 +258,7 @@ var _ = Describe("Runner", func() { It("fails if getting the data source fails", func() { testErr := errorsTest.RandomError() - dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(nil, testErr).Times(1) + dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(nil, testErr) taskRunner.Run(ctx) assertTaskState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -266,7 +266,7 @@ var _ = Describe("Runner", func() { }) It("fails if the data source is missing", func() { - dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(nil, nil).Times(1) + dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(nil, nil) taskRunner.Run(ctx) assertTaskState(task.TaskStateFailed) assertTaskRetryCountNotPresent() @@ -282,7 +282,7 @@ var _ = Describe("Runner", func() { ProviderSessionID: pointer.FromString("test-provider-session-id"), State: dataSource.StateConnected, } - dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(dataSrc, nil).Times(1) + dataSourceClient.EXPECT().Get(matchContext(), "test-data-source-id").Return(dataSrc, nil) }) assertTaskAndDataSourceState := func(state string) { @@ -312,7 +312,7 @@ var _ = Describe("Runner", func() { It("fails if provider session id is missing and update data source returns an error", func() { testErr := errorsTest.RandomError() delete(tsk.Data, dexcom.DataKeyProviderSessionID) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).Return(nil, testErr).Times(1) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).Return(nil, testErr) taskRunner.Run(ctx) assertTaskState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -321,7 +321,7 @@ var _ = Describe("Runner", func() { It("fails if provider session id is missing", func() { delete(tsk.Data, dexcom.DataKeyProviderSessionID) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStateFailed) assertTaskRetryCountNotPresent() @@ -330,7 +330,7 @@ var _ = Describe("Runner", func() { It("fails if provider session id is empty", func() { tsk.Data[dexcom.DataKeyProviderSessionID] = "" - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStateFailed) assertTaskRetryCountNotPresent() @@ -339,8 +339,8 @@ var _ = Describe("Runner", func() { It("fails if getting the provider session fails", func() { testErr := errorsTest.RandomError() - authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(nil, testErr).Times(1) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(nil, testErr) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -348,8 +348,8 @@ var _ = Describe("Runner", func() { }) It("fails if the provider session is missing", func() { - authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(nil, nil).Times(1) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(nil, nil) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStateFailed) assertTaskRetryCountNotPresent() @@ -372,8 +372,8 @@ var _ = Describe("Runner", func() { UserID: "test-user-id", OAuthToken: oauthToken, } - authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(providerSession, nil).Times(1) - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(1) + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(providerSession, nil) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) }) assertProviderSessionRefreshedTimes := func(times int) { @@ -412,7 +412,7 @@ var _ = Describe("Runner", func() { It("fails if get data ranges returns a general error", func() { testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, nil, testErr)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, nil, testErr)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -424,7 +424,7 @@ var _ = Describe("Runner", func() { latestDataTime := pointer.FromTime(time.Now().Add(-Day)) dataSrc.LatestDataTime = latestDataTime testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetDataRange(matchContext(), latestDataTime, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, nil, testErr)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), latestDataTime, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, nil, testErr)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -434,8 +434,8 @@ var _ = Describe("Runner", func() { It("fails if get data ranges refreshes the token and returns a general error", func() { testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -445,8 +445,8 @@ var _ = Describe("Runner", func() { It("fails if get data ranges refreshes the token and returns an authentication error", func() { testErr := request.ErrorUnauthenticated() - dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCount(1) @@ -468,12 +468,13 @@ var _ = Describe("Runner", func() { End: &dexcom.Moment{SystemTime: &dexcom.Time{Time: endTime}}, }, } - dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, dataRangeResponse, nil)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(&MockTokenSource{Refresh: true}, dataRangeResponse, nil)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) }) It("is successful if the Dexcom data ranges is not valid", func() { dataRangeResponse.Calibrations.Start = nil + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -483,6 +484,7 @@ var _ = Describe("Runner", func() { It("is successful if the Dexcom data ranges start is not before end", func() { dataRangeResponse.Calibrations.Start = &dexcom.Moment{SystemTime: &dexcom.Time{Time: time.Now().Add(-2 * Day)}} + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -492,8 +494,8 @@ var _ = Describe("Runner", func() { It("fails if get alerts returns a general error", func() { testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](nil, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](nil, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -503,8 +505,8 @@ var _ = Describe("Runner", func() { It("fails if get alerts refreshes the token and returns a general error", func() { testErr := errorsTest.RandomError() - dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](&MockTokenSource{Refresh: true}, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](&MockTokenSource{Refresh: true}, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -514,8 +516,8 @@ var _ = Describe("Runner", func() { It("fails if get alerts refreshes the token and returns an authentication error", func() { testErr := request.ErrorUnauthenticated() - dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](&MockTokenSource{Refresh: true}, nil, testErr)).Times(1) - authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(1) + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](&MockTokenSource{Refresh: true}, nil, testErr)) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCount(1) @@ -532,9 +534,9 @@ var _ = Describe("Runner", func() { BeforeEach(func() { alertsResponse = &dexcom.AlertsResponse{Records: &dexcom.Alerts{}} - dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, alertsResponse, nil)).Times(1) + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, alertsResponse, nil)) calibrationsResponse = &dexcom.CalibrationsResponse{Records: &dexcom.Calibrations{}} - dexcomClient.EXPECT().GetCalibrations(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, calibrationsResponse, nil)).Times(1) + dexcomClient.EXPECT().GetCalibrations(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, calibrationsResponse, nil)) devicesResponse = &dexcom.DevicesResponse{ Records: &dexcom.Devices{ { @@ -547,11 +549,11 @@ var _ = Describe("Runner", func() { }, }, } - dexcomClient.EXPECT().GetDevices(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, devicesResponse, nil)).Times(1) + dexcomClient.EXPECT().GetDevices(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, devicesResponse, nil)) egvsResponse = &dexcom.EGVsResponse{Records: &dexcom.EGVs{}} - dexcomClient.EXPECT().GetEGVs(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, egvsResponse, nil)).Times(1) + dexcomClient.EXPECT().GetEGVs(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, egvsResponse, nil)) eventsResponse = &dexcom.EventsResponse{Records: &dexcom.Events{}} - dexcomClient.EXPECT().GetEvents(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, eventsResponse, nil)).Times(1) + dexcomClient.EXPECT().GetEvents(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, eventsResponse, nil)) authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).Times(5) }) @@ -570,8 +572,8 @@ var _ = Describe("Runner", func() { UploadID: pointer.FromString("test-data-set-upload-id"), } dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(3) - dataClient.EXPECT().CreateUserDataSet(matchContext(), "test-user-id", matchNotNil()).DoAndReturn(mockDataClientCreateUserDataSet(dataSet, nil)).Times(1) - dataClient.EXPECT().CreateDataSetsData(matchContext(), "test-data-set-upload-id", matchNotNil()).DoAndReturn(mockDataClientCreateDataSetsData(nil)).Times(1) + dataClient.EXPECT().CreateUserDataSet(matchContext(), "test-user-id", matchNotNil()).DoAndReturn(mockDataClientCreateUserDataSet(dataSet, nil)) + dataClient.EXPECT().CreateDataSetsData(matchContext(), "test-data-set-upload-id", matchNotNil()).DoAndReturn(mockDataClientCreateDataSetsData(nil)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskDeviceHashesCount(3) diff --git a/dexcom/provider/provider.go b/dexcom/provider/provider.go index e4b98d4716..cafaff401d 100644 --- a/dexcom/provider/provider.go +++ b/dexcom/provider/provider.go @@ -2,15 +2,8 @@ package provider import ( "context" - "fmt" - "net/http" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/tidepool-org/platform/auth" - "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/config" dataSource "github.com/tidepool-org/platform/data/source" "github.com/tidepool-org/platform/dexcom" @@ -48,15 +41,7 @@ func New(configReporter config.Reporter, dataSourceClient dataSource.Client, tas return nil, errors.Wrap(err, "unable to create provider config") } - // Create http client - httpClient := &http.Client{ - Transport: prometheusRequestMetricsRoundTripper, - CheckRedirect: http.DefaultClient.CheckRedirect, - Jar: http.DefaultClient.Jar, - Timeout: 2 * time.Minute, - } - - prvdr, err := oauthProvider.New(dexcom.ProviderName, cfg, httpClient, nil) + prvdr, err := oauthProvider.New(dexcom.ProviderName, cfg, nil) if err != nil { return nil, err } @@ -172,44 +157,3 @@ func (p *Provider) OnDelete(ctx context.Context, providerSession *auth.ProviderS } return nil } - -// Some Dexcom API responses include a "request-time" header with, supposedly, the internal duration of the request. -// This could be useful for debugging connection issues if compared against the calculated request duration. -// See client/prometheus.go for details on how the request duration is calculated and recorded. - -const RequestTimeHeaderName = "request-time" - -type PrometheusRequestMetricsRoundTripper struct { - *client.PrometheusRequestMetricsRoundTripper - requestTimeHistogramVec *prometheus.HistogramVec -} - -func NewPrometheusRequestMetricsRoundTripper(name string, help string) *PrometheusRequestMetricsRoundTripper { - return &PrometheusRequestMetricsRoundTripper{ - PrometheusRequestMetricsRoundTripper: client.NewPrometheusRequestMetricsRoundTripper(name, help), - requestTimeHistogramVec: promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Name: fmt.Sprintf("%s_request_time_seconds", name), - Help: fmt.Sprintf("%s request time (seconds)", help), - Buckets: client.DurationBucketsDefault, - }, - client.PrometheusLabelNames(), - ), - } -} - -func (p *PrometheusRequestMetricsRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - res, err := p.ResolvedRoundTripper().RoundTrip(req) - - if res != nil { - if labels := p.Labels(req, res); labels != nil { - if requestTime, parseErr := time.ParseDuration(res.Header.Get(RequestTimeHeaderName)); parseErr == nil { - p.requestTimeHistogramVec.With(*labels).Observe(requestTime.Seconds()) - } - } - } - - return res, err -} - -var prometheusRequestMetricsRoundTripper = NewPrometheusRequestMetricsRoundTripper("tidepool_dexcom_api", "Tidepool Dexcom API") diff --git a/dexcom/provider/provider_test.go b/dexcom/provider/provider_test.go deleted file mode 100644 index 93744dd3eb..0000000000 --- a/dexcom/provider/provider_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package provider_test - -import ( - "net/http" - "strconv" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/tidepool-org/platform/client" - dexcomProvider "github.com/tidepool-org/platform/dexcom/provider" - errorsTest "github.com/tidepool-org/platform/errors/test" - prometheusTest "github.com/tidepool-org/platform/prometheus/test" - "github.com/tidepool-org/platform/test" - testHttp "github.com/tidepool-org/platform/test/http" -) - -var _ = Describe("Provider", func() { - It("RequestTimeHeaderName is expected", func() { - Expect(dexcomProvider.RequestTimeHeaderName).To(Equal("request-time")) - }) - - Context("PrometheusRequestMetricsRoundTripper", func() { - Context("NewPrometheusRequestMetricsRoundTripper", func() { - It("returns successfully", func() { - roundTripper := dexcomProvider.NewPrometheusRequestMetricsRoundTripper(prometheusTest.RandomMetricName(), prometheusTest.RandomMetricHelp()) - Expect(roundTripper).ToNot(BeNil()) - Expect(roundTripper.PrometheusRequestMetricsRoundTripper).ToNot(BeNil()) - }) - }) - - Context("RoundTrip", func() { - var testRoundTripper *testHttp.RoundTripper - var name string - var roundTripper *dexcomProvider.PrometheusRequestMetricsRoundTripper - var request *http.Request - - BeforeEach(func() { - testRoundTripper = testHttp.NewRoundTripper() - name = prometheusTest.RandomMetricName() - roundTripper = dexcomProvider.NewPrometheusRequestMetricsRoundTripper(name, prometheusTest.RandomMetricHelp()) - roundTripper.WithRoundTripper(testRoundTripper) - request = testHttp.NewRequest() - }) - - It("returns the response from the resolved round tripper", func() { - testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode()} - - result := test.Must(roundTripper.RoundTrip(request)) - Expect(result).To(BeIdenticalTo(testRoundTripper.Response)) - Expect(testRoundTripper.Request).To(BeIdenticalTo(request)) - }) - - It("returns the error from the resolved round tripper", func() { - testErr := errorsTest.RandomError() - testRoundTripper.Error = testErr - - result, err := roundTripper.RoundTrip(request) - Expect(err).To(Equal(testErr)) - Expect(result).To(BeNil()) - }) - - It("does not record a request time metric when the resolved round tripper returns an error", func() { - testRoundTripper.Error = errorsTest.RandomError() - - _, _ = roundTripper.RoundTrip(request) - - Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) - }) - - It("records a request time metric when the response has a valid request-time header", func() { - statusCode := testHttp.NewStatusCode() - requestTime := time.Duration(test.RandomIntFromRange(1, 60*1000)) * time.Millisecond - header := http.Header{} - header.Set(dexcomProvider.RequestTimeHeaderName, requestTime.String()) - testRoundTripper.Response = &http.Response{StatusCode: statusCode, Header: header} - - _ = test.Must(roundTripper.RoundTrip(request)) - - family := prometheusTest.MetricFamilyFromName(name + "_request_time_seconds") - Expect(family).ToNot(BeNil()) - Expect(family.GetMetric()).To(HaveLen(1)) - metric := family.GetMetric()[0] - Expect(metric.GetHistogram().GetSampleCount()).To(Equal(uint64(1))) - Expect(metric.GetHistogram().GetSampleSum()).To(Equal(requestTime.Seconds())) - Expect(prometheusTest.LabelPairsToMap(metric.GetLabel())).To(Equal(map[string]string{ - client.PrometheusLabelNameMethod: request.Method, - client.PrometheusLabelNamePath: request.URL.Path, - client.PrometheusLabelNameStatus: strconv.Itoa(statusCode), - })) - }) - - It("does not record a request time metric when the response does not have a request-time header", func() { - testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode(), Header: http.Header{}} - - _ = test.Must(roundTripper.RoundTrip(request)) - - Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) - }) - - It("does not record a request time metric when the request-time header is not a valid duration", func() { - header := http.Header{} - header.Set(dexcomProvider.RequestTimeHeaderName, test.RandomString()) - testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode(), Header: header} - - _ = test.Must(roundTripper.RoundTrip(request)) - - Expect(prometheusTest.MetricFamilyFromName(name + "_request_time_seconds")).To(BeNil()) - }) - }) - }) -}) diff --git a/ehr/sync/runner.go b/ehr/sync/runner.go index e3ec5c7360..107fb665b1 100644 --- a/ehr/sync/runner.go +++ b/ehr/sync/runner.go @@ -58,9 +58,8 @@ func (r *Runner) Run(ctx context.Context, tsk *task.Task) { func (r *Runner) doRun(ctx context.Context, tsk *task.Task) { clinicId, err := GetClinicId(tsk.Data) if err != nil { - tsk.AppendError(errors.Wrap(err, "unable to get clinicId from task data")) // Unrecoverable condition, move the task to failed state so it won't be retried - tsk.SetFailed() + tsk.SetFailedWithError(errors.Wrap(err, "unable to get clinicId from task data")) return } diff --git a/errors/errors.go b/errors/errors.go index a31ba89677..faadebf934 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -156,15 +156,7 @@ func WithSource(err error, src Source) error { if _, arrayOK := err.(*array); arrayOK { return err } else if objectErr, objectOK := err.(*object); objectOK { - return &object{ - Code: objectErr.Code, - Title: objectErr.Title, - Detail: objectErr.Detail, - Source: s, - Meta: objectErr.Meta, - Caller: objectErr.Caller, - Cause: objectErr.Cause, - } + return objectErr.withSource(s) } else if err != nil { return &object{ Detail: err.Error(), @@ -200,15 +192,7 @@ func WithMeta(err error, meta interface{}) error { if _, arrayOK := err.(*array); arrayOK { return err } else if objectErr, objectOK := err.(*object); objectOK { - return &object{ - Code: objectErr.Code, - Title: objectErr.Title, - Detail: objectErr.Detail, - Source: objectErr.Source, - Meta: meta, - Caller: objectErr.Caller, - Cause: objectErr.Cause, - } + return objectErr.withMeta(meta) } else if err != nil { return &object{ Detail: err.Error(), @@ -315,9 +299,6 @@ type Serializable struct { } func NewSerializable(err error) *Serializable { - if err == nil { - return nil - } return &Serializable{ Error: err, } @@ -363,11 +344,15 @@ func (s Serializable) MarshalJSON() ([]byte, error) { if arrayErr, arrayOK := s.Error.(*array); arrayOK { return json.Marshal(arrayErr.Errors) } else if objectErr, objectOK := s.Error.(*object); objectOK { - return json.Marshal(objectErr) + bites, err := json.Marshal(objectErr) + if err != nil && objectErr.Meta != nil { // If failure to marshal with meta, then remove meta and try again; likely unmarshallable value within meta + bites, err = json.Marshal(objectErr.withMeta(nil)) + } + return bites, err } else if s.Error != nil { return []byte(strconv.Quote(s.Error.Error())), nil } - return nil, nil + return []byte("null"), nil } func (s *Serializable) UnmarshalJSON(bites []byte) error { @@ -399,7 +384,11 @@ func (s Serializable) MarshalBSONValue() (bsontype.Type, []byte, error) { if arrayErr, arrayOK := s.Error.(*array); arrayOK { return bson.MarshalValue(arrayErr.Errors) } else if objectErr, objectOK := s.Error.(*object); objectOK { - return bson.MarshalValue(objectErr) + bsonType, bites, err := bson.MarshalValue(objectErr) + if err != nil && objectErr.Meta != nil { // If failure to marshal with meta, then remove meta and try again; likely unmarshallable value within meta + bsonType, bites, err = bson.MarshalValue(objectErr.withMeta(nil)) + } + return bsonType, bites, err } else if s.Error != nil { return bsontype.String, bsoncore.AppendString(nil, s.Error.Error()), nil } @@ -668,6 +657,30 @@ func (o *object) Is(target error) bool { return o.Cause != nil && o.Cause.Error == target } +func (o *object) withSource(src *source) *object { + return &object{ + Code: o.Code, + Title: o.Title, + Detail: o.Detail, + Source: src, + Meta: o.Meta, + Caller: o.Caller, + Cause: o.Cause, + } +} + +func (o *object) withMeta(meta any) *object { + return &object{ + Code: o.Code, + Title: o.Title, + Detail: o.Detail, + Source: o.Source, + Meta: meta, + Caller: o.Caller, + Cause: o.Cause, + } +} + type contextKey string const errorContextKey contextKey = "error" diff --git a/errors/errors_test.go b/errors/errors_test.go index 7a5596be8b..13876ee7eb 100644 --- a/errors/errors_test.go +++ b/errors/errors_test.go @@ -182,9 +182,10 @@ var _ = Describe("Errors", func() { }) Context("NewSerializable", func() { - It("returns nil if the error is nil", func() { + It("returns a serializable if the error is nil", func() { serializable := errors.NewSerializable(nil) - Expect(serializable).To(BeNil()) + Expect(serializable).ToNot(BeNil()) + Expect(serializable.Error).To(BeNil()) }) It("returns a serializable if the error is not nil", func() { diff --git a/oauth/client/client.go b/oauth/client/client.go index 3b0dde4187..9b2198ff56 100644 --- a/oauth/client/client.go +++ b/oauth/client/client.go @@ -2,6 +2,9 @@ package client import ( "context" + "net/http" + + "golang.org/x/oauth2" "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/errors" @@ -12,23 +15,24 @@ import ( type Client struct { baseClient *client.Client + httpClient *http.Client tokenSourceSource oauth.TokenSourceSource } -func New(config *client.Config, tokenSourceSource oauth.TokenSourceSource) (*Client, error) { - return NewWithErrorParser(config, tokenSourceSource, nil) +func New(config *client.Config, httpClient *http.Client, tokenSourceSource oauth.TokenSourceSource) (*Client, error) { + return NewWithErrorParser(config, httpClient, tokenSourceSource, nil) } -func NewWithErrorParser(config *client.Config, tokenSourceSource oauth.TokenSourceSource, errorResponseParser client.ErrorResponseParser) (*Client, error) { +func NewWithErrorParser(config *client.Config, httpClient *http.Client, tokenSourceSource oauth.TokenSourceSource, errorResponseParser client.ErrorResponseParser) (*Client, error) { baseClient, err := client.NewWithErrorParser(config, errorResponseParser) if err != nil { return nil, err } - return NewWithClient(baseClient, tokenSourceSource) + return NewWithClient(baseClient, httpClient, tokenSourceSource) } -func NewWithClient(baseClient *client.Client, tokenSourceSource oauth.TokenSourceSource) (*Client, error) { +func NewWithClient(baseClient *client.Client, httpClient *http.Client, tokenSourceSource oauth.TokenSourceSource) (*Client, error) { if baseClient == nil { return nil, errors.New("base client is missing") } @@ -39,6 +43,7 @@ func NewWithClient(baseClient *client.Client, tokenSourceSource oauth.TokenSourc return &Client{ baseClient: baseClient, tokenSourceSource: tokenSourceSource, + httpClient: httpClient, }, nil } @@ -81,6 +86,14 @@ func (c *Client) SendOAuthRequest(ctx context.Context, method string, url string } func (c *Client) sendOAuthRequest(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody interface{}, responseBody interface{}, inspectors []request.ResponseInspector, tokenSource oauth.TokenSource) error { + if ctx == nil { + return errors.New("context is missing") + } + + if c.httpClient != nil { + ctx = context.WithValue(ctx, oauth2.HTTPClient, c.httpClient) + } + httpClient, err := tokenSource.HTTPClient(ctx, c.tokenSourceSource) if err != nil { return err diff --git a/oauth/client/client_test.go b/oauth/client/client_test.go index 3ea6e58958..fdf2dd832e 100644 --- a/oauth/client/client_test.go +++ b/oauth/client/client_test.go @@ -11,6 +11,8 @@ import ( . "github.com/onsi/gomega" . "github.com/onsi/gomega/ghttp" + "go.uber.org/mock/gomock" + "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/errors" errorsTest "github.com/tidepool-org/platform/errors/test" @@ -36,7 +38,8 @@ var _ = Describe("Client", func() { var address string var userAgent string var baseConfig *client.Config - var tokenSourceSource *oauthTest.TokenSourceSource + var mockController *gomock.Controller + var mockTokenSourceSource *oauthTest.MockTokenSourceSource BeforeEach(func() { address = testHttp.NewAddress() @@ -44,22 +47,19 @@ var _ = Describe("Client", func() { baseConfig = client.NewConfig() baseConfig.Address = address baseConfig.UserAgent = userAgent - tokenSourceSource = oauthTest.NewTokenSourceSource() - }) - - AfterEach(func() { - tokenSourceSource.AssertOutputsEmpty() + mockController = gomock.NewController(GinkgoT()) + mockTokenSourceSource = oauthTest.NewMockTokenSourceSource(mockController) }) Context("New", func() { It("returns an error when token source source is missing", func() { - clnt, err := oauthClient.New(baseConfig, nil) + clnt, err := oauthClient.New(baseConfig, nil, nil) Expect(err).To(MatchError("token source source is missing")) Expect(clnt).To(BeNil()) }) It("returns successfully", func() { - Expect(oauthClient.New(baseConfig, tokenSourceSource)).ToNot(BeNil()) + Expect(oauthClient.New(baseConfig, nil, mockTokenSourceSource)).ToNot(BeNil()) }) }) @@ -68,7 +68,7 @@ var _ = Describe("Client", func() { JustBeforeEach(func() { var err error - clnt, err = oauthClient.New(baseConfig, tokenSourceSource) + clnt, err = oauthClient.New(baseConfig, nil, mockTokenSourceSource) Expect(err).ToNot(HaveOccurred()) Expect(clnt).ToNot(BeNil()) }) @@ -172,7 +172,7 @@ var _ = Describe("Client", func() { var requestString string var requestBody *RequestBody var responseString string - var tokenSource *oauthTest.TokenSource + var mockTokenSource *oauthTest.MockTokenSource var clnt *oauthClient.Client BeforeEach(func() { @@ -188,13 +188,13 @@ var _ = Describe("Client", func() { requestString = test.RandomStringFromRangeAndCharset(0, 32, test.CharsetText) requestBody = &RequestBody{Request: requestString} responseString = test.RandomStringFromRangeAndCharset(0, 32, test.CharsetText) - tokenSource = oauthTest.NewTokenSource() + mockTokenSource = oauthTest.NewMockTokenSource(mockController) baseConfig.Address = server.URL() }) JustBeforeEach(func() { var err error - clnt, err = oauthClient.New(baseConfig, tokenSourceSource) + clnt, err = oauthClient.New(baseConfig, nil, mockTokenSourceSource) Expect(err).ToNot(HaveOccurred()) Expect(clnt).ToNot(BeNil()) }) @@ -203,7 +203,6 @@ var _ = Describe("Client", func() { if server != nil { server.Close() } - tokenSource.AssertOutputsEmpty() }) Context("SendOAuthRequest", func() { @@ -213,26 +212,21 @@ var _ = Describe("Client", func() { responseBody = &ResponseBody{} }) + It("returns error when context is missing", func() { + Expect(clnt.SendOAuthRequest(context.Context(nil), method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(MatchError("context is missing")) + Expect(server.ReceivedRequests()).To(BeEmpty()) + }) + It("returns error when token source is missing", func() { Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, nil)).To(MatchError("token source is missing")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) When("token source is not missing", func() { - var expectedHTTPClientInputs []oauthTest.HTTPClientInput - - BeforeEach(func() { - expectedHTTPClientInputs = []oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}} - }) - - AfterEach(func() { - Expect(tokenSource.HTTPClientInputs).To(Equal(expectedHTTPClientInputs)) - }) - It("returns error when token source returns an error", func() { responseErr := errorsTest.RandomError() - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: nil, Error: responseErr}} - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(Equal(responseErr)) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(nil, responseErr) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(Equal(responseErr)) Expect(server.ReceivedRequests()).To(BeEmpty()) }) @@ -241,44 +235,37 @@ var _ = Describe("Client", func() { BeforeEach(func() { httpClient = http.DefaultClient - tokenSource.HTTPClientOutputs = []oauthTest.HTTPClientOutput{{HTTPClient: httpClient, Error: nil}} - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - }) - - It("returns error when context is missing", func() { - ctx = nil - expectedHTTPClientInputs = []oauthTest.HTTPClientInput{{Context: nil, TokenSourceSource: tokenSourceSource}} - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(MatchError("context is missing")) - Expect(server.ReceivedRequests()).To(BeEmpty()) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) }) It("returns error when method is missing", func() { - Expect(clnt.SendOAuthRequest(ctx, "", url, mutators, requestBody, responseBody, nil, tokenSource)).To(MatchError("method is missing")) + Expect(clnt.SendOAuthRequest(ctx, "", url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(MatchError("method is missing")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when url is missing", func() { - Expect(clnt.SendOAuthRequest(ctx, method, "", mutators, requestBody, responseBody, nil, tokenSource)).To(MatchError("url is missing")) + Expect(clnt.SendOAuthRequest(ctx, method, "", mutators, requestBody, responseBody, nil, mockTokenSource)).To(MatchError("url is missing")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when the request object cannot be encoded", func() { invalidRequestBody := struct{ Func interface{} }{func() {}} - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, invalidRequestBody, responseBody, nil, tokenSource).Error()).To(MatchRegexp("unable to serialize request to .*; json: unsupported type: func()")) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, invalidRequestBody, responseBody, nil, mockTokenSource).Error()).To(MatchRegexp("unable to serialize request to .*; json: unsupported type: func()")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when mutator returns an error", func() { errorMutator := request.NewHeaderMutator("", "") invalidMutators := []request.RequestMutator{headerMutator, errorMutator, parameterMutator} - Expect(clnt.SendOAuthRequest(ctx, method, url, invalidMutators, requestBody, responseBody, nil, tokenSource).Error()).To(MatchRegexp("unable to mutate request to .*; key is missing")) + Expect(clnt.SendOAuthRequest(ctx, method, url, invalidMutators, requestBody, responseBody, nil, mockTokenSource).Error()).To(MatchRegexp("unable to mutate request to .*; key is missing")) Expect(server.ReceivedRequests()).To(BeEmpty()) }) It("returns error when the server is not reachable", func() { server.Close() server = nil - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource).Error()).To(MatchRegexp("unable to perform request to .*: connect: connection refused")) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource).Error()).To(MatchRegexp("unable to perform request to .*: connect: connection refused")) }) Context("with a successful response and no request body", func() { @@ -295,7 +282,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, nil, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, nil, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(Equal(responseString)) @@ -317,7 +304,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorBadRequest()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -341,7 +328,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorBadRequest()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -349,9 +336,9 @@ var _ = Describe("Client", func() { Context("with an unauthorized response 401", func() { BeforeEach(func() { - tokenSource.HTTPClientOutputs = append(tokenSource.HTTPClientOutputs, oauthTest.HTTPClientOutput{HTTPClient: httpClient, Error: nil}) - tokenSource.UpdateTokenOutputs = append(tokenSource.UpdateTokenOutputs, oauthTest.UpdateTokenOutput{Updated: true, Error: nil}) - tokenSource.ExpireTokenOutputs = append(tokenSource.ExpireTokenOutputs, oauthTest.ExpireTokenOutput{Expired: true, Error: nil}) + mockTokenSource.EXPECT().HTTPClient(gomock.Not(gomock.Nil()), gomock.Eq(mockTokenSourceSource)).Return(httpClient, nil) + mockTokenSource.EXPECT().UpdateToken(gomock.Not(gomock.Nil())).Return(true, nil) + mockTokenSource.EXPECT().ExpireToken(gomock.Not(gomock.Nil())).Return(true, nil) server.AppendHandlers( CombineHandlers( VerifyRequest(method, path, fmt.Sprintf("%s=%s", parameterMutator.Key, parameterMutator.Value)), @@ -370,11 +357,10 @@ var _ = Describe("Client", func() { RespondWith(http.StatusUnauthorized, "NOT JSON", responseHeaders), ), ) - expectedHTTPClientInputs = []oauthTest.HTTPClientInput{{Context: ctx, TokenSourceSource: tokenSourceSource}, {Context: ctx, TokenSourceSource: tokenSourceSource}} }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorUnauthenticated()) Expect(server.ReceivedRequests()).To(HaveLen(2)) }) @@ -395,7 +381,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorUnauthorized()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -416,7 +402,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorResourceNotFound()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -440,7 +426,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorResourceNotFound()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -461,7 +447,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorTooManyRequests()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -482,7 +468,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) Expect(err).To(MatchError(fmt.Sprintf(`unexpected response status code 500 from %s "%s?%s=%s"`, method, url, parameterMutator.Key, parameterMutator.Value))) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -510,7 +496,7 @@ var _ = Describe("Client", func() { Expect(err).ToNot(HaveOccurred()) Expect(req).ToNot(BeNil()) res := &http.Response{StatusCode: http.StatusInternalServerError} - err = clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err = clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorUnexpectedResponse(res, req)) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -531,7 +517,7 @@ var _ = Describe("Client", func() { }) It("returns an error", func() { - err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource) + err := clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource) errorsTest.ExpectEqual(err, request.ErrorJSONMalformed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) @@ -552,7 +538,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(BeEmpty()) @@ -574,7 +560,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(BeEmpty()) @@ -595,7 +581,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, nil, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, nil, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(Equal(responseString)) @@ -616,7 +602,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, strings.NewReader(requestString), responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, strings.NewReader(requestString), responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(Equal(responseString)) @@ -638,7 +624,7 @@ var _ = Describe("Client", func() { }) It("returns success", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, responseBody, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) Expect(responseBody).ToNot(BeNil()) Expect(responseBody.Response).To(Equal(responseString)) @@ -660,7 +646,7 @@ var _ = Describe("Client", func() { }) It("returns success without parsing response body", func() { - Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, nil, nil, tokenSource)).To(Succeed()) + Expect(clnt.SendOAuthRequest(ctx, method, url, mutators, requestBody, nil, nil, mockTokenSource)).To(Succeed()) Expect(server.ReceivedRequests()).To(HaveLen(1)) }) }) diff --git a/oauth/provider/client/client.go b/oauth/provider/client/client.go index 14ad91571a..8aa13ece97 100644 --- a/oauth/provider/client/client.go +++ b/oauth/provider/client/client.go @@ -28,11 +28,11 @@ func NewWithErrorParser(name string, config *Config, httpClient *http.Client, jw return nil, errors.Wrap(err, "config is invalid") } - prvdr, err := oauthProvider.New(name, config.ProviderConfig, httpClient, jwks) + prvdr, err := oauthProvider.New(name, config.ProviderConfig, jwks) if err != nil { return nil, err } - clnt, err := oauthClient.NewWithErrorParser(config.ClientConfig, prvdr, errorResponseParser) + clnt, err := oauthClient.NewWithErrorParser(config.ClientConfig, httpClient, prvdr, errorResponseParser) if err != nil { return nil, err } diff --git a/oauth/provider/provider.go b/oauth/provider/provider.go index 42ed36a78d..117eb2a8eb 100644 --- a/oauth/provider/provider.go +++ b/oauth/provider/provider.go @@ -3,7 +3,6 @@ package provider import ( "context" "fmt" - "net/http" "github.com/golang-jwt/jwt/v4" "github.com/lestrrat-go/jwx/v2/jwk" @@ -19,12 +18,11 @@ import ( type Provider struct { name string config Config - httpClient *http.Client jwks jwk.Set oauth2Config *oauth2.Config } -func New(name string, config *Config, httpClient *http.Client, jwks jwk.Set) (*Provider, error) { +func New(name string, config *Config, jwks jwk.Set) (*Provider, error) { if name == "" { return nil, errors.New("name is missing") } @@ -33,9 +31,6 @@ func New(name string, config *Config, httpClient *http.Client, jwks jwk.Set) (*P } else if err := config.Validate(); err != nil { return nil, errors.Wrap(err, "config is invalid") } - if httpClient == nil { - return nil, errors.New("http client is missing") - } oauth2Config := &oauth2.Config{ ClientID: config.ClientID, @@ -54,7 +49,6 @@ func New(name string, config *Config, httpClient *http.Client, jwks jwk.Set) (*P return &Provider{ name: name, config: *config, - httpClient: httpClient, jwks: jwks, oauth2Config: oauth2Config, }, nil @@ -119,7 +113,7 @@ func (p *Provider) TokenSource(ctx context.Context, token *auth.OAuthToken) (oau return nil, errors.New("token is missing") } - tknSrc := p.oauth2Config.TokenSource(context.WithValue(ctx, oauth2.HTTPClient, p.httpClient), token.RawToken()) + tknSrc := p.oauth2Config.TokenSource(ctx, token.RawToken()) if tknSrc == nil { return nil, errors.New("unable to create token source") } diff --git a/oauth/test/token_source.go b/oauth/test/token_source.go deleted file mode 100644 index 9223340bf7..0000000000 --- a/oauth/test/token_source.go +++ /dev/null @@ -1,113 +0,0 @@ -package test - -import ( - "context" - "net/http" - - "github.com/tidepool-org/platform/oauth" -) - -type HTTPClientInput struct { - Context context.Context - TokenSourceSource oauth.TokenSourceSource -} - -type HTTPClientOutput struct { - HTTPClient *http.Client - Error error -} - -type UpdateTokenOutput struct { - Updated bool - Error error -} - -type ExpireTokenOutput struct { - Expired bool - Error error -} - -type TokenSource struct { - HTTPClientInvocations int - HTTPClientInputs []HTTPClientInput - HTTPClientStub func(ctx context.Context, tokenSourceSource oauth.TokenSourceSource) (*http.Client, error) - HTTPClientOutputs []HTTPClientOutput - HTTPClientOutput *HTTPClientOutput - UpdateTokenInvocations int - UpdateTokenInputs []context.Context - UpdateTokenStub func(ctx context.Context) (bool, error) - UpdateTokenOutputs []UpdateTokenOutput - UpdateTokenOutput *UpdateTokenOutput - ExpireTokenInvocations int - ExpireTokenInputs []context.Context - ExpireTokenStub func(ctx context.Context) (bool, error) - ExpireTokenOutputs []ExpireTokenOutput - ExpireTokenOutput *ExpireTokenOutput -} - -func NewTokenSource() *TokenSource { - return &TokenSource{} -} - -func (t *TokenSource) HTTPClient(ctx context.Context, tokenSourceSource oauth.TokenSourceSource) (*http.Client, error) { - t.HTTPClientInvocations++ - t.HTTPClientInputs = append(t.HTTPClientInputs, HTTPClientInput{Context: ctx, TokenSourceSource: tokenSourceSource}) - if t.HTTPClientStub != nil { - return t.HTTPClientStub(ctx, tokenSourceSource) - } - if len(t.HTTPClientOutputs) > 0 { - output := t.HTTPClientOutputs[0] - t.HTTPClientOutputs = t.HTTPClientOutputs[1:] - return output.HTTPClient, output.Error - } - if t.HTTPClientOutput != nil { - return t.HTTPClientOutput.HTTPClient, t.HTTPClientOutput.Error - } - panic("HTTPClient has no output") -} - -func (t *TokenSource) UpdateToken(ctx context.Context) (bool, error) { - t.UpdateTokenInvocations++ - t.UpdateTokenInputs = append(t.UpdateTokenInputs, ctx) - if t.UpdateTokenStub != nil { - return t.UpdateTokenStub(ctx) - } - if len(t.UpdateTokenOutputs) > 0 { - output := t.UpdateTokenOutputs[0] - t.UpdateTokenOutputs = t.UpdateTokenOutputs[1:] - return output.Updated, output.Error - } - if t.UpdateTokenOutput != nil { - return t.UpdateTokenOutput.Updated, t.UpdateTokenOutput.Error - } - panic("UpdateToken has no output") -} - -func (t *TokenSource) ExpireToken(ctx context.Context) (bool, error) { - t.ExpireTokenInvocations++ - t.ExpireTokenInputs = append(t.ExpireTokenInputs, ctx) - if t.ExpireTokenStub != nil { - return t.ExpireTokenStub(ctx) - } - if len(t.ExpireTokenOutputs) > 0 { - output := t.ExpireTokenOutputs[0] - t.ExpireTokenOutputs = t.ExpireTokenOutputs[1:] - return output.Expired, output.Error - } - if t.ExpireTokenOutput != nil { - return t.ExpireTokenOutput.Expired, t.ExpireTokenOutput.Error - } - panic("ExpireToken has no output") -} - -func (t *TokenSource) AssertOutputsEmpty() { - if len(t.HTTPClientOutputs) > 0 { - panic("HTTPClientOutputs is not empty") - } - if len(t.UpdateTokenOutputs) > 0 { - panic("UpdateTokenOutputs is not empty") - } - if len(t.ExpireTokenOutputs) > 0 { - panic("ExpireTokenOutputs is not empty") - } -} diff --git a/oauth/test/token_source_source.go b/oauth/test/token_source_source.go deleted file mode 100644 index 2ae4f4d6b6..0000000000 --- a/oauth/test/token_source_source.go +++ /dev/null @@ -1,54 +0,0 @@ -package test - -import ( - "context" - - "golang.org/x/oauth2" - - "github.com/tidepool-org/platform/auth" -) - -type TokenSourceInput struct { - Context context.Context - Token *auth.OAuthToken -} - -type TokenSourceOutput struct { - TokenSource oauth2.TokenSource - Error error -} - -type TokenSourceSource struct { - TokenSourceInvocations int - TokenSourceInputs []TokenSourceInput - TokenSourceStub func(ctx context.Context, token *auth.OAuthToken) (oauth2.TokenSource, error) - TokenSourceOutputs []TokenSourceOutput - TokenSourceOutput *TokenSourceOutput -} - -func NewTokenSourceSource() *TokenSourceSource { - return &TokenSourceSource{} -} - -func (t *TokenSourceSource) TokenSource(ctx context.Context, token *auth.OAuthToken) (oauth2.TokenSource, error) { - t.TokenSourceInvocations++ - t.TokenSourceInputs = append(t.TokenSourceInputs, TokenSourceInput{Context: ctx, Token: token}) - if t.TokenSourceStub != nil { - return t.TokenSourceStub(ctx, token) - } - if len(t.TokenSourceOutputs) > 0 { - output := t.TokenSourceOutputs[0] - t.TokenSourceOutputs = t.TokenSourceOutputs[1:] - return output.TokenSource, output.Error - } - if t.TokenSourceOutput != nil { - return t.TokenSourceOutput.TokenSource, t.TokenSourceOutput.Error - } - panic("TokenSource has no output") -} - -func (t *TokenSourceSource) AssertOutputsEmpty() { - if len(t.TokenSourceOutputs) > 0 { - panic("TokenSourceOutputs is not empty") - } -} diff --git a/oura/client/client_test.go b/oura/client/client_test.go index 957bbaf50b..ea2520a995 100644 --- a/oura/client/client_test.go +++ b/oura/client/client_test.go @@ -61,7 +61,7 @@ var _ = Describe("client", func() { mockTokenSourceSource = oauthTest.NewMockTokenSourceSource(mockController) mockProvider = ouraClientTest.NewMockProvider(mockController) server = NewServer() - baseClient, err = oauthClient.NewWithErrorParser(&client.Config{Address: server.URL()}, mockTokenSourceSource, &ouraClient.ErrorResponseParser{}) + baseClient, err = oauthClient.NewWithErrorParser(&client.Config{Address: server.URL()}, nil, mockTokenSourceSource, &ouraClient.ErrorResponseParser{}) Expect(err).ToNot(HaveOccurred()) Expect(baseClient).ToNot(BeNil()) }) diff --git a/oura/provider/provider.go b/oura/provider/provider.go index 2e1920a97d..b04f99ff48 100644 --- a/oura/provider/provider.go +++ b/oura/provider/provider.go @@ -67,12 +67,15 @@ func New(dependencies Dependencies) (*Provider, error) { return nil, errors.Wrap(err, "dependencies is invalid") } - // Create http client + if dependencies.Config.ClientConfig.Timeout == 0 { + dependencies.Config.ClientConfig.Timeout = 1 * time.Minute + } + httpClient := &http.Client{ Transport: prometheusRequestMetricsRoundTripper, CheckRedirect: http.DefaultClient.CheckRedirect, Jar: http.DefaultClient.Jar, - Timeout: 2 * time.Minute, + Timeout: http.DefaultClient.Timeout, } oauthProviderClient, err := oauthProviderClient.NewWithErrorParser(oura.ProviderName, dependencies.Config.Config, httpClient, nil, &ouraClient.ErrorResponseParser{}) diff --git a/private/plugin/abbott b/private/plugin/abbott index cd4a03ffc7..749f8eb780 160000 --- a/private/plugin/abbott +++ b/private/plugin/abbott @@ -1 +1 @@ -Subproject commit cd4a03ffc7812e92d6c7c9a2b2e54aa220ce1371 +Subproject commit 749f8eb7805e4f88b931a788a60057f65ee0aa60 diff --git a/summary/task/migrationrunner.go b/summary/task/migrationrunner.go index ea4b37f62d..8ff2ca122c 100644 --- a/summary/task/migrationrunner.go +++ b/summary/task/migrationrunner.go @@ -169,17 +169,17 @@ func (t *MigrationTaskRunner) run() error { pagination.Size = t.GetBatch() typ := t.summaryType - t.logger.Infof("Searching for User %s Summaries requiring Migration", typ) + t.logger.Debugf("Searching for User %s Summaries requiring Migration", typ) outdatedUserIds, err := t.dataClient.GetMigratableUserIDs(t.context, typ, pagination) if err != nil { return err } if len(outdatedUserIds) == 0 { - t.logger.Infof("No %s Summaries requiring migrations found", typ) + t.logger.Debugf("No %s Summaries requiring migrations found", typ) return nil } - t.logger.Infof("Found batch of %d %s Summaries to Migrate", len(outdatedUserIds), typ) + t.logger.Debugf("Found batch of %d %s Summaries to Migrate", len(outdatedUserIds), typ) t.logger.Debugf("Starting User %s Summary Migration", typ) err = updateSummaries(t.context, t.logger, t.dataClient, typ, outdatedUserIds, MigrationWorkerCount, t.deadline, "Migrating") diff --git a/summary/task/updaterunner.go b/summary/task/updaterunner.go index c923282dae..3ceaecdc76 100644 --- a/summary/task/updaterunner.go +++ b/summary/task/updaterunner.go @@ -171,17 +171,17 @@ func (t *UpdateTaskRunner) run() error { t.logger.Debugf("Starting User %s Summary Update", typ) for i := 1; i <= IterLimit; i++ { - t.logger.Infof("Searching for User %s Summaries requiring Update", typ) + t.logger.Debugf("Searching for User %s Summaries requiring Update", typ) outdated, err := t.dataClient.GetOutdatedUserIDs(t.context, typ, pagination) if err != nil { return err } if len(outdated.UserIds) == 0 { - t.logger.Infof("No %s Summaries requiring updates found", typ) + t.logger.Debugf("No %s Summaries requiring updates found", typ) return nil } - t.logger.Infof("Found batch of %d %s Summaries to Update", len(outdated.UserIds), typ) + t.logger.Debugf("Found batch of %d %s Summaries to Update", len(outdated.UserIds), typ) err = updateSummaries(t.context, t.logger, t.dataClient, typ, outdated.UserIds, UpdateWorkerCount, t.deadline, "Updating") if err != nil { diff --git a/task/queue/multi.go b/task/queue/multi.go index bb0deb3aa4..19472f6259 100644 --- a/task/queue/multi.go +++ b/task/queue/multi.go @@ -13,7 +13,7 @@ import ( // that runner's type. Like Queue, it is single-use. The queues map is immutable after // NewMultiQueue, so no synchronization is required. type MultiQueue struct { - queues map[string]Queue + queues map[string]*Queue } func NewMultiQueue(cfg *Config, lgr log.Logger, str store.Store, runners ...Runner) (*MultiQueue, error) { @@ -27,7 +27,7 @@ func NewMultiQueue(cfg *Config, lgr log.Logger, str store.Store, runners ...Runn return nil, errors.New("store is missing") } - queues := make(map[string]Queue, len(runners)) + queues := make(map[string]*Queue, len(runners)) for _, runner := range runners { if runner == nil { return nil, errors.New("runner is missing") @@ -69,8 +69,6 @@ func (m *MultiQueue) Stop() { // GetQueues returns a copy of the type-to-queue map so callers cannot mutate the // internal map. The queue values are shared references, not copies. -func (m *MultiQueue) GetQueues() map[string]Queue { +func (m *MultiQueue) GetQueues() map[string]*Queue { return maps.Clone(m.queues) } - -var _ Queue = &MultiQueue{} diff --git a/task/queue/queue.go b/task/queue/queue.go index 1ce042e42c..80d8b45180 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -20,47 +20,10 @@ import ( taskStore "github.com/tidepool-org/platform/task/store" ) -var ( - // RunnerTimeoutExceededTotal counts task runs that exceeded the runner timeout without - // returning, sorted by type. A non-zero value indicates a runner that does not honor context - // cancellation, which leaves its worker blocked until the process restarts. - RunnerTimeoutExceededTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tidepool_task_runner_timeout_exceeded_total", - Help: "The total number of task runs that exceeded the runner timeout without returning, sorted by type", - }, []string{"type"}) - - // RunDurationSeconds observes how long each task run took, sorted by type. Use it to track - // run latency percentiles and to alert when durations approach the runner timeout. - RunDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "tidepool_task_run_duration_seconds", - Help: "The duration of task runs in seconds, sorted by type", - Buckets: prometheus.ExponentialBuckets(0.1, 2, 15), - }, []string{"type"}) - - // RunPanicTotal counts task runs that panicked and were recovered, sorted by type. - RunPanicTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tidepool_task_run_panic_total", - Help: "The total number of task runs that panicked, sorted by type", - }, []string{"type"}) - - // WorkersAvailable reports the number of idle workers per queue. A value pinned at zero - // indicates a saturated queue or workers wedged in non-cooperative runners. - WorkersAvailable = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "tidepool_task_workers_available", - Help: "The number of idle task queue workers, sorted by queue", - }, []string{"queue"}) - - // WorkersTotal reports the configured number of workers per queue. - WorkersTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "tidepool_task_workers_total", - Help: "The configured number of task queue workers, sorted by queue", - }, []string{"queue"}) -) - const ( WorkersDefault = 5 - DelayDefault = time.Minute - DelayInitialDefault = time.Minute + DelayDefault = 1 * time.Minute + DelayInitialDefault = 1 * time.Minute DelayUnstickDefault = 5 * time.Minute // StopWaitTimeoutDefault bounds how long Stop waits for in-flight tasks to observe @@ -74,7 +37,7 @@ const ( // TaskDeadlineDefault bounds how long a task is allowed to run before being forcefully // reset if a runner for the task type is not registered. - TaskDeadlineDefault = time.Minute + TaskDeadlineDefault = 1 * time.Minute // RunnerWatchdogGracePeriodDefault is the extra time beyond the runner timeout that the // watchdog waits before reporting a runner as blocked. The runner context is still canceled @@ -83,6 +46,11 @@ const ( RunnerWatchdogGracePeriodDefault = 5 * time.Second ) +// ErrRunnerTimeout is the cancellation cause set on the Run context when a run exceeds the +// runner timeout. A runner distinguishes a timeout from a shutdown with +// errors.Is(context.Cause(ctx), ErrRunnerTimeout); a shutdown instead cancels with context.Canceled. +var ErrRunnerTimeout = errors.New("task runner timeout exceeded") + type Config struct { Workers int Delay time.Duration @@ -176,49 +144,11 @@ func (c *Config) Validate() error { return nil } -type Runner interface { - // The type of tasks that the runner supports. - GetRunnerType() string - - // The duration after which the task manager will forcefully reset the task back to pending - // and available. This is a duration significantly longer than the task duration maximum. Normally, - // this would only be used on a task that is in the running state even though it is not running - // (likely due to a system crash or interruption). Must exceed the runner timeout, or a task - // could be unstuck and re-claimed while the original run is still executing. - GetRunnerDeadline() time.Duration - - // The duration of a task where the task manager will forcefully cancel the task context to interrupt - // the task and force completion. This is typically a duration somewhat longer than the task - // duration maximum. Must exceed the runner duration maximum. - GetRunnerTimeout() time.Duration - - // The typical duration maximum of the task after which a warning will be displayed. - // Must be positive. - GetRunnerDurationMaximum() time.Duration - - // Execute the specified task within the specified context. The context will be forcefully - // canceled after a duration specified by GetRunnerTimeout. Before returning, the runner - // must move the task out of the running state (typically pending via RepeatAvailableAt or - // RepeatAvailableAfter to run again, or completed, or failed); a task left running when - // the runner returns is marked failed, unless the run was interrupted by queue shutdown, - // in which case it is reverted to pending. - Run(ctx context.Context, tsk *task.Task) -} - -// Queue runs tasks via runners provided at construction. A queue is single-use: it may be -// started at most once and stopped at most once, and once stopped it cannot be restarted -// (create a new queue instead). Start after Stop, a second Start, and a second Stop are -// all no-ops. -type Queue interface { - Start() - Stop() -} - -// The queue's fields are all immutable after New, except the lifecycle fields, which are +// The Queue's fields are all immutable after New, except the lifecycle fields, which are // guarded by the lifecycle mutex, and workersAvailable, which is owned exclusively by the // manager goroutine. The workers and manager therefore read the channels and runners map // freely, without synchronization. -type queue struct { +type Queue struct { name string config *Config logger log.Logger @@ -235,7 +165,7 @@ type queue struct { workersAvailable int } -func New(name string, cfg *Config, lgr log.Logger, str taskStore.Store, runners ...Runner) (Queue, error) { +func New(name string, cfg *Config, lgr log.Logger, str taskStore.Store, runners ...Runner) (*Queue, error) { if name == "" { return nil, errors.New("name is missing") } @@ -267,7 +197,7 @@ func New(name string, cfg *Config, lgr log.Logger, str taskStore.Store, runners runnerMap[runner.GetRunnerType()] = runner } - return &queue{ + return &Queue{ name: name, config: cfg, logger: lgr.WithField("queue", name), @@ -283,7 +213,7 @@ func New(name string, cfg *Config, lgr log.Logger, str taskStore.Store, runners }, nil } -func (q *queue) Start() { +func (q *Queue) Start() { q.lifecycleMutex.Lock() defer q.lifecycleMutex.Unlock() @@ -292,14 +222,18 @@ func (q *queue) Start() { } q.started = true + q.logger.Info("Task queue starting") + ctx, cancelFunc := context.WithCancel(log.NewContextWithLogger(context.Background(), q.logger)) q.cancelFunc = cancelFunc q.startWorkers(ctx) q.startManager(ctx) + + q.logger.Info("Task queue started") } -func (q *queue) Stop() { +func (q *Queue) Stop() { // Hold the mutex for the entire stop, including the waits, so a concurrent Start cannot // observe a partially stopped queue. q.lifecycleMutex.Lock() @@ -315,6 +249,8 @@ func (q *queue) Stop() { return } + q.logger.Info("Task queue stopping") + lgr := q.logger.WithField("stopWaitTimeout", q.config.StopWaitTimeout) // Cancel the manager, so it stops dispatching new tasks and begins draining completions @@ -327,7 +263,7 @@ func (q *queue) Stop() { // a closed completion channel. Instead we leave the goroutines orphaned (reaped at process // exit); the abandoned task stays running and is recovered by the deadline/unstick mechanism. if !waitWithTimeout(&q.workersWaitGroup, q.config.StopWaitTimeout) { - lgr.Error("Task queue workers did not stop within timeout; abandoning in-flight tasks") + lgr.Error("Task queue workers did not stop within timeout; abandoning in-flight tasks; will be fixed with UnstickTasks later") return } @@ -345,9 +281,11 @@ func (q *queue) Stop() { // dispatch channel is not buffered, no dispatched task can be stranded in it; any // task the manager could not hand off was reverted to pending during dispatch. close(q.dispatchChannel) + + q.logger.Info("Task queue stopped") } -func (q *queue) startWorkers(ctx context.Context) { +func (q *Queue) startWorkers(ctx context.Context) { for q.workersAvailable = 0; q.workersAvailable < q.config.Workers; q.workersAvailable++ { q.startWorker(log.ContextWithField(ctx, "worker", q.workersAvailable)) } @@ -355,7 +293,7 @@ func (q *queue) startWorkers(ctx context.Context) { WorkersTotal.WithLabelValues(q.name).Set(float64(q.config.Workers)) } -func (q *queue) startWorker(ctx context.Context) { +func (q *Queue) startWorker(ctx context.Context) { q.workersWaitGroup.Go(func() { lgr := log.LoggerFromContext(ctx) @@ -370,7 +308,7 @@ func (q *queue) startWorker(ctx context.Context) { }) } -func (q *queue) executeWorker(ctx context.Context) error { +func (q *Queue) executeWorker(ctx context.Context) error { select { case <-ctx.Done(): return context.Cause(ctx) @@ -390,16 +328,19 @@ func (q *queue) executeWorker(ctx context.Context) error { return nil } -func (q *queue) runTask(ctx context.Context, tsk *task.Task) { +func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { + ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) + runner, ok := q.runners[tsk.Type] if !ok { - tsk.AppendError(errors.New("runner not found for task")) - tsk.SetFailed() + // A whole task type is unprocessable by this queue; surface it distinctly (a configuration + // problem) rather than letting it blend in with ordinary per-task failures downstream. + lgr.Error("Runner not found for task; task cannot be processed") + tsk.SetFailedWithError(errors.New("runner not found for task")) + RunnerNotFoundTotal.WithLabelValues(tsk.Type).Inc() return } - ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) - defer func() { if err := recover(); err != nil { lgr.WithFields(log.Fields{"error": err, "stack": string(debug.Stack())}).Error("Unhandled panic while running task") @@ -412,25 +353,28 @@ func (q *queue) runTask(ctx context.Context, tsk *task.Task) { // If runner does not respect its own maximum duration, then enforce a context-based timeout. // This forces the task to cancel via the context. - runnerContext, cancel := context.WithTimeoutCause(ctx, runner.GetRunnerTimeout(), errors.New("task runner timeout exceeded")) + runnerContext, cancel := context.WithTimeoutCause(ctx, runner.GetRunnerTimeout(), ErrRunnerTimeout) defer cancel() // Watchdog for a runner that ignores cancellation. Go cannot preempt a goroutine, so if the // runner blows past its timeout without returning, this worker stays blocked until the // process restarts (the task itself is recovered by the deadline/unstick mechanism). We // cannot unblock the worker, but we surface the condition via a log and metric so a - // non-cooperative runner is detectable rather than silent. The grace period keeps a - // cooperative runner that returns promptly after the timeout cancellation from being - // falsely reported. + // non-cooperative runner is detectable rather than silent. The grace period lets a cooperative + // runner that returns promptly after the timeout cancellation avoid being reported as blocked + // here; such a run is instead logged and counted as "recovered" during reconciliation. runnerWatchdog := time.AfterFunc(runner.GetRunnerTimeout()+q.config.RunnerWatchdogGracePeriod, func() { lgr.Error("Task runner exceeded timeout without returning; worker is blocked until it returns") - RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType()).Inc() + RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType(), "blocked").Inc() }) defer runnerWatchdog.Stop() // Run the task via the runner runner.Run(runnerContext, tsk) + // Immediate stop the runner watchdog + runnerWatchdog.Stop() + // If the runner left the task running, reconcile its state based on why the run ended. if tsk.State == task.TaskStateRunning { switch { @@ -439,9 +383,11 @@ func (q *queue) runTask(ctx context.Context, tsk *task.Task) { // again for retry rather than treating the interruption as a completion. tsk.RepeatAvailableAfter(0) case context.Cause(runnerContext) != nil: - // The runner exceeded its timeout; record the cause so the failure is attributed - // to the timeout rather than the generic missing terminal state error. + // The runner exceeded its timeout but returned; record the cause so the failure is + // attributed to the timeout rather than the generic missing terminal state error. + lgr.Warn("Task runner exceeded timeout; task will be failed") tsk.AppendError(context.Cause(runnerContext)) + RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType(), "recovered").Inc() } } @@ -452,11 +398,13 @@ func (q *queue) runTask(ctx context.Context, tsk *task.Task) { } } -func (q *queue) startManager(ctx context.Context) { +func (q *Queue) startManager(ctx context.Context) { q.managerWaitGroup.Go(func() { lgr := log.LoggerFromContext(ctx) - lgr.Info("Task queue manager started") + lgr.Debug("Task queue manager started") + + lgr.Debug("Task queue manager initial delay initiated") // Start at a random future time to help prevent thundering herd problem select { @@ -497,7 +445,7 @@ func (q *queue) startManager(ctx context.Context) { }) } -func (q *queue) executeManager(ctx context.Context) error { +func (q *Queue) executeManager(ctx context.Context) error { select { case <-ctx.Done(): return context.Cause(ctx) @@ -521,7 +469,7 @@ func (q *queue) executeManager(ctx context.Context) error { return nil } -func (q *queue) unstickTasks(ctx context.Context) { +func (q *Queue) unstickTasks(ctx context.Context) { ids, err := q.repository.UnstickTasks(ctx) if count := len(ids); count > 0 { log.LoggerFromContext(ctx).WithFields(log.Fields{"count": count, "ids": ids}).Info("Unstuck tasks") @@ -535,7 +483,7 @@ func (q *queue) unstickTasks(ctx context.Context) { } } -func (q *queue) dispatchTasks(ctx context.Context) { +func (q *Queue) dispatchTasks(ctx context.Context) { if q.workersAvailable < 1 { return } @@ -571,7 +519,7 @@ func (q *queue) dispatchTasks(ctx context.Context) { } } -func (q *queue) dispatchTask(ctx context.Context, tsk *task.Task) error { +func (q *Queue) dispatchTask(ctx context.Context, tsk *task.Task) error { ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) // we don't error here if missing, as the task will be failed during runTask, which persists error to database @@ -611,11 +559,9 @@ func (q *queue) dispatchTask(ctx context.Context, tsk *task.Task) error { // completeTask persists the task's completion. It deliberately does not touch // workersAvailable (the caller accounts for the freed worker where relevant), so it is safe // to call concurrently during the manager's shutdown drain. -func (q *queue) completeTask(ctx context.Context, tsk *task.Task) { +func (q *Queue) completeTask(ctx context.Context, tsk *task.Task) { ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) - q.computeState(ctx, tsk) - var duration *time.Duration if tsk.RunTime != nil { // Clamp to a zero minimum: the run time round-trips through the database without a @@ -624,8 +570,16 @@ func (q *queue) completeTask(ctx context.Context, tsk *task.Task) { duration = pointer.From(max(time.Since(*tsk.RunTime), 0)) } - if tsk.HasError() { - lgr.Error("Error occurred while running task") + q.computeState(ctx, tsk) + + // computeState has already settled the terminal state. A failed task is a genuine problem + // worth an error; a task that errored but will run again (e.g. reverted to pending for + // retry) is an expected, recoverable outcome, so log it at warning to avoid error-level + // noise from routine retries. + if err := tsk.GetError(); tsk.State == task.TaskStateFailed { + lgr.WithError(err).Error("Task failed while running") + } else if err != nil { + lgr.WithError(err).Warn("Error occurred while running task that did not fail") } // Data and Error use non-nil wrappers so that a task whose data or error was cleared during @@ -641,7 +595,7 @@ func (q *queue) completeTask(ctx context.Context, tsk *task.Task) { } } -func (q *queue) computeState(ctx context.Context, tsk *task.Task) { +func (q *Queue) computeState(ctx context.Context, tsk *task.Task) { switch tsk.State { case task.TaskStatePending: now := time.Now().UTC() @@ -657,15 +611,15 @@ func (q *queue) computeState(ctx context.Context, tsk *task.Task) { case task.TaskStateRunning: // The runner returned without moving the task out of the running state, violating // the runner contract; fail the task rather than guessing whether it succeeded. - if !tsk.HasError() { - tsk.AppendError(errors.New("runner failed to set terminal task state")) + if tsk.HasError() { + tsk.SetFailed() + } else { + tsk.SetFailedWithError(errors.New("runner failed to set state")) } - tsk.SetFailed() case task.TaskStateFailed, task.TaskStateCompleted: tsk.AvailableTime = nil default: - tsk.AppendError(errors.New("unknown task state")) - tsk.SetFailed() + tsk.SetFailedWithError(errors.New("unknown task state")) } } @@ -718,3 +672,51 @@ func durationWithJitter(duration time.Duration) time.Duration { jitter := time.Duration(float64(duration) * DurationJitterFactor) return duration + (randomDuration(jitter*2) - jitter) } + +var ( + // RunnerTimeoutExceededTotal counts task runs that exceeded the runner timeout, sorted by type + // and disposition: + // - "blocked" - for runs the watchdog caught still running past the grace period (the worker is + // wedged until the runner eventually returns or the process restarts, so this is the severe case) + // - "recovered" - for runs that exceeded the timeout, but returned once their context was + // canceled (the timeout mechanism worked as designed) + RunnerTimeoutExceededTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_runner_timeout_exceeded_total", + Help: "The total number of task runs that exceeded the runner timeout, sorted by type and disposition", + }, []string{"type", "disposition"}) + + // RunnerNotFoundTotal counts task runs for which no runner is registered for the task's type, + // sorted by type. A non-zero value indicates pending tasks of a type this queue cannot process + // (a configuration problem); such tasks are failed immediately. + RunnerNotFoundTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_runner_not_found_total", + Help: "The total number of task runs with no registered runner for the task type, sorted by type", + }, []string{"type"}) + + // RunDurationSeconds observes how long each task run took, sorted by type. Use it to track + // run latency percentiles and to alert when durations approach the runner timeout. + RunDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tidepool_task_run_duration_seconds", + Help: "The duration of task runs in seconds, sorted by type", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 15), + }, []string{"type"}) + + // RunPanicTotal counts task runs that panicked and were recovered, sorted by type. + RunPanicTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_run_panic_total", + Help: "The total number of task runs that panicked, sorted by type", + }, []string{"type"}) + + // WorkersAvailable reports the number of idle workers per queue. A value pinned at zero + // indicates a saturated queue or workers wedged in non-cooperative runners. + WorkersAvailable = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_task_workers_available", + Help: "The number of idle task queue workers, sorted by queue", + }, []string{"queue"}) + + // WorkersTotal reports the configured number of workers per queue. + WorkersTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_task_workers_total", + Help: "The configured number of task queue workers, sorted by queue", + }, []string{"queue"}) +) diff --git a/task/queue/queue_internal_test.go b/task/queue/queue_internal_test.go index e59ebadffe..f8b044d78c 100644 --- a/task/queue/queue_internal_test.go +++ b/task/queue/queue_internal_test.go @@ -24,7 +24,7 @@ var _ = Describe("Queue", func() { lgr := logTest.NewLogger() ctx := log.NewContextWithLogger(context.Background(), lgr) cfg := NewConfig() - que := &queue{ + que := &Queue{ name: taskTest.RandomType(), config: cfg, logger: lgr, @@ -39,7 +39,7 @@ var _ = Describe("Queue", func() { lgr := logTest.NewLogger() ctx := log.NewContextWithLogger(context.Background(), lgr) cfg := NewConfig() - que := &queue{ + que := &Queue{ name: taskTest.RandomType(), config: cfg, logger: lgr, @@ -54,13 +54,13 @@ var _ = Describe("Queue", func() { Context("computeState", func() { It("clears the available time for a completed task", func() { completedTask := &task.Task{State: task.TaskStateCompleted, AvailableTime: pointer.FromTime(test.RandomTimeBeforeNow())} - (&queue{}).computeState(context.Background(), completedTask) + (&Queue{}).computeState(context.Background(), completedTask) Expect(completedTask.AvailableTime).To(BeNil()) }) It("clears the available time for a failed task", func() { failedTask := &task.Task{State: task.TaskStateFailed, AvailableTime: pointer.FromTime(test.RandomTimeBeforeNow())} - (&queue{}).computeState(context.Background(), failedTask) + (&Queue{}).computeState(context.Background(), failedTask) Expect(failedTask.AvailableTime).To(BeNil()) }) }) diff --git a/task/queue/queue_test.go b/task/queue/queue_test.go index 1acb9e38c0..c67e834552 100644 --- a/task/queue/queue_test.go +++ b/task/queue/queue_test.go @@ -33,11 +33,11 @@ var _ = Describe("Queue", func() { }) It("DelayDefault is expected", func() { - Expect(taskQueue.DelayDefault).To(Equal(time.Minute)) + Expect(taskQueue.DelayDefault).To(Equal(1 * time.Minute)) }) It("DelayInitialDefault is expected", func() { - Expect(taskQueue.DelayInitialDefault).To(Equal(time.Minute)) + Expect(taskQueue.DelayInitialDefault).To(Equal(1 * time.Minute)) }) It("DelayUnstickDefault is expected", func() { @@ -57,7 +57,7 @@ var _ = Describe("Queue", func() { }) It("TaskDeadlineDefault is expected", func() { - Expect(taskQueue.TaskDeadlineDefault).To(Equal(time.Minute)) + Expect(taskQueue.TaskDeadlineDefault).To(Equal(1 * time.Minute)) }) Context("Config", func() { @@ -302,7 +302,7 @@ var _ = Describe("Queue", func() { }) Context("with a new queue", func() { - var que taskQueue.Queue + var que *taskQueue.Queue BeforeEach(func() { cfg := taskQueue.NewConfig() @@ -364,7 +364,7 @@ var _ = Describe("Queue", func() { var countingRunner *taskQueueTest.CountingRunner var panicRunner *taskQueueTest.PanicRunner var cfg *taskQueue.Config - var que taskQueue.Queue + var que *taskQueue.Queue BeforeEach(func() { countingRunner = taskQueueTest.NewCountingRunner(taskTest.RandomType()) @@ -442,7 +442,7 @@ var _ = Describe("Queue", func() { // The completion's compare-and-swap misses, which is logged rather than silently swallowed. Eventually(func() bool { defer func() { _ = recover() }() - lgr.AssertWarn("Unable to stop task; no running task matched the expected condition") + lgr.AssertError("Unable to stop task; no running task matched the expected condition") return true }, "5s", "100ms").To(BeTrue()) @@ -472,7 +472,8 @@ var _ = Describe("Queue", func() { }) It("fails a pending task that does not match any registered runner", func() { - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: taskTest.RandomType()})) + unregisteredType := taskTest.RandomType() + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: unregisteredType})) que.Start() @@ -486,6 +487,9 @@ var _ = Describe("Queue", func() { Expect(actualTask.State).To(Equal(task.TaskStateFailed)) Expect(actualTask.Error).ToNot(BeNil()) Expect(actualTask.Error.Error).To(MatchError("runner not found for task")) + + lgr.AssertError("Runner not found for task; task cannot be processed") + Expect(testutil.ToFloat64(taskQueue.RunnerNotFoundTotal.WithLabelValues(unregisteredType))).To(Equal(float64(1))) }) It("fails a task whose runner leaves it in an unknown state", func() { @@ -630,7 +634,7 @@ var _ = Describe("Queue", func() { actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) Expect(actualTask.State).To(Equal(task.TaskStateFailed)) Expect(actualTask.Error).ToNot(BeNil()) - Expect(actualTask.Error.Error).To(MatchError("runner failed to set terminal task state")) + Expect(actualTask.Error.Error).To(MatchError("runner failed to set state")) }) It("fails a task that exceeds its timeout without setting its own state", func() { @@ -649,6 +653,12 @@ var _ = Describe("Queue", func() { Expect(actualTask.State).To(Equal(task.TaskStateFailed)) Expect(actualTask.Error).ToNot(BeNil()) Expect(actualTask.Error.Error).To(MatchError("task runner timeout exceeded")) + + // The runner returns after its timeout (well within the watchdog grace period), so + // the timeout is logged and counted as "recovered" during reconciliation rather than + // caught as "blocked" by the watchdog. + lgr.AssertWarn("Task runner exceeded timeout; task will be failed") + Expect(testutil.ToFloat64(taskQueue.RunnerTimeoutExceededTotal.WithLabelValues(hangingRunner.GetRunnerType(), "recovered"))).To(Equal(float64(1))) }) It("logs a warning if a task that exceeds its maximum duration", func() { @@ -694,7 +704,7 @@ var _ = Describe("Queue", func() { Context("with a blocking runner", func() { var blockingRunner *taskQueueTest.BlockingRunner - var que taskQueue.Queue + var que *taskQueue.Queue BeforeEach(func() { blockingRunner = taskQueueTest.NewBlockingRunner(taskTest.RandomType(), time.Minute) @@ -723,13 +733,13 @@ var _ = Describe("Queue", func() { }() Eventually(stopped, "5s").Should(BeClosed()) - lgr.AssertError("Task queue workers did not stop within timeout; abandoning in-flight tasks") + lgr.AssertError("Task queue workers did not stop within timeout; abandoning in-flight tasks; will be fixed with UnstickTasks later") }) }) Context("with a runner that exceeds its timeout without returning", func() { var blockingRunner *taskQueueTest.BlockingRunner - var que taskQueue.Queue + var que *taskQueue.Queue BeforeEach(func() { // A short duration maximum yields a short runner timeout (3x), and a short grace @@ -754,9 +764,9 @@ var _ = Describe("Queue", func() { return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State }, "5s", "50ms").To(Equal(task.TaskStateRunning)) - // The watchdog fires after the runner timeout, recording the stuck run. + // The watchdog fires after the runner timeout, recording the stuck run as blocked. Eventually(func() float64 { - return testutil.ToFloat64(taskQueue.RunnerTimeoutExceededTotal.WithLabelValues(runnerType)) + return testutil.ToFloat64(taskQueue.RunnerTimeoutExceededTotal.WithLabelValues(runnerType, "blocked")) }, "5s", "50ms").Should(BeNumerically(">=", float64(1))) lgr.AssertError("Task runner exceeded timeout without returning; worker is blocked until it returns") @@ -777,13 +787,13 @@ var _ = Describe("Queue", func() { const taskCount = 2 * queueCount * workersCount var runner *taskQueueTest.RepeatRunner - var ques []taskQueue.Queue + var ques []*taskQueue.Queue BeforeEach(func() { runner = taskQueueTest.NewRepeatRunner(taskTest.RandomType()) cfg := &taskQueue.Config{Workers: workersCount, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: time.Millisecond, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault} - ques = make([]taskQueue.Queue, queueCount) + ques = make([]*taskQueue.Queue, queueCount) for index := range len(ques) { ques[index] = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) } diff --git a/task/queue/runner.go b/task/queue/runner.go new file mode 100644 index 0000000000..2cf2fb1c4b --- /dev/null +++ b/task/queue/runner.go @@ -0,0 +1,104 @@ +package queue + +import ( + "context" + "time" + + "github.com/tidepool-org/platform/task" +) + +// Runner processes one task type on the queue. This comment is the complete contract; no other +// queue code need be consulted to implement it. +// +// Registration: one Runner per task type, registered at queue construction (duplicate types +// fail construction). The queue owns the single instance for the queue's lifetime. +// +// Concurrency: the queue runs multiple workers (configurable; 5 by default) that share the one Runner instance, +// so Run is called concurrently from multiple goroutines — any state shared across calls must be +// synchronized. Each call's *task.Task is distinct and call-owned: don't retain it past Run or +// touch another call's task. The Get* methods are also called concurrently and must be stable +// (in practice, constants). +// +// Duration contract, validated at construction (violations fail construction): +// +// GetRunnerDeadline() > GetRunnerTimeout() > GetRunnerDurationMaximum() > 0 +// +// Run's obligation: the task arrives running and already claimed under a unique per-run state +// lock — a token the queue set when it claimed the task; a later write-back to the task only lands +// while that same lock is still in place (see the lost-write note under Delivery). Before +// returning, Run MUST move it out of running by calling exactly one terminal helper on tsk: +// +// - tsk.SetCompleted() — success; done. +// - tsk.SetFailed() — permanent failure; done. +// - tsk.SetFailedWithError(e) — permanent failure with error; done. +// - tsk.RepeatAvailableAfter(d) — reschedule after duration d. +// - tsk.RepeatAvailableAt(t) — reschedule at time t. +// +// The task arrives with AvailableTime cleared, so reschedule via the Repeat* helpers (which set +// both pending state and a new available time); a reschedule time in the past is clamped to now +// (run ASAP). Setting pending without an available time is a bug the queue only papers over with +// a warning. If Run returns with the task still running, the +// queue: reverts it to pending if interrupted by shutdown (parent context canceled); else fails +// it, recording the timeout as cause if the context was canceled for timeout, otherwise a +// "runner failed to set state" error. +// +// Context: canceled (with cause) after GetRunnerTimeout and on shutdown — cooperative runners +// select on ctx.Done() and return promptly. To tell the two apart, inspect context.Cause(ctx): a +// timeout cancels with ErrRunnerTimeout (errors.Is(context.Cause(ctx), ErrRunnerTimeout)), a +// shutdown with context.Canceled. The queue cannot preempt a runner that ignores cancellation; it +// stays blocked until it returns or the process restarts, recovered by the deadline/unstick +// mechanism (a periodic sweep that resets tasks still running past their deadline — see +// GetRunnerDeadline). The logger is on the context (log.LoggerFromContext(ctx)). Wrap must-complete writes +// in context.WithoutCancel so cancellation doesn't abandon them. +// +// Delivery is at-least-once, so Run must tolerate re-execution of the same logical task: a runner +// that overruns its deadline is unstuck and re-dispatched (possibly while the original run is still +// in flight), and a crash or kill mid-run re-runs the task from the start. Conversely a final write +// can be lost — if the task was unstuck and re-claimed while Run was still working, the queue's +// write-back on return no longer matches the state lock and is dropped (logged and counted, not +// applied). Design terminal effects to be idempotent and safe to repeat. +// +// Errors: accumulate with tsk.AppendError(err), reset with tsk.ClearError() (commonly at run +// start). Setting an error does not change state — pair it with a terminal helper. Persisted with +// the task; logged at error level if failed, warning if rescheduled. A panic is recovered and an +// "unhandled panic" error attached; the task is then failed unless the runner had already moved it +// out of running before panicking (e.g. a task completed before the panic stays completed, with +// the error attached) or a concurrent shutdown reverts it to pending instead. +// +// Persistence: on return the queue persists only the terminal state, the available time (written +// only when rescheduling; a terminal task's available time was already cleared when the task was +// claimed), tsk.Data (mutate the map in place, or assign a new map if it arrived nil; the map +// is written as a whole-field replacement, so its final contents are exactly what is stored — keys +// dropped from the map are gone, and a nil map unsets the field), and the accumulated error — +// mutate only these. Run time and duration are recorded by the queue; other fields are +// queue-managed, don't set them. To persist Data mid-run, mutate tsk +// and let the queue write it back on return; a Runner that instead updates its OWN task via +// task.Client must replace tsk with the returned task or the queue's write-back clobbers it (see +// task.Client.UpdateTask). +type Runner interface { + // GetRunnerType returns the task type this runner processes. Unique across a queue's runners; + // must match task.Task.Type of the tasks it handles. + GetRunnerType() string + + // GetRunnerDeadline returns the duration, measured from when the task starts running, after + // which a still-running task is forcibly reset to pending/available by the unstick mechanism, + // recovering tasks orphaned by crashes, kills, or a runner that never returns. The reset is not + // immediate: unstick runs on a periodic sweep, so a task is recovered on the first sweep after + // its deadline elapses. Must exceed GetRunnerTimeout so a task still within its timeout is never + // unstuck and re-claimed mid-run. + GetRunnerDeadline() time.Duration + + // GetRunnerTimeout returns the hard cap on a run; at elapse the Run context is canceled so a + // cooperative runner can return. Must exceed GetRunnerDurationMaximum. + GetRunnerTimeout() time.Duration + + // GetRunnerDurationMaximum returns the expected upper bound of a normal run. Advisory: exceeding + // it logs a warning but does not interrupt the run. Must be positive. + GetRunnerDurationMaximum() time.Duration + + // Run executes tsk within ctx. Before returning it must move tsk out of running (completed, + // failed, or rescheduled via Repeat*); a task left running is failed by the queue, unless + // interrupted by shutdown, in which case it is reverted to pending. ctx is canceled after + // GetRunnerTimeout and on shutdown. Called concurrently for distinct tasks; must be concurrency-safe. + Run(ctx context.Context, tsk *task.Task) +} diff --git a/task/service/service/service.go b/task/service/service/service.go index da108c1589..4fd343d7a0 100644 --- a/task/service/service/service.go +++ b/task/service/service/service.go @@ -36,7 +36,7 @@ type Service struct { dataClient dataClient.Client dataSourceClient dataSource.Client dexcomClient dexcom.Client - taskQueue queue.Queue + taskQueue *queue.MultiQueue clinicsClient clinics.Client } diff --git a/task/store/mongo/mongo.go b/task/store/mongo/mongo.go index 1edf9464c4..2aeb33eeb3 100644 --- a/task/store/mongo/mongo.go +++ b/task/store/mongo/mongo.go @@ -25,21 +25,6 @@ import ( taskStore "github.com/tidepool-org/platform/task/store" ) -var TasksStateTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tidepool_task_tasks_state_total", - Help: "The total number of tasks sorted by state and type", -}, []string{"state", "type"}) - -// TasksLostCompletionTotal counts task completions dropped because the compare-and-swap in -// StopTask missed (the running claim was concurrently modified, unstuck, or deleted). The task -// is recovered by the deadline/unstick mechanism, but the intended terminal state is lost. The -// type label is populated only when the repository is type-filtered (as it is for each queue in -// a MultiQueue); otherwise it is empty. -var TasksLostCompletionTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tidepool_task_lost_completion_total", - Help: "The total number of task completions dropped because the state-lock compare-and-swap missed, sorted by type", -}, []string{"type"}) - const ( MaxTaskCreationDuration = 30 * time.Second @@ -123,25 +108,33 @@ func (t *TaskRepository) EnsureIndexes() error { { Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index(). - SetUnique(true). - SetBackground(true), + SetUnique(true), }, { Keys: bson.D{{Key: "name", Value: 1}}, Options: options.Index(). SetUnique(true). - SetSparse(true). - SetBackground(true), + SetSparse(true), }, { Keys: bson.D{{Key: "availableTime", Value: 1}}, - Options: options.Index(). - SetBackground(true), }, { Keys: bson.D{{Key: "state", Value: 1}}, + }, + { + // Used by IteratePending; type equality, then availableTime for range and sort; partial + // on pending since that is the only state it queries. + Keys: bson.D{{Key: "type", Value: 1}, {Key: "availableTime", Value: 1}}, Options: options.Index(). - SetBackground(true), + SetPartialFilterExpression(bson.D{{Key: "state", Value: task.TaskStatePending}}), + }, + { + // Used by UnstickTasks; type equality, then deadlineTime for range and sort; partial + // on running since that is the only state it queries. + Keys: bson.D{{Key: "type", Value: 1}, {Key: "deadlineTime", Value: 1}}, + Options: options.Index(). + SetPartialFilterExpression(bson.D{{Key: "state", Value: task.TaskStateRunning}}), }, }) } @@ -208,10 +201,11 @@ func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, return nil, err } - now := time.Now().UTC() logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"filter": filter, "pagination": pagination}) - tasks := task.Tasks{} + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("ListTasks") }() + selector := bson.M{} if filter.Name != nil { @@ -223,20 +217,21 @@ func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, if filter.State != nil { selector["state"] = *filter.State } + if t.typeFilter != nil { selector["type"] = *t.typeFilter } - opts := storeStructuredMongo.FindWithPagination(pagination). - SetSort(bson.M{"createdTime": -1}) + + opts := storeStructuredMongo.FindWithPagination(pagination).SetSort(bson.M{"createdTime": -1}) cursor, err := t.Find(ctx, selector, opts) if err != nil { - logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("ListTasks") + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to list tasks") } - err = cursor.All(ctx, &tasks) - logger.WithFields(log.Fields{"count": len(tasks), "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("ListTasks") - if err != nil { + tasks := task.Tasks{} + if err = cursor.All(ctx, &tasks); err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to decode tasks") } @@ -244,6 +239,12 @@ func (t *TaskRepository) ListTasks(ctx context.Context, filter *task.TaskFilter, tasks = task.Tasks{} } + taskIds := make([]string, len(tasks)) + for index, tsk := range tasks { + taskIds[index] = tsk.ID + } + logger = logger.WithField("taskIds", taskIds) + return tasks, nil } @@ -262,15 +263,18 @@ func (t *TaskRepository) CreateTask(ctx context.Context, create *task.TaskCreate return nil, err } - now := time.Now().UTC() logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"create": create}) - _, err = t.InsertOne(ctx, tsk) - logger.WithFields(log.Fields{"task": tsk.LogFields(), "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("CreateTask") - if err != nil { + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("CreateTask") }() + + if _, err = t.InsertOne(ctx, tsk); err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to create task") } + logger = logger.WithField("task", tsk.LogFields()) + TasksStateTotal.WithLabelValues(task.TaskStatePending, create.Type).Inc() return tsk, nil } @@ -288,16 +292,17 @@ func (t *TaskRepository) GetTask(ctx context.Context, id string, condition *stor return nil, errors.Wrap(err, "condition is invalid") } - now := time.Now().UTC() logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "condition": condition}) + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("GetTask") }() + tsk := &task.Task{} err := t.FindOne(ctx, t.selector(id, condition)).Decode(tsk) - logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("GetTask") - if errors.Is(err, mongo.ErrNoDocuments) { return nil, nil } else if err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to get task") } @@ -322,19 +327,21 @@ func (t *TaskRepository) UpdateTask(ctx context.Context, id string, condition *s return nil, errors.Wrap(err, "update is invalid") } - now := time.Now().UTC() logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "condition": condition, "update": update}) + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("UpdateTask") }() + set, unset := t.parseUpdate(update) set["modifiedTime"] = now updatedTask := &task.Task{} opts := options.FindOneAndUpdate().SetReturnDocument(options.After) err := t.FindOneAndUpdate(ctx, t.selector(id, condition), t.ConstructUpdate(set, unset), opts).Decode(updatedTask) - logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("UpdateTask") if errors.Is(err, mongo.ErrNoDocuments) { return nil, nil } else if err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to update task") } @@ -354,13 +361,16 @@ func (t *TaskRepository) DeleteTask(ctx context.Context, id string, condition *s return errors.Wrap(err, "condition is invalid") } - now := time.Now().UTC() logger := log.LoggerFromContext(ctx).WithField("id", id) - changeInfo, err := t.DeleteOne(ctx, t.selector(id, condition)) - logger.WithFields(log.Fields{"changeInfo": changeInfo, "duration": time.Since(now) / time.Microsecond}).WithError(err).Debug("DeleteTask") - if err != nil { + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("DeleteTask") }() + + if changeInfo, err := t.DeleteOne(ctx, t.selector(id, condition)); err != nil { + logger = logger.WithError(err) return errors.Wrap(err, "unable to delete task") + } else { + logger = logger.WithField("changeInfo", changeInfo) } return nil @@ -385,8 +395,10 @@ func (t *TaskRepository) StartTask(ctx context.Context, id string, revision int, ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), TransitionTimeout) defer cancel() + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "revision": revision, "deadline": deadline.String()}) + now := time.Now().UTC() - logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "revision": revision, "deadline": deadline}) + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("StartTask") }() set := bson.M{ "state": task.TaskStateRunning, @@ -406,10 +418,10 @@ func (t *TaskRepository) StartTask(ctx context.Context, id string, revision int, tsk := &task.Task{} opts := options.FindOneAndUpdate().SetReturnDocument(options.After) err := t.FindOneAndUpdate(ctx, selector, t.ConstructUpdate(set, unset), opts).Decode(tsk) - logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("StartTask") if errors.Is(err, mongo.ErrNoDocuments) { return nil, nil } else if err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to start task") } @@ -450,9 +462,11 @@ func (t *TaskRepository) StopTask(ctx context.Context, id string, stateLock *str ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), TransitionTimeout) defer cancel() - now := time.Now().UTC() logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "stateLock": stateLock, "state": state, "duration": duration, "update": update}) + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("StopTask") }() + set, unset := t.parseUpdate(update) set["modifiedTime"] = now set["state"] = state @@ -474,17 +488,17 @@ func (t *TaskRepository) StopTask(ctx context.Context, id string, stateLock *str tsk := &task.Task{} err := t.FindOneAndUpdate(ctx, selector, t.ConstructUpdate(set, unset)).Decode(tsk) - logger.WithField("duration", time.Since(now)/time.Microsecond).WithError(err).Debug("StopTask") if errors.Is(err, mongo.ErrNoDocuments) { // The compare-and-swap missed: no running task matched the expected state lock // (it was concurrently modified, unstuck, or deleted since it started). // The state transition is dropped; the deadline and unstick mechanism will // recover the task if it was left running. This is logged and counted so lost // completions are observable rather than silently swallowed. - logger.Warn("Unable to stop task; no running task matched the expected condition") + logger.Error("Unable to stop task; no running task matched the expected condition") TasksLostCompletionTotal.WithLabelValues(pointer.Default(t.typeFilter, "")).Inc() return nil } else if err != nil { + logger = logger.WithError(err) return errors.Wrap(err, "unable to stop task") } @@ -497,10 +511,10 @@ func (t *TaskRepository) UnstickTasks(ctx context.Context) ([]string, error) { return nil, errors.New("context is missing") } - lgr := log.LoggerFromContext(ctx) + logger := log.LoggerFromContext(ctx) now := time.Now().UTC() - defer func() { lgr.WithField("duration", time.Since(now)/time.Microsecond).Debug("UnstickTasks") }() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("UnstickTasks") }() findSelector := bson.M{ "state": task.TaskStateRunning, @@ -513,6 +527,7 @@ func (t *TaskRepository) UnstickTasks(ctx context.Context) ([]string, error) { opts := options.Find().SetSort(bson.M{"deadlineTime": 1}) cursor, err := t.Find(ctx, findSelector, opts) if err != nil { + logger = logger.WithError(err) return nil, errors.Wrap(err, "unable to list tasks") } defer storeStructuredMongo.CloseCursor(ctx, cursor) @@ -521,6 +536,7 @@ func (t *TaskRepository) UnstickTasks(ctx context.Context) ([]string, error) { for cursor.Next(ctx) { tsk := &task.Task{} if err = cursor.Decode(tsk); err != nil { + logger = logger.WithError(err) log.LoggerFromContext(ctx).WithError(err).Error("Unable to decode task") continue } @@ -543,13 +559,21 @@ func (t *TaskRepository) UnstickTasks(ctx context.Context) ([]string, error) { "stateLock": 1, } if result, updateErr := t.UpdateOne(ctx, updateSelector, t.ConstructUpdate(set, unset)); updateErr != nil { + logger = logger.WithError(updateErr) return ids, updateErr } else if result.ModifiedCount > 0 { ids = append(ids, tsk.ID) } } - return ids, cursor.Err() + logger = logger.WithField("taskIds", ids) + + if err = cursor.Err(); err != nil { + logger = logger.WithError(err) + return ids, err // Still want to return the ids of tasks that were successfully unstuck + } + + return ids, nil } func (t *TaskRepository) IteratePending(ctx context.Context) (*mongo.Cursor, error) { @@ -632,3 +656,20 @@ func (t *TaskRepository) assertType(expected *string, actual *string) error { func newStateLock() string { return id.Must(id.New(16)) } + +var ( + TasksStateTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_tasks_state_total", + Help: "The total number of tasks sorted by state and type", + }, []string{"state", "type"}) + + // TasksLostCompletionTotal counts task completions dropped because the compare-and-swap in + // StopTask missed (the running claim was concurrently modified, unstuck, or deleted). The task + // is recovered by the deadline/unstick mechanism, but the intended terminal state is lost. The + // type label is populated only when the repository is type-filtered (as it is for each queue in + // a MultiQueue); otherwise it is empty. + TasksLostCompletionTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_lost_completion_total", + Help: "The total number of task completions dropped because the state-lock compare-and-swap missed, sorted by type", + }, []string{"type"}) +) diff --git a/task/store/mongo/mongo_test.go b/task/store/mongo/mongo_test.go index 600f8854ca..ed6cc71c52 100644 --- a/task/store/mongo/mongo_test.go +++ b/task/store/mongo/mongo_test.go @@ -84,23 +84,31 @@ var _ = Describe("Mongo", func() { "Key": Equal(storeStructuredMongoTest.MakeKeySlice("_id")), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("id")), - "Background": Equal(true), - "Unique": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("id")), + "Unique": Equal(true), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("name")), - "Background": Equal(true), - "Unique": Equal(true), - "Sparse": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("name")), + "Unique": Equal(true), + "Sparse": Equal(true), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("availableTime")), - "Background": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("availableTime")), }), MatchFields(IgnoreExtras, Fields{ - "Key": Equal(storeStructuredMongoTest.MakeKeySlice("state")), - "Background": Equal(true), + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("state")), + }), + MatchFields(IgnoreExtras, Fields{ + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("type", "availableTime")), + "PartialFilterExpression": Equal(bson.D{ + {Key: "state", Value: task.TaskStatePending}, + }), + }), + MatchFields(IgnoreExtras, Fields{ + "Key": Equal(storeStructuredMongoTest.MakeKeySlice("type", "deadlineTime")), + "PartialFilterExpression": Equal(bson.D{ + {Key: "state", Value: task.TaskStateRunning}, + }), }), )) }) diff --git a/task/task.go b/task/task.go index 6410b13a8c..034448e35e 100644 --- a/task/task.go +++ b/task/task.go @@ -294,6 +294,11 @@ func (t *Task) SetFailed() { t.State = TaskStateFailed } +func (t *Task) SetFailedWithError(err error) { + t.State = TaskStateFailed + t.AppendError(err) +} + func (t *Task) IsCompleted() bool { return t.State == TaskStateCompleted } diff --git a/twiist/provider/provider.go b/twiist/provider/provider.go index 477b58307e..20594721de 100644 --- a/twiist/provider/provider.go +++ b/twiist/provider/provider.go @@ -2,14 +2,12 @@ package provider import ( "context" - "net/http" "time" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/tidepool-org/platform/auth" providerSession "github.com/tidepool-org/platform/auth/providersession" - "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/config" "github.com/tidepool-org/platform/data" dataDeduplicatorDeduplicator "github.com/tidepool-org/platform/data/deduplicator/deduplicator" @@ -67,15 +65,7 @@ func New(providerDependencies ProviderDependencies) (*Provider, error) { return nil, errors.Wrap(err, "unable to create provider config") } - // Create http client - httpClient := &http.Client{ - Transport: prometheusRequestMetricsRoundTripper, - CheckRedirect: http.DefaultClient.CheckRedirect, - Jar: http.DefaultClient.Jar, - Timeout: 2 * time.Minute, - } - - prvdr, err := oauthProvider.New(twiist.ProviderName, cfg, httpClient, providerDependencies.JWKS) + prvdr, err := oauthProvider.New(twiist.ProviderName, cfg, providerDependencies.JWKS) if err != nil { return nil, err } @@ -289,5 +279,3 @@ func NewDataSetCreate() *data.DataSetCreate { TimeProcessing: pointer.FromString(data.TimeProcessingNone), } } - -var prometheusRequestMetricsRoundTripper = client.NewPrometheusRequestMetricsRoundTripper("tidepool_twiist_api", "Tidepool twiist API") From 8f81b8310e480d2eaae44e6cbc3eafbee77d489f Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Wed, 22 Jul 2026 21:56:37 -0700 Subject: [PATCH 09/20] Update platform-plugin-abbott dependency --- private/plugin/abbott | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private/plugin/abbott b/private/plugin/abbott index 749f8eb780..1b63c21c2d 160000 --- a/private/plugin/abbott +++ b/private/plugin/abbott @@ -1 +1 @@ -Subproject commit 749f8eb7805e4f88b931a788a60057f65ee0aa60 +Subproject commit 1b63c21c2d4a4147fd1692b9a49400727f72d761 From 1a6ff9024253c085a21ca604ba339274928ad010 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Thu, 23 Jul 2026 15:12:53 -0700 Subject: [PATCH 10/20] Fix issues with task metrics and queue - Update Prometheus metrics - Capture Dexcom response request-time header correctly - Warn and capture if task revision changed during run - Simplify task queue tests --- README.md | 44 ++++ client/prometheus.go | 4 +- dexcom/client/client.go | 10 +- task/queue/queue.go | 73 ++++--- task/queue/queue_test.go | 362 +++++++++++++++++++-------------- task/queue/test/runner.go | 188 ++++------------- task/store/mongo/mongo.go | 51 +++-- task/store/mongo/mongo_test.go | 10 +- task/store/store.go | 2 +- 9 files changed, 384 insertions(+), 360 deletions(-) diff --git a/README.md b/README.md index 30bc439961..a500323fb5 100644 --- a/README.md +++ b/README.md @@ -159,3 +159,47 @@ Review all pending changes to all dependencies. If any changes could have a nega Ensure the `ci-build` and `ci-test` Makefile targets pass using the target Golang version. If you previously noted any changes or issues of concern, perform any explicit tests necessary. + +## Prometheus Metrics + +See source files for further details about and usage of each metric. + +### Summary Store + +* `tidepool_summary_queue_lag` - (histogram) - the current summary queue lag, in minutes +* `tidepool_summary_queue_length` - (gauge) - the current summary queue length, in number of summaries + +### C2C + +#### Abbott + +* `tidepool_abbott_api_request_count` - (counter) - Abbott API request count, sorted by method, path, and status +* `tidepool_abbott_api_request_duration_seconds` - (histogram) - Abbott API duration of each request, in seconds, sorted by method, path, and status + +#### Dexcom + +* `tidepool_dexcom_api_request_count` - (counter) - Dexcom API request count, sorted by method, path, and status +* `tidepool_dexcom_api_request_duration_seconds` - (histogram) - Dexcom API duration of each request, in seconds, sorted by method, path, and status +* `tidepool_dexcom_api_request_time_seconds` - (histogram) - Dexcom API duration of each request, as reported in the "request-time" response header from Dexcom, in seconds, sorted by method, path, and status + +#### Oura + +* `tidepool_oura_api_request_count` - (counter) - Oura API request count, sorted by method, path, and status +* `tidepool_oura_api_request_duration_seconds` - (histogram) - Oura API duration of each request, in seconds, sorted by method, path, and status + +### Task + +#### Queue + +* `tidepool_task_workers_total` - (gauge) - configured number of task queue workers, sorted by queue (5, per config) +* `tidepool_task_workers_available` - (gauge) - number of available task queue workers, sorted by queue (5, per config) +* `tidepool_task_runner_not_found_total` - (counter) - total number of task runs with no registered runner for the task type, sorted by type (ideally zero) +* `tidepool_task_run_duration_seconds` - (histogram) - duration of task runs in seconds, sorted by type +* `tidepool_task_runner_timeout_exceeded_total` - (counter) - total number of task runs that exceeded the runner timeout, sorted by type and disposition ("blocked", "recovered") (ideally zero) +* `tidepool_task_run_panic_total` - (counter) - total number of task runs that panicked, sorted by type (ideally 0) + +#### Store + +* `tidepool_task_type_state_total` - (counter) - total number of tasks run, sorted by type and state +* `tidepool_task_type_lost_completion_total` - (counter) - total number of task completions dropped because the state-lock compare-and-swap missed, sorted by type (ideally low-ish) +* `tidepool_task_type_revision_mismatch_total` - (counter) - total number of task revisions that do not match the task revision in the database, sorted by type (ideally zero) diff --git a/client/prometheus.go b/client/prometheus.go index 79275da865..a207076ec7 100644 --- a/client/prometheus.go +++ b/client/prometheus.go @@ -99,14 +99,14 @@ func NewPrometheusRequestMetricsRoundTripperWithPathPatternsAndDurationBuckets(n requestCountCounterVec: promauto.NewCounterVec( prometheus.CounterOpts{ Name: fmt.Sprintf("%s_request_count", name), - Help: fmt.Sprintf("%s request count", help), + Help: fmt.Sprintf("%s request count, sorted by method, path, and status", help), }, PrometheusLabelNames(), ), requestDurationHistogramVec: promauto.NewHistogramVec( prometheus.HistogramOpts{ Name: fmt.Sprintf("%s_request_duration_seconds", name), - Help: fmt.Sprintf("%s request duration (seconds)", help), + Help: fmt.Sprintf("%s request duration, in seconds, sorted by method, path, and status", help), Buckets: pointer.DefaultArray(durationBuckets, DurationBucketsDefault), }, PrometheusLabelNames(), diff --git a/dexcom/client/client.go b/dexcom/client/client.go index e026fb1256..55ad46d203 100644 --- a/dexcom/client/client.go +++ b/dexcom/client/client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strings" "time" "github.com/prometheus/client_golang/prometheus" @@ -142,7 +143,8 @@ func (c *Client) sendDexcomRequest(ctx context.Context, method string, url strin // Some Dexcom API responses include a "request-time" header with, supposedly, the internal duration of the request. // This could be useful for debugging connection issues if compared against the calculated request duration. -// See client/prometheus.go for details on how the request duration is calculated and recorded. +// See client/prometheus.go for details on how the request duration is calculated and recorded. The format for +// this header is non-standard duration (e.g. "1234 ms") and the space needs to be removed for Golang to parse. const RequestTimeHeaderName = "request-time" @@ -170,8 +172,10 @@ func (p *PrometheusRequestMetricsRoundTripper) RoundTrip(req *http.Request) (*ht if res != nil { if labels := p.Labels(req, res); labels != nil { - if requestTime, parseErr := time.ParseDuration(res.Header.Get(RequestTimeHeaderName)); parseErr == nil { - p.requestTimeHistogramVec.With(*labels).Observe(requestTime.Seconds()) + if requestTimeHeader := strings.ReplaceAll(res.Header.Get(RequestTimeHeaderName), " ", ""); requestTimeHeader != "" { + if requestTime, parseErr := time.ParseDuration(requestTimeHeader); parseErr == nil { + p.requestTimeHistogramVec.With(*labels).Observe(requestTime.Seconds()) + } } } } diff --git a/task/queue/queue.go b/task/queue/queue.go index 80d8b45180..3d7b6c4050 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -222,7 +222,7 @@ func (q *Queue) Start() { } q.started = true - q.logger.Info("Task queue starting") + q.logger.Debug("Task queue starting") ctx, cancelFunc := context.WithCancel(log.NewContextWithLogger(context.Background(), q.logger)) q.cancelFunc = cancelFunc @@ -230,7 +230,7 @@ func (q *Queue) Start() { q.startWorkers(ctx) q.startManager(ctx) - q.logger.Info("Task queue started") + q.logger.Debug("Task queue started") } func (q *Queue) Stop() { @@ -335,8 +335,8 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { if !ok { // A whole task type is unprocessable by this queue; surface it distinctly (a configuration // problem) rather than letting it blend in with ordinary per-task failures downstream. - lgr.Error("Runner not found for task; task cannot be processed") - tsk.SetFailedWithError(errors.New("runner not found for task")) + lgr.Error("Runner not found for task type; task cannot be processed") + tsk.SetFailedWithError(errors.New("runner not found for task type")) RunnerNotFoundTotal.WithLabelValues(tsk.Type).Inc() return } @@ -349,8 +349,6 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { } }() - startTime := time.Now() - // If runner does not respect its own maximum duration, then enforce a context-based timeout. // This forces the task to cancel via the context. runnerContext, cancel := context.WithTimeoutCause(ctx, runner.GetRunnerTimeout(), ErrRunnerTimeout) @@ -370,7 +368,9 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { defer runnerWatchdog.Stop() // Run the task via the runner + startTime := time.Now() runner.Run(runnerContext, tsk) + duration := time.Since(startTime).Truncate(time.Millisecond) // Immediate stop the runner watchdog runnerWatchdog.Stop() @@ -391,10 +391,9 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { } } - taskDuration := time.Since(startTime) - RunDurationSeconds.WithLabelValues(runner.GetRunnerType()).Observe(taskDuration.Seconds()) - if taskDuration > runner.GetRunnerDurationMaximum() { - lgr.WithField("taskDuration", taskDuration.Truncate(time.Millisecond).Seconds()).Warn("Task duration exceeds maximum") + RunDurationSeconds.WithLabelValues(runner.GetRunnerType()).Observe(duration.Seconds()) + if duration > runner.GetRunnerDurationMaximum() { + lgr.WithField("duration", duration.Seconds()).Warn("Task duration exceeds maximum") } } @@ -545,7 +544,7 @@ func (q *Queue) dispatchTask(ctx context.Context, tsk *task.Task) error { // uses the started task state lock so it reliably matches. select { case <-ctx.Done(): - if err := q.repository.StopTask(ctx, startedTask.ID, startedTask.StateLock, task.TaskStatePending, nil, nil); err != nil { + if err := q.repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.StateLock, task.TaskStatePending, nil, nil); err != nil { return errors.Wrap(err, "unable to revert task to pending") } case q.dispatchChannel <- startedTask: @@ -590,7 +589,7 @@ func (q *Queue) completeTask(ctx context.Context, tsk *task.Task) { AvailableTime: tsk.AvailableTime, Error: &errors.Serializable{Error: tsk.GetError()}, } - if err := q.repository.StopTask(ctx, tsk.ID, tsk.StateLock, tsk.State, duration, update); err != nil { + if err := q.repository.StopTask(ctx, tsk.ID, tsk.Revision, tsk.StateLock, tsk.State, duration, update); err != nil { lgr.WithError(err).Error("Unable to complete task") } } @@ -674,16 +673,18 @@ func durationWithJitter(duration time.Duration) time.Duration { } var ( - // RunnerTimeoutExceededTotal counts task runs that exceeded the runner timeout, sorted by type - // and disposition: - // - "blocked" - for runs the watchdog caught still running past the grace period (the worker is - // wedged until the runner eventually returns or the process restarts, so this is the severe case) - // - "recovered" - for runs that exceeded the timeout, but returned once their context was - // canceled (the timeout mechanism worked as designed) - RunnerTimeoutExceededTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tidepool_task_runner_timeout_exceeded_total", - Help: "The total number of task runs that exceeded the runner timeout, sorted by type and disposition", - }, []string{"type", "disposition"}) + // WorkersTotal reports the configured number of workers, per queue. + WorkersTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_task_workers_total", + Help: "The configured number of task queue workers, sorted by queue", + }, []string{"queue"}) + + // WorkersAvailable reports the number of available workers, per queue. A value pinned at zero + // indicates a saturated queue or workers wedged in non-cooperative runners. + WorkersAvailable = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "tidepool_task_workers_available", + Help: "The number of available task queue workers, sorted by queue", + }, []string{"queue"}) // RunnerNotFoundTotal counts task runs for which no runner is registered for the task's type, // sorted by type. A non-zero value indicates pending tasks of a type this queue cannot process @@ -697,26 +698,24 @@ var ( // run latency percentiles and to alert when durations approach the runner timeout. RunDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "tidepool_task_run_duration_seconds", - Help: "The duration of task runs in seconds, sorted by type", + Help: "The duration of task runs, in seconds, sorted by type", Buckets: prometheus.ExponentialBuckets(0.1, 2, 15), }, []string{"type"}) - // RunPanicTotal counts task runs that panicked and were recovered, sorted by type. + // RunnerTimeoutExceededTotal counts task runs that exceeded the runner timeout, sorted by type + // and disposition, where disposition can be: + // - "blocked" - for runs the watchdog caught still running past the grace period (the worker is + // wedged until the runner eventually returns or the process restarts, so this is the severe case) + // - "recovered" - for runs that exceeded the timeout, but returned once their context was + // canceled (the timeout mechanism worked as designed) + RunnerTimeoutExceededTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_runner_timeout_exceeded_total", + Help: "The total number of task runs that exceeded the runner timeout, sorted by type and disposition", + }, []string{"type", "disposition"}) + + // RunPanicTotal counts task runs that panicked and were recovered and failed, sorted by type. RunPanicTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "tidepool_task_run_panic_total", Help: "The total number of task runs that panicked, sorted by type", }, []string{"type"}) - - // WorkersAvailable reports the number of idle workers per queue. A value pinned at zero - // indicates a saturated queue or workers wedged in non-cooperative runners. - WorkersAvailable = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "tidepool_task_workers_available", - Help: "The number of idle task queue workers, sorted by queue", - }, []string{"queue"}) - - // WorkersTotal reports the configured number of workers per queue. - WorkersTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "tidepool_task_workers_total", - Help: "The configured number of task queue workers, sorted by queue", - }, []string{"queue"}) ) diff --git a/task/queue/queue_test.go b/task/queue/queue_test.go index c67e834552..a242bac1fa 100644 --- a/task/queue/queue_test.go +++ b/task/queue/queue_test.go @@ -15,6 +15,7 @@ import ( "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" logTest "github.com/tidepool-org/platform/log/test" + metadataTest "github.com/tidepool-org/platform/metadata/test" "github.com/tidepool-org/platform/pointer" storeStructuredMongoTest "github.com/tidepool-org/platform/store/structured/mongo/test" "github.com/tidepool-org/platform/task" @@ -268,21 +269,30 @@ var _ = Describe("Queue", func() { }) It("returns an error when a runner duration maximum is not positive", func() { - runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 3*time.Minute, 2*time.Minute, 0, 0) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDeadline(3 * time.Minute). + WithTimeout(2 * time.Minute). + WithDurationMaximum(0) que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner) Expect(err).To(MatchError("runner duration maximum is invalid")) Expect(que).To(BeNil()) }) It("returns an error when a runner timeout does not exceed its duration maximum", func() { - runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 3*time.Minute, time.Minute, time.Minute, 0) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDeadline(3 * time.Minute). + WithTimeout(time.Minute). + WithDurationMaximum(time.Minute) que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner) Expect(err).To(MatchError("runner timeout is invalid")) Expect(que).To(BeNil()) }) It("returns an error when a runner deadline does not exceed its timeout", func() { - runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 2*time.Minute, 2*time.Minute, time.Minute, 0) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDeadline(2 * time.Minute). + WithTimeout(2 * time.Minute). + WithDurationMaximum(time.Minute) que, err := taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner) Expect(err).To(MatchError("runner deadline is invalid")) Expect(que).To(BeNil()) @@ -360,28 +370,56 @@ var _ = Describe("Queue", func() { } }) - Context("with a single queue", func() { - var countingRunner *taskQueueTest.CountingRunner - var panicRunner *taskQueueTest.PanicRunner + Context("with successful shutdown", func() { var cfg *taskQueue.Config var que *taskQueue.Queue BeforeEach(func() { - countingRunner = taskQueueTest.NewCountingRunner(taskTest.RandomType()) - panicRunner = taskQueueTest.NewPanicRunner(taskTest.RandomType()) - - cfg = &taskQueue.Config{Workers: 2, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault} - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, countingRunner, panicRunner)) + cfg = &taskQueue.Config{ + Workers: 2, + Delay: time.Millisecond, + DelayInitial: time.Millisecond, + DelayUnstick: taskQueue.DelayUnstickDefault, + StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, + RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault, + } }) AfterEach(func() { - que.Stop() + if que != nil { + que.Stop() + } lgr.AssertDebug("Task queue worker stopped", log.Fields{"error": errors.NewSerializable(context.Canceled)}) lgr.AssertDebug("Task queue manager stopped", log.Fields{"error": errors.NewSerializable(context.Canceled)}) }) + It("fails a pending task that does not match any registered runner", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + unregisteredType := taskTest.RandomType() + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: unregisteredType})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateFailed)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateFailed)) + Expect(actualTask.Error).ToNot(BeNil()) + Expect(actualTask.Error.Error).To(MatchError("runner not found for task type")) + + lgr.AssertError("Runner not found for task type; task cannot be processed") + Expect(testutil.ToFloat64(taskQueue.RunnerNotFoundTotal.WithLabelValues(unregisteredType))).To(Equal(float64(1))) + }) + It("dispatches, runs, and completes a pending task matching a registered runner", func() { - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: countingRunner.GetRunnerType()})) + runner := taskQueueTest.NewCountingRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -389,25 +427,20 @@ var _ = Describe("Queue", func() { return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) - Expect(countingRunner.GetCount()).To(Equal(1)) + Expect(runner.GetCount()).To(Equal(1)) }) It("completes a task that the runner updated while it was running", func() { - updatingRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { - // A runner may update its own task mid-run. The update bumps the revision but - // leaves the state lock intact, so the queue's completion must still match. - data := map[string]any{"key": "value"} - updated, err := str.NewTaskRepository().UpdateTask(ctx, tsk.ID, nil, &task.TaskUpdate{Data: &data}) - if err != nil || updated == nil { - tsk.AppendError(errors.New("unable to update task during run")) - return - } - *tsk = *updated // replace the in-memory task with the updated one, per the runner contract - tsk.SetCompleted() - }) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, updatingRunner)) + updatedData := metadataTest.RandomMetadataMap() - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: updatingRunner.GetRunnerType()})) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { + *tsk = *test.Must(str.NewTaskRepository().UpdateTask(ctx, tsk.ID, nil, &task.TaskUpdate{Data: &updatedData})) + tsk.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -418,41 +451,66 @@ var _ = Describe("Queue", func() { actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) Expect(actualTask.Error).To(BeNil()) - Expect(actualTask.Data).To(HaveKeyWithValue("key", "value")) + Expect(actualTask.Data).To(Equal(updatedData)) + }) + + It("logs a warning if the runner update the task while it was running, but did not use the updated task", func() { + updatedData := metadataTest.RandomMetadataMap() + + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { + test.Must(str.NewTaskRepository().UpdateTask(ctx, tsk.ID, nil, &task.TaskUpdate{Data: &updatedData})) + tsk.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.Error).To(BeNil()) + Expect(actualTask.Data).To(BeNil()) + + lgr.AssertWarn("Database task revision does not match running task revision; Runner contract broken or concurrent update") }) It("does not complete a task whose state lock changed while it was running", func() { - stealingRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { - // Simulate the task being unstuck and re-claimed elsewhere by changing the - // state lock out from under this run. The completion must then miss rather - // than falsely complete another run's task. - _, err := str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": tsk.ID}, bson.M{"$set": bson.M{"stateLock": "ffffffffffffffffffffffffffffffff"}}) - if err != nil { - tsk.AppendError(err) - return - } - tsk.SetCompleted() - }) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, stealingRunner)) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { + // Simulate the task being unstuck and re-claimed elsewhere by changing the + // state lock out from under this run. The completion must then miss rather + // than falsely complete another run task. + test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": tsk.ID}, bson.M{"$set": bson.M{"stateLock": ""}})) + tsk.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: stealingRunner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() - // The completion's compare-and-swap misses, which is logged rather than silently swallowed. Eventually(func() bool { defer func() { _ = recover() }() lgr.AssertError("Unable to stop task; no running task matched the expected condition") return true }, "5s", "100ms").To(BeTrue()) - // The task was not falsely completed; it remains running (recovered later by unstick). actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) Expect(actualTask.State).To(Equal(task.TaskStateRunning)) }) It("cleans up a task that panics during execution", func() { - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: panicRunner.GetRunnerType()})) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { panic("panic test") }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -460,45 +518,21 @@ var _ = Describe("Queue", func() { return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State }, "5s", "50ms").To(Equal(task.TaskStateFailed)) - Expect(countingRunner.GetCount()).To(Equal(0)) - actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) Expect(actualTask.State).To(Equal(task.TaskStateFailed)) Expect(actualTask.Error).ToNot(BeNil()) Expect(actualTask.Error.Error).To(MatchError("unhandled panic")) lgr.AssertError("Unhandled panic while running task") - Expect(testutil.ToFloat64(taskQueue.RunPanicTotal.WithLabelValues(panicRunner.GetRunnerType()))).To(Equal(float64(1))) - }) - - It("fails a pending task that does not match any registered runner", func() { - unregisteredType := taskTest.RandomType() - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: unregisteredType})) - - que.Start() - - Eventually(func() string { - return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State - }, "5s", "50ms").To(Equal(task.TaskStateFailed)) - - Expect(countingRunner.GetCount()).To(Equal(0)) - - actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) - Expect(actualTask.State).To(Equal(task.TaskStateFailed)) - Expect(actualTask.Error).ToNot(BeNil()) - Expect(actualTask.Error.Error).To(MatchError("runner not found for task")) - - lgr.AssertError("Runner not found for task; task cannot be processed") - Expect(testutil.ToFloat64(taskQueue.RunnerNotFoundTotal.WithLabelValues(unregisteredType))).To(Equal(float64(1))) + Expect(testutil.ToFloat64(taskQueue.RunPanicTotal.WithLabelValues(runner.GetRunnerType()))).To(Equal(float64(1))) }) It("fails a task whose runner leaves it in an unknown state", func() { - unknownStateRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { - tsk.State = "unknown-state" - }) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, unknownStateRunner)) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { tsk.State = "unknown-state" }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: unknownStateRunner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -513,55 +547,66 @@ var _ = Describe("Queue", func() { }) It("warns and sets the available time for a pending task left without one", func() { - missingAvailableTimeRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { - tsk.State = task.TaskStatePending - tsk.AvailableTime = nil - }) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, missingAvailableTimeRunner)) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { + tsk.State = task.TaskStatePending + tsk.AvailableTime = nil + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: missingAvailableTimeRunner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() - Eventually(func() bool { + Eventually(func() string { defer func() { _ = recover() }() lgr.AssertWarn("Available time missing for pending task") - return true - }, "5s", "50ms").To(BeTrue()) + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStatePending)) + + que.Stop() + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) }) - It("warns when a pending task's available time is significantly in the past", func() { - staleAvailableTimeRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), func(ctx context.Context, tsk *task.Task) { - tsk.RepeatAvailableAt(time.Now().Add(-2 * time.Minute)) - }) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, staleAvailableTimeRunner)) + It("warns when a pending task available time is significantly in the past", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { tsk.RepeatAvailableAt(time.Now().Add(-2 * time.Minute)) }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: staleAvailableTimeRunner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() - Eventually(func() bool { + Eventually(func() string { defer func() { _ = recover() }() lgr.AssertWarn("Available time significantly before now for pending task") - return true - }, "5s", "50ms").To(BeTrue()) + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStatePending)) + + que.Stop() + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) }) It("unsticks and logs a task left running past its deadline", func() { - stuckTask := &task.Task{ + cfg.DelayUnstick = time.Millisecond + + runner := taskQueueTest.NewCountingRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := &task.Task{ ID: task.NewID(), - Type: countingRunner.GetRunnerType(), + Type: runner.GetRunnerType(), State: task.TaskStateRunning, StateLock: pointer.FromString(taskTest.RandomType()), CreatedTime: time.Now(), Revision: 1, DeadlineTime: pointer.FromTime(time.Now().Add(-time.Minute)), } - _, err := str.GetCollection("tasks").InsertOne(ctx, stuckTask) - Expect(err).ToNot(HaveOccurred()) - - unstickConfig := &taskQueue.Config{Workers: 2, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: time.Millisecond, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault} - que = test.Must(taskQueue.New(taskTest.RandomType(), unstickConfig, lgr, str, countingRunner)) + test.Must(str.GetCollection("tasks").InsertOne(ctx, createdTask)) que.Start() @@ -573,15 +618,16 @@ var _ = Describe("Queue", func() { // Once unstuck, the task returns to pending and is dispatched, run, and completed. Eventually(func() string { - return test.Must(str.NewTaskRepository().GetTask(ctx, stuckTask.ID, nil)).State + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) }) It("reverts a task that is still running to pending when the queue is stopped", func() { - hangingRunner := taskQueueTest.NewHangingRunner(taskTest.RandomType(), time.Minute) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, hangingRunner)) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { <-ctx.Done() }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: hangingRunner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -600,10 +646,10 @@ var _ = Describe("Queue", func() { }) It("cancels the context of a task that exceeds its timeout", func() { - sleepRunner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), time.Minute, 100*time.Millisecond, 50*time.Millisecond, 10*time.Second) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, sleepRunner)) + runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), time.Minute, 100*time.Millisecond, 50*time.Millisecond, 10*time.Second) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: sleepRunner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -611,8 +657,6 @@ var _ = Describe("Queue", func() { return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State }, "1m", "50ms").To(Equal(task.TaskStateCompleted)) - Expect(countingRunner.GetCount()).To(Equal(0)) - actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) Expect(actualTask.Error).ToNot(BeNil()) @@ -620,10 +664,10 @@ var _ = Describe("Queue", func() { }) It("fails a task whose runner returns without setting a terminal state", func() { - noopRunner := taskQueueTest.NewCallbackRunner(taskTest.RandomType(), nil) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, noopRunner)) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: noopRunner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -638,10 +682,12 @@ var _ = Describe("Queue", func() { }) It("fails a task that exceeds its timeout without setting its own state", func() { - hangingRunner := taskQueueTest.NewHangingRunner(taskTest.RandomType(), 200*time.Millisecond) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, hangingRunner)) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDurationMaximum(100 * time.Millisecond). + WithStub(func(ctx context.Context, tsk *task.Task) { <-ctx.Done() }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: hangingRunner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -654,18 +700,15 @@ var _ = Describe("Queue", func() { Expect(actualTask.Error).ToNot(BeNil()) Expect(actualTask.Error.Error).To(MatchError("task runner timeout exceeded")) - // The runner returns after its timeout (well within the watchdog grace period), so - // the timeout is logged and counted as "recovered" during reconciliation rather than - // caught as "blocked" by the watchdog. lgr.AssertWarn("Task runner exceeded timeout; task will be failed") - Expect(testutil.ToFloat64(taskQueue.RunnerTimeoutExceededTotal.WithLabelValues(hangingRunner.GetRunnerType(), "recovered"))).To(Equal(float64(1))) + Expect(testutil.ToFloat64(taskQueue.RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType(), "recovered"))).To(Equal(float64(1))) }) It("logs a warning if a task that exceeds its maximum duration", func() { - sleepRunner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 2*time.Minute, time.Minute, time.Millisecond, 10*time.Millisecond) - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, sleepRunner)) + runner := taskQueueTest.NewSleepRunner(taskTest.RandomType(), 2*time.Minute, time.Minute, time.Millisecond, 10*time.Millisecond) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: sleepRunner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -673,8 +716,6 @@ var _ = Describe("Queue", func() { return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State }, "1m", "50ms").To(Equal(task.TaskStateCompleted)) - Expect(countingRunner.GetCount()).To(Equal(0)) - actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) Expect(actualTask.Error).To(BeNil()) @@ -683,7 +724,10 @@ var _ = Describe("Queue", func() { }) It("does not race or panic when Stop is called concurrently", func() { - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: countingRunner.GetRunnerType()})) + runner := taskQueueTest.NewCountingRunner(taskTest.RandomType()) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -692,7 +736,7 @@ var _ = Describe("Queue", func() { }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) var waitGroup sync.WaitGroup - for range 3 { + for range 10 { waitGroup.Go(func() { defer GinkgoRecover() que.Stop() @@ -702,19 +746,34 @@ var _ = Describe("Queue", func() { }) }) - Context("with a blocking runner", func() { - var blockingRunner *taskQueueTest.BlockingRunner + Context("without successful shutdown", func() { + var cfg *taskQueue.Config var que *taskQueue.Queue BeforeEach(func() { - blockingRunner = taskQueueTest.NewBlockingRunner(taskTest.RandomType(), time.Minute) + cfg = &taskQueue.Config{ + Workers: 2, + Delay: time.Millisecond, + DelayInitial: time.Millisecond, + DelayUnstick: taskQueue.DelayUnstickDefault, + StopWaitTimeout: 250 * time.Millisecond, + RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault, + } + }) - cfg := &taskQueue.Config{Workers: 1, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: 250 * time.Millisecond, RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault} - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, blockingRunner)) + AfterEach(func() { + if que != nil { + que.Stop() + } }) It("returns from Stop within the stop timeout when a runner ignores cancellation", func() { - createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: blockingRunner.GetRunnerType()})) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDurationMaximum(time.Minute). + WithStub(func(ctx context.Context, tsk *task.Task) { select {} }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) que.Start() @@ -735,24 +794,19 @@ var _ = Describe("Queue", func() { lgr.AssertError("Task queue workers did not stop within timeout; abandoning in-flight tasks; will be fixed with UnstickTasks later") }) - }) - Context("with a runner that exceeds its timeout without returning", func() { - var blockingRunner *taskQueueTest.BlockingRunner - var que *taskQueue.Queue + It("logs and counts the run once its timeout elapses", func() { + cfg.RunnerWatchdogGracePeriod = 50 * time.Millisecond - BeforeEach(func() { // A short duration maximum yields a short runner timeout (3x), and a short grace // period keeps the watchdog prompt, so the watchdog fires quickly while the runner // is still blocked, without an unstick reclaiming the task. - blockingRunner = taskQueueTest.NewBlockingRunner(taskTest.RandomType(), 20*time.Millisecond) - - cfg := &taskQueue.Config{Workers: 1, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: taskQueue.DelayUnstickDefault, StopWaitTimeout: 250 * time.Millisecond, RunnerWatchdogGracePeriod: 50 * time.Millisecond} - que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, blockingRunner)) - }) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithDurationMaximum(20 * time.Millisecond). + WithStub(func(ctx context.Context, tsk *task.Task) { select {} }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) - It("logs and counts the run once its timeout elapses", func() { - runnerType := blockingRunner.GetRunnerType() + runnerType := runner.GetRunnerType() taskQueue.RunnerTimeoutExceededTotal.Reset() createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runnerType})) @@ -786,13 +840,21 @@ var _ = Describe("Queue", func() { const workersCount = 10 const taskCount = 2 * queueCount * workersCount - var runner *taskQueueTest.RepeatRunner + var runner *taskQueueTest.StubRunner var ques []*taskQueue.Queue BeforeEach(func() { - runner = taskQueueTest.NewRepeatRunner(taskTest.RandomType()) - - cfg := &taskQueue.Config{Workers: workersCount, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: time.Millisecond, StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault} + runner = taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { tsk.State = task.TaskStatePending }) + + cfg := &taskQueue.Config{ + Workers: workersCount, + Delay: time.Millisecond, + DelayInitial: time.Millisecond, + DelayUnstick: time.Millisecond, + StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, + RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault, + } ques = make([]*taskQueue.Queue, queueCount) for index := range len(ques) { ques[index] = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -805,7 +867,7 @@ var _ = Describe("Queue", func() { } }) - It("completes all runnings tasks when stopped", func() { + It("completes all running tasks when stopped", func() { tasks := make(task.Tasks, taskCount) for index := range len(tasks) { tasks[index] = test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) @@ -831,7 +893,7 @@ var _ = Describe("Queue", func() { }) It("eventually a queue attempts to run a task that is already running in another queue", func() { - tsk := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) for _, que := range ques { que.Start() @@ -847,7 +909,7 @@ var _ = Describe("Queue", func() { que.Stop() } - actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, tsk.ID, nil)) + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) Expect(actualTask.State).To(Equal(task.TaskStatePending)) }) }) diff --git a/task/queue/test/runner.go b/task/queue/test/runner.go index dfc4d396a9..b9ddc69ba9 100644 --- a/task/queue/test/runner.go +++ b/task/queue/test/runner.go @@ -55,38 +55,6 @@ func (c *CountingRunner) GetCount() int { var _ queue.Runner = &CountingRunner{} -type PanicRunner struct { - Type string -} - -func NewPanicRunner(typ string) *PanicRunner { - return &PanicRunner{ - Type: typ, - } -} - -func (p *PanicRunner) GetRunnerType() string { - return p.Type -} - -func (p *PanicRunner) GetRunnerDeadline() time.Duration { - return p.GetRunnerDurationMaximum() * 5 -} - -func (p *PanicRunner) GetRunnerTimeout() time.Duration { - return p.GetRunnerDurationMaximum() * 3 -} - -func (p *PanicRunner) GetRunnerDurationMaximum() time.Duration { - return time.Second -} - -func (p *PanicRunner) Run(ctx context.Context, tsk *task.Task) { - panic("panic test") -} - -var _ queue.Runner = &PanicRunner{} - type SleepRunner struct { Type string Deadline time.Duration @@ -133,144 +101,72 @@ func (s *SleepRunner) Run(ctx context.Context, tsk *task.Task) { var _ queue.Runner = &SleepRunner{} -type RepeatRunner struct { - Type string +type StubRunner struct { + Type string + Stub func(ctx context.Context, tsk *task.Task) + deadline *time.Duration + timeout *time.Duration + durationMaximum *time.Duration } -func NewRepeatRunner(typ string) *RepeatRunner { - return &RepeatRunner{ +func NewStubRunner(typ string) *StubRunner { + return &StubRunner{ Type: typ, } } -func (r *RepeatRunner) GetRunnerType() string { - return r.Type -} - -func (r *RepeatRunner) GetRunnerDeadline() time.Duration { - return r.GetRunnerDurationMaximum() * 5 -} - -func (r *RepeatRunner) GetRunnerTimeout() time.Duration { - return r.GetRunnerDurationMaximum() * 3 -} - -func (r *RepeatRunner) GetRunnerDurationMaximum() time.Duration { - return time.Minute -} - -func (r *RepeatRunner) Run(ctx context.Context, tsk *task.Task) { - tsk.State = task.TaskStatePending -} - -var _ queue.Runner = &RepeatRunner{} - -type HangingRunner struct { - Type string - Timeout time.Duration +func (s *StubRunner) GetRunnerType() string { + return s.Type } -func NewHangingRunner(typ string, timeout time.Duration) *HangingRunner { - return &HangingRunner{ - Type: typ, - Timeout: timeout, +func (s *StubRunner) GetRunnerDeadline() time.Duration { + if s.deadline != nil { + return *s.deadline + } else { + return s.GetRunnerDurationMaximum() * 4 } } -func (h *HangingRunner) GetRunnerType() string { - return h.Type -} - -func (h *HangingRunner) GetRunnerDeadline() time.Duration { - return h.Timeout * 5 -} - -func (h *HangingRunner) GetRunnerTimeout() time.Duration { - return h.Timeout -} - -func (h *HangingRunner) GetRunnerDurationMaximum() time.Duration { - return h.Timeout / 2 -} - -// Run blocks until its context is canceled and returns without setting a terminal -// state, leaving the task running to exercise shutdown interruption and timeout handling. -func (h *HangingRunner) Run(ctx context.Context, tsk *task.Task) { - <-ctx.Done() -} - -var _ queue.Runner = &HangingRunner{} - -type BlockingRunner struct { - Type string - DurationMaximum time.Duration -} - -func NewBlockingRunner(typ string, durationMaximum time.Duration) *BlockingRunner { - return &BlockingRunner{ - Type: typ, - DurationMaximum: durationMaximum, +func (s *StubRunner) GetRunnerTimeout() time.Duration { + if s.timeout != nil { + return *s.timeout + } else { + return s.GetRunnerDurationMaximum() * 2 } } -func (b *BlockingRunner) GetRunnerType() string { - return b.Type -} - -func (b *BlockingRunner) GetRunnerDeadline() time.Duration { - return b.GetRunnerDurationMaximum() * 5 -} - -func (b *BlockingRunner) GetRunnerTimeout() time.Duration { - return b.GetRunnerDurationMaximum() * 3 -} - -func (b *BlockingRunner) GetRunnerDurationMaximum() time.Duration { - return b.DurationMaximum -} - -// Run blocks forever, ignoring context cancellation entirely, to simulate a runner that -// does not honor shutdown. Used to verify Stop returns within its timeout regardless. -func (b *BlockingRunner) Run(ctx context.Context, tsk *task.Task) { - select {} -} - -var _ queue.Runner = &BlockingRunner{} - -type CallbackRunner struct { - Type string - Callback func(ctx context.Context, tsk *task.Task) -} - -func NewCallbackRunner(typ string, callback func(ctx context.Context, tsk *task.Task)) *CallbackRunner { - return &CallbackRunner{ - Type: typ, - Callback: callback, +func (s *StubRunner) GetRunnerDurationMaximum() time.Duration { + if s.durationMaximum != nil { + return *s.durationMaximum + } else { + return time.Minute } } -func (c *CallbackRunner) GetRunnerType() string { - return c.Type +func (s *StubRunner) Run(ctx context.Context, tsk *task.Task) { + if s.Stub != nil { + s.Stub(ctx, tsk) + } } -func (c *CallbackRunner) GetRunnerDeadline() time.Duration { - return c.GetRunnerDurationMaximum() * 5 +func (s *StubRunner) WithStub(stub func(ctx context.Context, tsk *task.Task)) *StubRunner { + s.Stub = stub + return s } -func (c *CallbackRunner) GetRunnerTimeout() time.Duration { - return c.GetRunnerDurationMaximum() * 3 +func (s *StubRunner) WithDeadline(deadline time.Duration) *StubRunner { + s.deadline = &deadline + return s } -func (c *CallbackRunner) GetRunnerDurationMaximum() time.Duration { - return time.Minute +func (s *StubRunner) WithTimeout(timeout time.Duration) *StubRunner { + s.timeout = &timeout + return s } -// Run invokes the callback, letting a test inject arbitrary behavior (such as updating the -// task while it is running) into the task run. -func (c *CallbackRunner) Run(ctx context.Context, tsk *task.Task) { - if c.Callback != nil { - c.Callback(ctx, tsk) - } +func (s *StubRunner) WithDurationMaximum(durationMaximum time.Duration) *StubRunner { + s.durationMaximum = &durationMaximum + return s } -var _ queue.Runner = &CallbackRunner{} +var _ queue.Runner = &StubRunner{} diff --git a/task/store/mongo/mongo.go b/task/store/mongo/mongo.go index 2aeb33eeb3..64003cce77 100644 --- a/task/store/mongo/mongo.go +++ b/task/store/mongo/mongo.go @@ -177,7 +177,7 @@ func (t *TaskRepository) ensureTask(ctx context.Context, create *task.TaskCreate if result, err := t.UpdateOne(ctx, bson.M{"name": tsk.Name}, bson.M{"$setOnInsert": tsk}, options.Update().SetUpsert(true)); err != nil { return errors.Wrap(err, "unable to create task") } else if result.UpsertedCount > 0 { - TasksStateTotal.WithLabelValues(task.TaskStatePending, create.Type).Inc() + TypeStateTotal.WithLabelValues(create.Type, task.TaskStatePending).Inc() } return nil @@ -275,7 +275,7 @@ func (t *TaskRepository) CreateTask(ctx context.Context, create *task.TaskCreate logger = logger.WithField("task", tsk.LogFields()) - TasksStateTotal.WithLabelValues(task.TaskStatePending, create.Type).Inc() + TypeStateTotal.WithLabelValues(create.Type, task.TaskStatePending).Inc() return tsk, nil } @@ -425,12 +425,12 @@ func (t *TaskRepository) StartTask(ctx context.Context, id string, revision int, return nil, errors.Wrap(err, "unable to start task") } - TasksStateTotal.WithLabelValues(task.TaskStateRunning, tsk.Type).Inc() + TypeStateTotal.WithLabelValues(tsk.Type, task.TaskStateRunning).Inc() return tsk, nil } // Will only timeout after 10 seconds even if parent context is canceled. -func (t *TaskRepository) StopTask(ctx context.Context, id string, stateLock *string, state string, duration *time.Duration, update *task.TaskUpdate) error { +func (t *TaskRepository) StopTask(ctx context.Context, id string, revision int, stateLock *string, state string, duration *time.Duration, update *task.TaskUpdate) error { if ctx == nil { return errors.New("context is missing") } @@ -462,7 +462,7 @@ func (t *TaskRepository) StopTask(ctx context.Context, id string, stateLock *str ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), TransitionTimeout) defer cancel() - logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "stateLock": stateLock, "state": state, "duration": duration, "update": update}) + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "revision": revision, "stateLock": stateLock, "state": state, "duration": duration, "update": update}) now := time.Now().UTC() defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("StopTask") }() @@ -495,14 +495,23 @@ func (t *TaskRepository) StopTask(ctx context.Context, id string, stateLock *str // recover the task if it was left running. This is logged and counted so lost // completions are observable rather than silently swallowed. logger.Error("Unable to stop task; no running task matched the expected condition") - TasksLostCompletionTotal.WithLabelValues(pointer.Default(t.typeFilter, "")).Inc() + TypeLostCompletionTotal.WithLabelValues(pointer.Default(t.typeFilter, "")).Inc() return nil } else if err != nil { logger = logger.WithError(err) return errors.Wrap(err, "unable to stop task") } - TasksStateTotal.WithLabelValues(state, tsk.Type).Inc() + // If the on-disk task revision does not match the runner task revision, then either: + // - the runner did not follow the Runner contract; i.e. the runner updated the task during + // run, but it did not use the updated task, or, + // - the task was concurrently modified outside of the runner while the task was running. + if tsk.Revision != revision { + logger.WithField("revision", log.Fields{"expected": revision, "actual": tsk.Revision}).Warn("Database task revision does not match running task revision; Runner contract broken or concurrent update") + TypeRevisionMismatchTotal.WithLabelValues(tsk.Type).Inc() + } + + TypeStateTotal.WithLabelValues(tsk.Type, state).Inc() return nil } @@ -658,18 +667,28 @@ func newStateLock() string { } var ( - TasksStateTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tidepool_task_tasks_state_total", - Help: "The total number of tasks sorted by state and type", - }, []string{"state", "type"}) - - // TasksLostCompletionTotal counts task completions dropped because the compare-and-swap in - // StopTask missed (the running claim was concurrently modified, unstuck, or deleted). The task + // TypeStateTotal counts the total number of tasks run, sorted by type and state. + TypeStateTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_type_state_total", + Help: "The total number of tasks run, sorted by type and state", + }, []string{"type", "state"}) + + // TypeLostCompletionTotal counts task completions dropped because the compare-and-swap in + // StopTask missed (task state lock was concurrently modified, task unstuck, or task deleted). The task // is recovered by the deadline/unstick mechanism, but the intended terminal state is lost. The // type label is populated only when the repository is type-filtered (as it is for each queue in // a MultiQueue); otherwise it is empty. - TasksLostCompletionTotal = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "tidepool_task_lost_completion_total", + TypeLostCompletionTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_type_lost_completion_total", Help: "The total number of task completions dropped because the state-lock compare-and-swap missed, sorted by type", }, []string{"type"}) + + // TypeRevisionMismatchTotal counts task completions where the task revision does not match + // the task revision in the database. This only occurs if the task runner does not follow the Runner + // contract when updating a task during run, or if there is a concurrent modification outside of + // the expected Runner behavior. + TypeRevisionMismatchTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_type_revision_mismatch_total", + Help: "The total number of task revisions that do not match the task revision in the database, sorted by type", + }, []string{"type"}) ) diff --git a/task/store/mongo/mongo_test.go b/task/store/mongo/mongo_test.go index ed6cc71c52..35006ba2eb 100644 --- a/task/store/mongo/mongo_test.go +++ b/task/store/mongo/mongo_test.go @@ -229,7 +229,7 @@ var _ = Describe("Mongo", func() { Expect(startedTask.RunTime).ToNot(BeNil()) Expect(startedTask.AvailableTime).To(BeNil()) - Expect(repository.StopTask(ctx, startedTask.ID, startedTask.StateLock, task.TaskStatePending, nil, nil)).To(Succeed()) + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.StateLock, task.TaskStatePending, nil, nil)).To(Succeed()) actualTask := &task.Task{} Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) @@ -250,7 +250,7 @@ var _ = Describe("Mongo", func() { Expect(startedTask.RunTime).ToNot(BeNil()) duration := time.Second - Expect(repository.StopTask(ctx, startedTask.ID, startedTask.StateLock, task.TaskStateCompleted, &duration, nil)).To(Succeed()) + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.StateLock, task.TaskStateCompleted, &duration, nil)).To(Succeed()) actualTask := &task.Task{} Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) @@ -265,7 +265,7 @@ var _ = Describe("Mongo", func() { Context("EnsureEHRReconcileTask", func() { BeforeEach(func() { - taskStoreMongo.TasksStateTotal.Reset() + taskStoreMongo.TypeStateTotal.Reset() }) It("creates the task and increments the pending metric only on the initial insert", func() { @@ -273,10 +273,10 @@ var _ = Describe("Mongo", func() { Expect(repository).ToNot(BeNil()) Expect(repository.EnsureEHRReconcileTask(ctx)).To(Succeed()) - Expect(testutil.ToFloat64(taskStoreMongo.TasksStateTotal)).To(Equal(1.0)) + Expect(testutil.ToFloat64(taskStoreMongo.TypeStateTotal)).To(Equal(1.0)) Expect(repository.EnsureEHRReconcileTask(ctx)).To(Succeed()) - Expect(testutil.ToFloat64(taskStoreMongo.TasksStateTotal)).To(Equal(1.0)) + Expect(testutil.ToFloat64(taskStoreMongo.TypeStateTotal)).To(Equal(1.0)) count := test.Must(collection.CountDocuments(context.Background(), bson.M{"type": reconcile.Type})) Expect(count).To(Equal(int64(1))) diff --git a/task/store/store.go b/task/store/store.go index 8df4b02174..e2f45fd096 100644 --- a/task/store/store.go +++ b/task/store/store.go @@ -27,7 +27,7 @@ type TaskRepository interface { UnstickTasks(ctx context.Context) ([]string, error) StartTask(ctx context.Context, id string, revision int, deadline time.Duration) (*task.Task, error) - StopTask(ctx context.Context, id string, stateLock *string, state string, duration *time.Duration, update *task.TaskUpdate) error + StopTask(ctx context.Context, id string, revision int, stateLock *string, state string, duration *time.Duration, update *task.TaskUpdate) error IteratePending(ctx context.Context) (*mongo.Cursor, error) } From 62c2b0f0157782a16e05d019ba3d42d53e3c57e4 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Fri, 24 Jul 2026 23:39:17 -0700 Subject: [PATCH 11/20] Add checks for task claims that have been lost - Add checks for task claims that have been lost - Update Dexcom task runner to correctly update last import time - Update Dexcom task runner to correctly reschedule - Fix delay constants - Update comments - Update tests --- dexcom/fetch/runner.go | 172 ++++++------- dexcom/fetch/runner_test.go | 56 ++++- task/queue/multi.go | 13 +- task/queue/multi_test.go | 59 ++--- task/queue/queue.go | 437 ++++++++++++++++++++------------- task/queue/queue_test.go | 388 +++++++++++++++++++++-------- task/queue/runner.go | 154 ++++++------ task/store/mongo/mongo.go | 158 +++++++----- task/store/mongo/mongo_test.go | 321 ++++++++++++++++++------ task/store/store.go | 7 +- task/task.go | 23 +- task/task_test.go | 12 +- 12 files changed, 1165 insertions(+), 635 deletions(-) diff --git a/dexcom/fetch/runner.go b/dexcom/fetch/runner.go index 28b0f4406b..be6f84a2ef 100644 --- a/dexcom/fetch/runner.go +++ b/dexcom/fetch/runner.go @@ -2,13 +2,13 @@ package fetch import ( "context" - "math/rand" "net/http" "sort" "strconv" "time" "github.com/tidepool-org/platform/auth" + "github.com/tidepool-org/platform/crypto" "github.com/tidepool-org/platform/data" dataDeduplicatorDeduplicator "github.com/tidepool-org/platform/data/deduplicator/deduplicator" dataSource "github.com/tidepool-org/platform/data/source" @@ -150,7 +150,9 @@ type TaskRunner struct { deviceHashes map[string]string dataSet *data.DataSet dataSetPreloaded bool - deadline time.Time + runTime time.Time + availableAfter *time.Duration + importCompleted bool } func NewTaskRunner(provider Provider, tsk *task.Task) (*TaskRunner, error) { @@ -170,21 +172,29 @@ func NewTaskRunner(provider Provider, tsk *task.Task) (*TaskRunner, error) { func (t *TaskRunner) Run(ctx context.Context) { t.context = ctx t.logger = log.LoggerFromContext(t.context) - t.deadline = time.Now().Add(t.GetRunnerDurationMaximum()) + t.runTime = time.Now() t.task.ClearError() - if err := t.run(); err == nil { - t.rescheduleTask() - } else if !t.task.HasError() { - t.rescheduleTaskWithResourceError(err) + + if err := t.run(); err != nil { + t.task.AppendError(err) + } + + // A permanently failed task is not rescheduled, unless its outcome could not be recorded on the data source, in + // which case run again so a later run can record it + err := t.updateDataSourceWithTaskState() + if err != nil { + t.task.AppendError(err) + } + if err != nil || !t.task.IsFailed() { + t.task.RepeatAvailableAfter(pointer.Default(t.availableAfter, availableAfterDuration())) } } func (t *TaskRunner) run() error { - defer t.updateDataSourceWithTaskState() - if len(t.task.Data) == 0 { - return t.failTaskWithInvalidStateError(errors.New("data is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data is missing")) } if err := t.getDataSource(); err != nil { @@ -202,9 +212,6 @@ func (t *TaskRunner) run() error { if err := t.fetchSinceLatestDataTime(); err != nil { return err } - if err := t.updateDataSourceWithLastImportTime(); err != nil { - return err - } return nil } @@ -212,14 +219,16 @@ func (t *TaskRunner) run() error { func (t *TaskRunner) getProviderSession() error { providerSessionID, ok := t.task.Data[dexcom.DataKeyProviderSessionID].(string) if !ok || providerSessionID == "" { - return t.failTaskWithInvalidStateError(errors.New("provider session id is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("provider session id is missing")) } providerSession, err := t.AuthClient().GetProviderSession(t.context, providerSessionID) if err != nil { - return t.rescheduleTaskWithResourceError(errors.Wrap(err, "unable to get provider session")) + return ErrorResourceFailureError(errors.Wrap(err, "unable to get provider session")) } else if providerSession == nil { - return t.failTaskWithInvalidStateError(errors.New("provider session is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("provider session is missing")) } t.providerSession = providerSession @@ -250,14 +259,16 @@ func (t *TaskRunner) updateProviderSession() error { func (t *TaskRunner) getDataSource() error { dataSourceID, ok := t.task.Data[dexcom.DataKeyDataSourceID].(string) if !ok || dataSourceID == "" { - return t.failTaskWithInvalidStateError(errors.New("data source id is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data source id is missing")) } source, err := t.DataSourceClient().Get(t.context, dataSourceID) if err != nil { - return t.rescheduleTaskWithResourceError(errors.Wrap(err, "unable to get data source")) + return ErrorResourceFailureError(errors.Wrap(err, "unable to get data source")) } else if source == nil { - return t.failTaskWithInvalidStateError(errors.New("data source is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data source is missing")) } t.dataSource = source @@ -282,14 +293,11 @@ func (t *TaskRunner) updateDataSourceWithDataTime(earliestDataTime *time.Time, l return t.updateDataSource(update) } -func (t *TaskRunner) updateDataSourceWithLastImportTime() error { - update := dataSource.NewUpdate() - update.LastImportTime = pointer.FromTime(time.Now()) - return t.updateDataSource(update) -} - func (t *TaskRunner) updateDataSourceWithTaskState() error { update := dataSource.NewUpdate() + if t.importCompleted { + update.LastImportTime = pointer.FromTime(time.Now()) + } if t.task.IsFailed() { update.State = pointer.FromString(dataSource.StateError) } @@ -305,9 +313,10 @@ func (t *TaskRunner) updateDataSource(update *dataSource.Update) error { // Without cancel to ensure data source is updated in the database dataSource, err := t.DataSourceClient().Update(context.WithoutCancel(t.context), t.dataSource.ID, nil, update) if err != nil { - return t.rescheduleTaskWithResourceError(errors.WithMeta(errors.Wrap(err, "unable to update data source"), update)) + return ErrorResourceFailureError(errors.WithMeta(errors.Wrap(err, "unable to update data source"), update)) } else if dataSource == nil { - return t.failTaskWithInvalidStateError(errors.New("data source is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data source is missing")) } t.dataSource = dataSource @@ -317,7 +326,8 @@ func (t *TaskRunner) updateDataSource(update *dataSource.Update) error { func (t *TaskRunner) createTokenSource() error { tokenSource, err := oauthToken.NewSourceWithToken(t.providerSession.OAuthToken) if err != nil { - return t.failTaskWithInvalidStateError(errors.Wrap(err, "unable to create token source")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.Wrap(err, "unable to create token source")) } t.tokenSource = tokenSource @@ -329,16 +339,18 @@ func (t *TaskRunner) getDeviceHashes() error { if !rawOK || raw == nil { return nil } - rawMap, rawMapOK := raw.(map[string]interface{}) + rawMap, rawMapOK := raw.(map[string]any) if !rawMapOK || rawMap == nil { - return t.failTaskWithInvalidStateError(errors.New("device hashes is invalid")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("device hashes is invalid")) } deviceHashes := map[string]string{} for key, value := range rawMap { if valueString, valueStringOK := value.(string); valueStringOK { deviceHashes[key] = valueString } else { - return t.failTaskWithInvalidStateError(errors.New("device hash is invalid")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("device hash is invalid")) } } @@ -381,9 +393,10 @@ func (t *TaskRunner) updateDataSet(update *data.DataSetUpdate) error { // Without cancel to ensure data set is updated in the database dataSet, err := t.DataClient().UpdateDataSet(context.WithoutCancel(t.context), *t.dataSet.UploadID, update) if err != nil { - return t.rescheduleTaskWithResourceError(errors.WithMeta(errors.Wrap(err, "unable to update data set"), update)) + return ErrorResourceFailureError(errors.WithMeta(errors.Wrap(err, "unable to update data set"), update)) } else if dataSet == nil { - return t.failTaskWithInvalidStateError(errors.New("data set is missing")) + t.task.SetFailed() + return ErrorInvalidStateError(errors.New("data set is missing")) } t.dataSet = dataSet @@ -395,7 +408,8 @@ func (t *TaskRunner) fetchSinceLatestDataTime() error { if err != nil { return err } else if dataRange == nil { - return nil // Nothing to fetch + t.importCompleted = true // No data, but still successful import + return nil } startTime := dataRange.StartTime @@ -409,20 +423,20 @@ func (t *TaskRunner) fetchSinceLatestDataTime() error { return err } - // If past deadline (based upon runner maximum duration), then bail - if time.Now().After(t.deadline) { - t.rescheduleTaskNow() + // If the task has been running for longer than the maximum duration, then stop fetching and repeat after a minute + if time.Since(t.runTime) > t.GetRunnerDurationMaximum() { + t.availableAfter = pointer.From(time.Minute) return nil } startTime = startTime.AddDate(0, 0, DataRangeDaysMaximum) } + t.importCompleted = true return nil } func (t *TaskRunner) fetchDataRange() (*DataRange, error) { - // HACK: Dexcom V3 (2024-05-30) - Can only use latest data time as last sync time if not // older than 100 days, otherwise will return erroneous results. Use 30 days to be on // the safe side. @@ -702,7 +716,7 @@ func (t *TaskRunner) findDataSet() (*data.DataSet, error) { } dataSet, err := t.DataClient().GetDataSet(t.context, *t.dataSource.DataSetID) if err != nil { - return nil, t.rescheduleTaskWithResourceError(errors.Wrap(err, "unable to get data set")) + return nil, ErrorResourceFailureError(errors.Wrap(err, "unable to get data set")) } return dataSet, nil } @@ -724,7 +738,7 @@ func (t *TaskRunner) createDataSet() (*data.DataSet, error) { dataSet, err := t.DataClient().CreateUserDataSet(t.context, t.providerSession.UserID, dataSetCreate) if err != nil { - return nil, t.rescheduleTaskWithResourceError(errors.WithMeta(errors.Wrap(err, "unable to create data set"), dataSetCreate)) + return nil, ErrorResourceFailureError(errors.WithMeta(errors.Wrap(err, "unable to create data set"), dataSetCreate)) } if err = t.updateDataSourceWithDataSet(dataSet); err != nil { return nil, err @@ -744,7 +758,7 @@ func (t *TaskRunner) storeDatumArray(datumArray data.Data) error { partialDatumArray := datumArray[startIndex:endIndex] if err := t.DataClient().CreateDataSetsData(t.context, *t.dataSet.UploadID, partialDatumArray); err != nil { - return t.rescheduleTaskWithResourceError(errors.Wrap(err, "unable to create data set data")) + return ErrorResourceFailureError(errors.Wrap(err, "unable to create data set data")) } earliestDataTime := partialDatumArray[0].GetTime() @@ -778,66 +792,27 @@ func (t *TaskRunner) afterLatestDataTime(latestDataTime *time.Time) bool { return latestDataTime != nil && (t.dataSource.LatestDataTime == nil || latestDataTime.After(*t.dataSource.LatestDataTime)) } -// Handle potential dexcom client error. If error, then retry or reschedule. -// Otherwise, reset retry count. func (t *TaskRunner) handleDexcomClientError(err error) error { - if err != nil { - return t.retryOrRescheduleTaskWithDexcomClientError(err) - } else { - return t.resetTaskRetryCount() + // If success, then reset retry count and return no error + if err == nil { + t.resetTaskRetryCount() + return nil } -} -// Retry task if Dexcom authentication failure. Otherwise, reschedule task. -func (t *TaskRunner) retryOrRescheduleTaskWithDexcomClientError(err error) error { - if request.IsErrorUnauthenticated(errors.Cause(err)) { - return t.retryTaskWithError(ErrorAuthenticationFailureError(err)) - } else { - return t.rescheduleTaskWithResourceError(err) + // If not an authentication error, then just treat as a resource failure + if !request.IsErrorUnauthenticated(errors.Cause(err)) { + return ErrorResourceFailureError(err) } -} -// Increment task retry count. If task retry count exceeds maximum, then fail task. -// Otherwise, reschedule task. Typically used for Dexcom authentication failures that -// may or may not be transient. -func (t *TaskRunner) retryTaskWithError(err error) error { - retryCount := t.incrementTaskRetryCount() - if retryCount > TaskRetryCountMaximum { - return t.failTaskWithError(err) + // It is an authentication error, attempt retry, if possible + err = ErrorAuthenticationFailureError(err) + if retryCount := t.incrementTaskRetryCount(); retryCount <= TaskRetryCountMaximum { + t.availableAfter = pointer.From(availableAfterDurationWithRetryCount(retryCount)) + return err } - t.task.AppendError(err) - t.task.RepeatAvailableAfter(availableAfterDurationWithFallbackFactor(fallbackFactorWithRetryCount(retryCount))) - return err -} - -func (t *TaskRunner) rescheduleTaskWithResourceError(err error) error { - return t.rescheduleTaskWithError(ErrorResourceFailureError(err)) -} - -// Reschedule task for next run. Append error to task. -func (t *TaskRunner) rescheduleTaskWithError(err error) error { - t.task.AppendError(err) - t.rescheduleTask() - return err -} - -func (t *TaskRunner) rescheduleTaskNow() { - t.task.RepeatAvailableAfter(0) -} - -func (t *TaskRunner) rescheduleTask() { - t.task.RepeatAvailableAfter(availableAfterDuration()) -} - -func (t *TaskRunner) failTaskWithInvalidStateError(err error) error { - return t.failTaskWithError(ErrorInvalidStateError(err)) -} - -// Fail task immediately and permanently. Do not reschedule. For situations where any future attempt is -// also guaranteed to fail. For example, when the task data is missing information. Should not normally happen. -func (t *TaskRunner) failTaskWithError(err error) error { - t.task.SetFailedWithError(err) + // Otherwise, we are failed + t.task.SetFailed() return err } @@ -852,9 +827,8 @@ func (t *TaskRunner) incrementTaskRetryCount() int { return retryCount } -func (t *TaskRunner) resetTaskRetryCount() error { +func (t *TaskRunner) resetTaskRetryCount() { delete(t.task.Data, dexcom.DataKeyRetryCount) - return nil } func (t *TaskRunner) HTTPClient(ctx context.Context, tokenSourceSource oauth.TokenSourceSource) (*http.Client, error) { @@ -906,8 +880,12 @@ func availableAfterDuration() time.Duration { return availableAfterDurationWithFallbackFactor(1) } +func availableAfterDurationWithRetryCount(retryCount int) time.Duration { + return availableAfterDurationWithFallbackFactor(fallbackFactorWithRetryCount(retryCount)) +} + func availableAfterDurationWithFallbackFactor(fallbackFactor float64) time.Duration { - return time.Duration(float64(AvailableAfterDuration)*fallbackFactor) + time.Duration(rand.Int63n(int64(2*AvailableAfterDurationJitter))) - AvailableAfterDurationJitter + return time.Duration(float64(AvailableAfterDuration)*fallbackFactor) + time.Duration(crypto.RandomInt64N(int64(2*AvailableAfterDurationJitter))) - AvailableAfterDurationJitter } func fallbackFactorWithRetryCount(retryCount int) float64 { diff --git a/dexcom/fetch/runner_test.go b/dexcom/fetch/runner_test.go index 70c632800c..fb6ae562fd 100644 --- a/dexcom/fetch/runner_test.go +++ b/dexcom/fetch/runner_test.go @@ -141,15 +141,17 @@ var _ = Describe("Runner", func() { Context("with provider and task", func() { var provider *dexcomFetchTest.MockProvider + var runnerDurationMaximum time.Duration var tsk *task.Task BeforeEach(func() { provider = dexcomFetchTest.NewMockProvider(mockController) + runnerDurationMaximum = time.Second provider.EXPECT().AuthClient().Return(authClient).AnyTimes() provider.EXPECT().DataClient().Return(dataClient).AnyTimes() provider.EXPECT().DataSourceClient().Return(dataSourceClient).AnyTimes() provider.EXPECT().DexcomClient().Return(dexcomClient).AnyTimes() - provider.EXPECT().GetRunnerDurationMaximum().Return(time.Second).AnyTimes() + provider.EXPECT().GetRunnerDurationMaximum().DoAndReturn(func() time.Duration { return runnerDurationMaximum }).AnyTimes() tsk = &task.Task{ State: task.TaskStateRunning, Data: map[string]any{ @@ -206,6 +208,17 @@ var _ = Describe("Runner", func() { } } + assertTaskAvailableSoon := func() { + Expect(tsk.AvailableTime).ToNot(BeNil()) + Expect(*tsk.AvailableTime).To(BeTemporally("~", time.Now().Add(time.Minute), 5*time.Second)) + } + + assertTaskAvailableAfterStandardDuration := func() { + Expect(tsk.AvailableTime).ToNot(BeNil()) + Expect(*tsk.AvailableTime).To(BeTemporally(">", time.Now().Add(dexcomFetch.AvailableAfterDuration-dexcomFetch.AvailableAfterDurationJitter-time.Second))) + Expect(*tsk.AvailableTime).To(BeTemporally("<", time.Now().Add(dexcomFetch.AvailableAfterDuration+dexcomFetch.AvailableAfterDurationJitter))) + } + assertTaskRetryCount := func(retryCount int) { Expect(tsk.Data[dexcom.DataKeyRetryCount]).To(Equal(int32(retryCount))) } @@ -474,7 +487,6 @@ var _ = Describe("Runner", func() { It("is successful if the Dexcom data ranges is not valid", func() { dataRangeResponse.Calibrations.Start = nil - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -484,7 +496,6 @@ var _ = Describe("Runner", func() { It("is successful if the Dexcom data ranges start is not before end", func() { dataRangeResponse.Calibrations.Start = &dexcom.Moment{SystemTime: &dexcom.Time{Time: time.Now().Add(-2 * Day)}} - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) assertTaskRetryCountNotPresent() @@ -571,16 +582,53 @@ var _ = Describe("Runner", func() { ID: pointer.FromString("test-data-set-id"), UploadID: pointer.FromString("test-data-set-upload-id"), } - dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(3) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(2) dataClient.EXPECT().CreateUserDataSet(matchContext(), "test-user-id", matchNotNil()).DoAndReturn(mockDataClientCreateUserDataSet(dataSet, nil)) dataClient.EXPECT().CreateDataSetsData(matchContext(), "test-data-set-upload-id", matchNotNil()).DoAndReturn(mockDataClientCreateDataSetsData(nil)) taskRunner.Run(ctx) assertTaskAndDataSourceState(task.TaskStatePending) + assertTaskAvailableAfterStandardDuration() assertTaskDeviceHashesCount(3) assertTaskRetryCountNotPresent() assertTaskAndDataSourceErrorNotPresent() assertProviderSessionRefreshedTimes(6) }) + + It("is available soon if the deadline is exceeded", func() { + runnerDurationMaximum = -time.Second + dataSet := &data.DataSet{ + ID: pointer.FromString("test-data-set-id"), + UploadID: pointer.FromString("test-data-set-upload-id"), + } + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)).Times(2) + dataClient.EXPECT().CreateUserDataSet(matchContext(), "test-user-id", matchNotNil()).DoAndReturn(mockDataClientCreateUserDataSet(dataSet, nil)) + dataClient.EXPECT().CreateDataSetsData(matchContext(), "test-data-set-upload-id", matchNotNil()).DoAndReturn(mockDataClientCreateDataSetsData(nil)) + taskRunner.Run(ctx) + assertTaskAndDataSourceState(task.TaskStatePending) + assertTaskAvailableSoon() + assertTaskRetryCountNotPresent() + assertTaskAndDataSourceErrorNotPresent() + assertProviderSessionRefreshedTimes(6) + }) + + It("is available soon if the deadline is exceeded and a later update fails", func() { + runnerDurationMaximum = -time.Second + testErr := errorsTest.RandomError() + dataSet := &data.DataSet{ + ID: pointer.FromString("test-data-set-id"), + UploadID: pointer.FromString("test-data-set-upload-id"), + } + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).Return(nil, testErr).Times(1) + dataClient.EXPECT().CreateUserDataSet(matchContext(), "test-user-id", matchNotNil()).DoAndReturn(mockDataClientCreateUserDataSet(dataSet, nil)) + dataClient.EXPECT().CreateDataSetsData(matchContext(), "test-data-set-upload-id", matchNotNil()).DoAndReturn(mockDataClientCreateDataSetsData(nil)) + taskRunner.Run(ctx) + assertTaskState(task.TaskStatePending) + assertTaskAvailableSoon() + assertTaskRetryCountNotPresent() + assertTaskError(dexcomFetch.ErrorCodeResourceFailure, "unable to update data source") + assertProviderSessionRefreshedTimes(6) + }) }) }) diff --git a/task/queue/multi.go b/task/queue/multi.go index 19472f6259..273f719b3e 100644 --- a/task/queue/multi.go +++ b/task/queue/multi.go @@ -9,9 +9,8 @@ import ( "github.com/tidepool-org/platform/task/store" ) -// MultiQueue runs a queue per runner provided at construction, each processing only tasks of -// that runner's type. Like Queue, it is single-use. The queues map is immutable after -// NewMultiQueue, so no synchronization is required. +// MultiQueue runs a queue per runner provided at construction, each processing only tasks of that runner's type. Like +// Queue, it is single-use. The queues map is immutable after NewMultiQueue, so no synchronization is required. type MultiQueue struct { queues map[string]*Queue } @@ -58,8 +57,8 @@ func (m *MultiQueue) Start() { } func (m *MultiQueue) Stop() { - // Stop the queues concurrently so total shutdown latency is bounded by a single - // queue's stop timeout rather than the sum across every queue. + // Stop the queues concurrently so total shutdown latency is bounded by a single queue's stop timeout rather than + // the sum across every queue. var waitGroup sync.WaitGroup for _, q := range m.queues { waitGroup.Go(q.Stop) @@ -67,8 +66,8 @@ func (m *MultiQueue) Stop() { waitGroup.Wait() } -// GetQueues returns a copy of the type-to-queue map so callers cannot mutate the -// internal map. The queue values are shared references, not copies. +// GetQueues returns a copy of the type-to-queue map so callers cannot mutate the internal map. The queue values are +// shared references, not copies. func (m *MultiQueue) GetQueues() map[string]*Queue { return maps.Clone(m.queues) } diff --git a/task/queue/multi_test.go b/task/queue/multi_test.go index 5c4a36ec82..c3831e068b 100644 --- a/task/queue/multi_test.go +++ b/task/queue/multi_test.go @@ -8,17 +8,18 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.mongodb.org/mongo-driver/bson" "github.com/tidepool-org/platform/log" - "github.com/tidepool-org/platform/log/null" + logNull "github.com/tidepool-org/platform/log/null" "github.com/tidepool-org/platform/page" storeStructuredMongo "github.com/tidepool-org/platform/store/structured/mongo" storeStructuredMongoTest "github.com/tidepool-org/platform/store/structured/mongo/test" "github.com/tidepool-org/platform/task" - "github.com/tidepool-org/platform/task/queue" - "github.com/tidepool-org/platform/task/queue/test" - "github.com/tidepool-org/platform/task/store/mongo" + taskQueue "github.com/tidepool-org/platform/task/queue" + taskQueueTest "github.com/tidepool-org/platform/task/queue/test" + taskStoreMongo "github.com/tidepool-org/platform/task/store/mongo" ) var ( @@ -28,19 +29,22 @@ var ( var _ = Describe("multi queue", func() { var config *storeStructuredMongo.Config - var queueConfig *queue.Config + var queueConfig *taskQueue.Config var lgr log.Logger - var str *mongo.Store - var multi *queue.MultiQueue + var str *taskStoreMongo.Store + var multi *taskQueue.MultiQueue BeforeEach(func() { config = storeStructuredMongoTest.NewConfig() var err error - str, err = mongo.NewStore(config) + str, err = taskStoreMongo.NewStore(config) Expect(err).ToNot(HaveOccurred()) Expect(str).ToNot(BeNil()) - lgr = null.NewLogger() - queueConfig = &queue.Config{Workers: 10, Delay: time.Millisecond, DelayInitial: time.Millisecond, DelayUnstick: queue.DelayUnstickDefault, StopWaitTimeout: queue.StopWaitTimeoutDefault, RunnerWatchdogGracePeriod: queue.RunnerWatchdogGracePeriodDefault} + lgr = logNull.NewLogger() + queueConfig = taskQueue.NewConfig() + queueConfig.Workers = 10 + queueConfig.StartManagerDelay = time.Millisecond + queueConfig.DispatchTasksDelay = time.Millisecond multi = nil }) @@ -53,13 +57,13 @@ var _ = Describe("multi queue", func() { Describe("NewMultiQueue", func() { It("creates a new queue for each runner type", func() { - runners := make([]queue.Runner, 0, len(types)) + runners := make([]taskQueue.Runner, 0, len(types)) for _, typ := range types { - runners = append(runners, test.NewCountingRunner(typ)) + runners = append(runners, taskQueueTest.NewCountingRunner(typ)) } var err error - multi, err = queue.NewMultiQueue(queueConfig, lgr, str, runners...) + multi, err = taskQueue.NewMultiQueue(queueConfig, lgr, str, runners...) Expect(err).ToNot(HaveOccurred()) Expect(multi).ToNot(BeNil()) @@ -71,38 +75,38 @@ var _ = Describe("multi queue", func() { }) It("returns an error when the config is missing", func() { - invalidMulti, err := queue.NewMultiQueue(nil, lgr, str) + invalidMulti, err := taskQueue.NewMultiQueue(nil, lgr, str) Expect(err).To(MatchError("config is missing")) Expect(invalidMulti).To(BeNil()) }) It("returns an error when the logger is missing", func() { - invalidMulti, err := queue.NewMultiQueue(queueConfig, nil, str) + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, nil, str) Expect(err).To(MatchError("logger is missing")) Expect(invalidMulti).To(BeNil()) }) It("returns an error when the store is missing", func() { - invalidMulti, err := queue.NewMultiQueue(queueConfig, lgr, nil) + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, lgr, nil) Expect(err).To(MatchError("store is missing")) Expect(invalidMulti).To(BeNil()) }) It("returns an error when a runner is missing", func() { - invalidMulti, err := queue.NewMultiQueue(queueConfig, lgr, str, nil) + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, lgr, str, nil) Expect(err).To(MatchError("runner is missing")) Expect(invalidMulti).To(BeNil()) }) It("returns an error when two runners have the same type", func() { - invalidMulti, err := queue.NewMultiQueue(queueConfig, lgr, str, test.NewCountingRunner(types[0]), test.NewCountingRunner(types[0])) + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, lgr, str, taskQueueTest.NewCountingRunner(types[0]), taskQueueTest.NewCountingRunner(types[0])) Expect(err).To(MatchError("runner type already registered")) Expect(invalidMulti).To(BeNil()) }) It("returns an error when a runner has invalid durations", func() { - runner := test.NewSleepRunner(types[0], 2*time.Minute, 2*time.Minute, time.Minute, 0) - invalidMulti, err := queue.NewMultiQueue(queueConfig, lgr, str, runner) + runner := taskQueueTest.NewSleepRunner(types[0], 2*time.Minute, 2*time.Minute, time.Minute, 0) + invalidMulti, err := taskQueue.NewMultiQueue(queueConfig, lgr, str, runner) Expect(err).To(MatchError("runner deadline is invalid")) Expect(invalidMulti).To(BeNil()) }) @@ -117,13 +121,13 @@ var _ = Describe("multi queue", func() { It("Are partitioned correctly", func() { ctx := log.NewContextWithLogger(context.Background(), lgr) creates := make([]*task.TaskCreate, 0, len(types)*tasksPerType) - countingRunners := make([]*test.CountingRunner, 0, len(types)) - runners := make([]queue.Runner, 0, len(types)) + countingRunners := make([]*taskQueueTest.CountingRunner, 0, len(types)) + runners := make([]taskQueue.Runner, 0, len(types)) now := time.Now() // Create tasks and runners for each task type for _, typ := range types { - runner := test.NewCountingRunner(typ) + runner := taskQueueTest.NewCountingRunner(typ) countingRunners = append(countingRunners, runner) runners = append(runners, runner) @@ -146,7 +150,7 @@ var _ = Describe("multi queue", func() { } var err error - multi, err = queue.NewMultiQueue(queueConfig, lgr, str, runners...) + multi, err = taskQueue.NewMultiQueue(queueConfig, lgr, str, runners...) Expect(err).ToNot(HaveOccurred()) Expect(multi).ToNot(BeNil()) @@ -154,10 +158,9 @@ var _ = Describe("multi queue", func() { nonTerminalStates := []string{task.TaskStatePending, task.TaskStateRunning} - // Wait until completion, within limits. On my local laptop, this typically - // takes < 15 seconds when run via Gingko (no parallel), but under Go test - // (parallel via package) it takes around 35 seconds. Who knows how long it - // would take running in parallel on a CI host. So give it plenty of time. + // Wait until completion, within limits. On my local laptop, this typically takes < 15 seconds when run via + // Gingko (no parallel), but under Go test (parallel via package) it takes around 35 seconds. Who knows how + // long it would take running in parallel on a CI host. So give it plenty of time. tCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() diff --git a/task/queue/queue.go b/task/queue/queue.go index 3d7b6c4050..3997db3cd7 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -21,53 +21,68 @@ import ( ) const ( - WorkersDefault = 5 - DelayDefault = 1 * time.Minute - DelayInitialDefault = 1 * time.Minute - DelayUnstickDefault = 5 * time.Minute - - // StopWaitTimeoutDefault bounds how long Stop waits for in-flight tasks to observe - // cancellation and exit before abandoning them (they are recovered by the deadline - // and unstick mechanism). Kept under the typical Kubernetes termination grace period - // since we could use it up to twice (once for workers and once for manager) and we - // still want to leave time for the store to flush any pending writes. - StopWaitTimeoutDefault = 10 * time.Second + // WorkersDefault is the default number of workers for the queue. + WorkersDefault = 5 - DurationJitterFactor = 0.2 + // StartManagerDelayDefault is the default upper bound on the randomized delay the manager waits before it begins + // dispatching tasks, spreading startup across instances. + StartManagerDelayDefault = 1 * time.Minute - // TaskDeadlineDefault bounds how long a task is allowed to run before being forcefully - // reset if a runner for the task type is not registered. - TaskDeadlineDefault = 1 * time.Minute + // DispatchTasksDelayDefault is the default delay, jittered, between polls for pending tasks to dispatch. Completing + // a task also dispatches immediately, without waiting for the poll. + DispatchTasksDelayDefault = 1 * time.Minute - // RunnerWatchdogGracePeriodDefault is the extra time beyond the runner timeout that the - // watchdog waits before reporting a runner as blocked. The runner context is still canceled - // at the runner timeout; the grace period only gives a cooperative runner time to observe - // that cancellation and return before the watchdog reports it as non-cooperative. + // MonitorTaskDelayDefault is the default interval between checks that each in-flight task's claim is still held + // (the task exists and its claim token is unchanged); a run whose claim is lost is canceled with ErrClaimLost. + MonitorTaskDelayDefault = 1 * time.Minute + + // RunnerWatchdogGracePeriodDefault is the extra time beyond the runner timeout that the watchdog waits before + // reporting a runner as blocked. The runner context is still canceled at the runner timeout; the grace period only + // gives a cooperative runner time to observe that cancellation and return before the watchdog reports it as + // non-cooperative. RunnerWatchdogGracePeriodDefault = 5 * time.Second -) -// ErrRunnerTimeout is the cancellation cause set on the Run context when a run exceeds the -// runner timeout. A runner distinguishes a timeout from a shutdown with -// errors.Is(context.Cause(ctx), ErrRunnerTimeout); a shutdown instead cancels with context.Canceled. -var ErrRunnerTimeout = errors.New("task runner timeout exceeded") + // UnstickTasksDelayDefault is the default delay, jittered, between attempts to unstick tasks. The first attempt is + // made after a randomized delay of at most this duration. + UnstickTasksDelayDefault = 5 * time.Minute + + // UnstickTasksAvailableGracePeriodDefault is the default grace period added to the claim monitor delay and the + // runner watchdog grace period to form the delay after a task is unstuck before it is made available for + // re-dispatch. This ensures that any still-running tasks have likely been canceled and exited before being + // re-dispatched. + UnstickTasksAvailableGracePeriodDefault = 1 * time.Minute + + // StopWaitTimeoutDefault bounds how long Stop waits for in-flight tasks to observe cancellation and exit before + // abandoning them (they are recovered by the deadline and unstick mechanism). Kept under the typical Kubernetes + // termination grace period since we could use it up to twice (once for workers and once for manager) and we still + // want to leave time for the store to flush any pending writes. + StopWaitTimeoutDefault = 10 * time.Second + + TaskDeadlineDefault = 1 * time.Minute + DurationJitterFactor = 0.2 +) type Config struct { - Workers int - Delay time.Duration - DelayInitial time.Duration - DelayUnstick time.Duration - StopWaitTimeout time.Duration - RunnerWatchdogGracePeriod time.Duration + Workers int + StartManagerDelay time.Duration + DispatchTasksDelay time.Duration + MonitorTaskDelay time.Duration + RunnerWatchdogGracePeriod time.Duration + UnstickTasksDelay time.Duration + UnstickTasksAvailableGracePeriod time.Duration + StopWaitTimeout time.Duration } func NewConfig() *Config { return &Config{ - Workers: WorkersDefault, - Delay: DelayDefault, - DelayInitial: DelayInitialDefault, - DelayUnstick: DelayUnstickDefault, - StopWaitTimeout: StopWaitTimeoutDefault, - RunnerWatchdogGracePeriod: RunnerWatchdogGracePeriodDefault, + Workers: WorkersDefault, + StartManagerDelay: StartManagerDelayDefault, + DispatchTasksDelay: DispatchTasksDelayDefault, + MonitorTaskDelay: MonitorTaskDelayDefault, + RunnerWatchdogGracePeriod: RunnerWatchdogGracePeriodDefault, + UnstickTasksDelay: UnstickTasksDelayDefault, + UnstickTasksAvailableGracePeriod: UnstickTasksAvailableGracePeriodDefault, + StopWaitTimeout: StopWaitTimeoutDefault, } } @@ -76,46 +91,60 @@ func (c *Config) Load(configReporter config.Reporter) error { return errors.New("config reporter is missing") } - if workersString, err := configReporter.Get("workers"); err == nil { - if workers, parseErr := strconv.ParseInt(workersString, 10, 0); parseErr != nil { + if valueString, err := configReporter.Get("workers"); err == nil { + if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { return errors.New("workers is invalid") } else { - c.Workers = int(workers) + c.Workers = int(value) } } - if delayString, err := configReporter.Get("delay"); err == nil { - if delay, parseErr := strconv.ParseInt(delayString, 10, 0); parseErr != nil { - return errors.New("delay is invalid") + if valueString, err := configReporter.Get("start_manager_delay"); err == nil { + if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { + return errors.New("start manager delay is invalid") } else { - c.Delay = time.Duration(delay) * time.Second + c.StartManagerDelay = time.Duration(value) * time.Second } } - if delayInitialString, err := configReporter.Get("delay_initial"); err == nil { - if delayInitial, parseErr := strconv.ParseInt(delayInitialString, 10, 0); parseErr != nil { - return errors.New("delay initial is invalid") + if valueString, err := configReporter.Get("dispatch_tasks_delay"); err == nil { + if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { + return errors.New("dispatch tasks delay is invalid") } else { - c.DelayInitial = time.Duration(delayInitial) * time.Second + c.DispatchTasksDelay = time.Duration(value) * time.Second } } - if delayUnstickString, err := configReporter.Get("delay_unstick"); err == nil { - if delayUnstick, parseErr := strconv.ParseInt(delayUnstickString, 10, 0); parseErr != nil { - return errors.New("delay unstick is invalid") + if valueString, err := configReporter.Get("monitor_task_delay"); err == nil { + if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { + return errors.New("monitor task delay is invalid") } else { - c.DelayUnstick = time.Duration(delayUnstick) * time.Second + c.MonitorTaskDelay = time.Duration(value) * time.Second } } - if stopWaitTimeoutString, err := configReporter.Get("stop_wait_timeout"); err == nil { - if stopWaitTimeout, parseErr := strconv.ParseInt(stopWaitTimeoutString, 10, 0); parseErr != nil { - return errors.New("stop wait timeout is invalid") + if valueString, err := configReporter.Get("runner_watchdog_grace_period"); err == nil { + if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { + return errors.New("runner watchdog grace period is invalid") } else { - c.StopWaitTimeout = time.Duration(stopWaitTimeout) * time.Second + c.RunnerWatchdogGracePeriod = time.Duration(value) * time.Second } } - if runnerWatchdogGracePeriodString, err := configReporter.Get("runner_watchdog_grace_period"); err == nil { - if runnerWatchdogGracePeriod, parseErr := strconv.ParseInt(runnerWatchdogGracePeriodString, 10, 0); parseErr != nil { - return errors.New("runner watchdog grace period is invalid") + if valueString, err := configReporter.Get("unstick_tasks_delay"); err == nil { + if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { + return errors.New("unstick tasks delay is invalid") + } else { + c.UnstickTasksDelay = time.Duration(value) * time.Second + } + } + if valueString, err := configReporter.Get("unstick_tasks_available_grace_period"); err == nil { + if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { + return errors.New("unstick tasks available grace period is invalid") } else { - c.RunnerWatchdogGracePeriod = time.Duration(runnerWatchdogGracePeriod) * time.Second + c.UnstickTasksAvailableGracePeriod = time.Duration(value) * time.Second + } + } + if valueString, err := configReporter.Get("stop_wait_timeout"); err == nil { + if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { + return errors.New("stop wait timeout is invalid") + } else { + c.StopWaitTimeout = time.Duration(value) * time.Second } } @@ -126,28 +155,34 @@ func (c *Config) Validate() error { if c.Workers < 1 { return errors.New("workers is invalid") } - if c.Delay <= 0 { - return errors.New("delay is invalid") - } - if c.DelayInitial <= 0 { - return errors.New("delay initial is invalid") + if c.StartManagerDelay <= 0 { + return errors.New("start manager delay is invalid") } - if c.DelayUnstick <= 0 { - return errors.New("delay unstick is invalid") + if c.DispatchTasksDelay <= 0 { + return errors.New("dispatch tasks delay is invalid") } - if c.StopWaitTimeout <= 0 { - return errors.New("stop wait timeout is invalid") + if c.MonitorTaskDelay <= 0 { + return errors.New("monitor task delay is invalid") } if c.RunnerWatchdogGracePeriod <= 0 { return errors.New("runner watchdog grace period is invalid") } + if c.UnstickTasksDelay <= 0 { + return errors.New("unstick tasks delay is invalid") + } + if c.UnstickTasksAvailableGracePeriod <= 0 { + return errors.New("unstick tasks available grace period is invalid") + } + if c.StopWaitTimeout <= 0 { + return errors.New("stop wait timeout is invalid") + } return nil } -// The Queue's fields are all immutable after New, except the lifecycle fields, which are -// guarded by the lifecycle mutex, and workersAvailable, which is owned exclusively by the -// manager goroutine. The workers and manager therefore read the channels and runners map -// freely, without synchronization. +// The Queue's fields are all immutable after New, except the lifecycle fields, which are guarded by the lifecycle +// mutex, and workersAvailable, which is initialized by Start before the manager exists and thereafter owned exclusively +// by the manager goroutine. The workers and manager therefore read the channels and runners map freely, without +// synchronization. type Queue struct { name string config *Config @@ -204,11 +239,12 @@ func New(name string, cfg *Config, lgr log.Logger, str taskStore.Store, runners repository: str.NewTaskRepository(), runners: runnerMap, - // NOT buffered so a task is only handed off when a worker is ready to receive it. This ensures a dispatched task is never - // stranded in a buffer during shutdown. + // NOT buffered so a task is only handed off when a worker is ready to receive it. This ensures a dispatched + // task is never stranded in a buffer during shutdown. dispatchChannel: make(chan *task.Task), - // Buffered so that a worker can complete a task and hand it off to the manager even if the manager is busy dispatching other tasks. + // Buffered so that a worker can complete a task and hand it off to the manager even if the manager is busy + // dispatching other tasks. completionChannel: make(chan *task.Task, cfg.Workers), }, nil } @@ -234,8 +270,8 @@ func (q *Queue) Start() { } func (q *Queue) Stop() { - // Hold the mutex for the entire stop, including the waits, so a concurrent Start cannot - // observe a partially stopped queue. + // Hold the mutex for the entire stop, including the waits, so a concurrent Start cannot observe a partially stopped + // queue. q.lifecycleMutex.Lock() defer q.lifecycleMutex.Unlock() @@ -253,15 +289,14 @@ func (q *Queue) Stop() { lgr := q.logger.WithField("stopWaitTimeout", q.config.StopWaitTimeout) - // Cancel the manager, so it stops dispatching new tasks and begins draining completions - // from the workers, and the workers, to interrupt any in-flight task. + // Cancel the manager, so it stops dispatching new tasks and begins draining completions from the workers, and the + // workers, to interrupt any in-flight task. q.cancelFunc() - // Wait for all workers to exit, but only up to a bounded timeout so a runner that does - // not honor cancellation cannot block shutdown forever. If a worker is still running we - // must NOT close the channels: a stuck worker that later finishes would panic sending on - // a closed completion channel. Instead we leave the goroutines orphaned (reaped at process - // exit); the abandoned task stays running and is recovered by the deadline/unstick mechanism. + // Wait for all workers to exit, but only up to a bounded timeout so a runner that does not honor cancellation + // cannot block shutdown forever. If a worker is still running we must NOT close the channels: a stuck worker that + // later finishes would panic sending on a closed completion channel. Instead we leave the goroutines orphaned + // (reaped at process exit); the abandoned task stays running and is recovered by the deadline/unstick mechanism. if !waitWithTimeout(&q.workersWaitGroup, q.config.StopWaitTimeout) { lgr.Error("Task queue workers did not stop within timeout; abandoning in-flight tasks; will be fixed with UnstickTasks later") return @@ -270,16 +305,16 @@ func (q *Queue) Stop() { // All workers have exited, so completion channel can be closed. close(q.completionChannel) - // Wait for manager to exit. This should be prompt now that the completion channel is - // closed, but bound it too in case a completion write is slow. + // Wait for manager to exit. This should be prompt now that the completion channel is closed, but bound it too in + // case a completion write is slow. if !waitWithTimeout(&q.managerWaitGroup, q.config.StopWaitTimeout) { lgr.Error("Task queue manager did not stop within timeout") return } - // Manager has exited, so no further tasks will be dispatched. Because the - // dispatch channel is not buffered, no dispatched task can be stranded in it; any - // task the manager could not hand off was reverted to pending during dispatch. + // Manager has exited, so no further tasks will be dispatched. Because the dispatch channel is not buffered, no + // dispatched task can be stranded in it; any task the manager could not hand off was reverted to pending during + // dispatch. close(q.dispatchChannel) q.logger.Info("Task queue stopped") @@ -333,14 +368,28 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { runner, ok := q.runners[tsk.Type] if !ok { - // A whole task type is unprocessable by this queue; surface it distinctly (a configuration - // problem) rather than letting it blend in with ordinary per-task failures downstream. + // A whole task type is unprocessable by this queue; surface it distinctly (a configuration problem) rather than + // letting it blend in with ordinary per-task failures downstream. lgr.Error("Runner not found for task type; task cannot be processed") tsk.SetFailedWithError(errors.New("runner not found for task type")) RunnerNotFoundTotal.WithLabelValues(tsk.Type).Inc() return } + // The claim context is canceled with ErrClaimLost by the task claim monitor when the task is deleted or re-claimed + // mid-run. Claim loss is irreversible for this run (every claim gets a fresh token), so once canceled the outcome + // can never be persisted. + claimContext, claimCancel := context.WithCancelCause(ctx) + defer claimCancel(nil) + + // Clearing the claim token marks the outcome as unpersistable, which completeTask discards. Done in a defer, after + // the recover below, so a panicking runner is also reconciled correctly. + defer func() { + if errors.Is(context.Cause(claimContext), ErrClaimLost) { + tsk.ClaimToken = nil + } + }() + defer func() { if err := recover(); err != nil { lgr.WithFields(log.Fields{"error": err, "stack": string(debug.Stack())}).Error("Unhandled panic while running task") @@ -349,51 +398,91 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { } }() - // If runner does not respect its own maximum duration, then enforce a context-based timeout. - // This forces the task to cancel via the context. - runnerContext, cancel := context.WithTimeoutCause(ctx, runner.GetRunnerTimeout(), ErrRunnerTimeout) + // If runner does not respect its own maximum duration, then enforce a context-based timeout. This forces the task + // to cancel via the context. + runnerContext, cancel := context.WithTimeoutCause(claimContext, runner.GetRunnerTimeout(), ErrRunnerTimeoutExceeded) defer cancel() - // Watchdog for a runner that ignores cancellation. Go cannot preempt a goroutine, so if the - // runner blows past its timeout without returning, this worker stays blocked until the - // process restarts (the task itself is recovered by the deadline/unstick mechanism). We - // cannot unblock the worker, but we surface the condition via a log and metric so a - // non-cooperative runner is detectable rather than silent. The grace period lets a cooperative - // runner that returns promptly after the timeout cancellation avoid being reported as blocked - // here; such a run is instead logged and counted as "recovered" during reconciliation. + // Watchdog for a runner that ignores cancellation. Go cannot preempt a goroutine, so if the runner blows past its + // timeout without returning, this worker stays blocked until the process restarts (the task itself is recovered by + // the deadline/unstick mechanism). We cannot unblock the worker, but we surface the condition via a log and metric + // so a non-cooperative runner is detectable rather than silent. The grace period lets a cooperative runner that + // returns promptly after the timeout cancellation avoid being reported as blocked here; such a run is instead + // logged and counted as "recovered" during reconciliation. runnerWatchdog := time.AfterFunc(runner.GetRunnerTimeout()+q.config.RunnerWatchdogGracePeriod, func() { lgr.Error("Task runner exceeded timeout without returning; worker is blocked until it returns") RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType(), "blocked").Inc() }) defer runnerWatchdog.Stop() + // Watch the claim for the duration of the run. The task is owned by the runner while it runs, so the watch gets its + // own copies of everything it needs rather than reading tsk. + go func(id string, typ string, claimToken string) { + if reason := q.monitorTaskForLostClaim(claimContext, id, claimToken); reason != nil { + log.LoggerFromContext(claimContext).Warnf("Task %s; canceling task run", *reason) + RunClaimLostTotal.WithLabelValues(typ, *reason).Inc() + claimCancel(ErrClaimLost) + } + }(tsk.ID, tsk.Type, *tsk.ClaimToken) + // Run the task via the runner startTime := time.Now() runner.Run(runnerContext, tsk) duration := time.Since(startTime).Truncate(time.Millisecond) - // Immediate stop the runner watchdog + // Immediately stop the runner watchdog runnerWatchdog.Stop() + RunDurationSeconds.WithLabelValues(runner.GetRunnerType()).Observe(duration.Seconds()) + if duration > runner.GetRunnerDurationMaximum() { + lgr.WithField("duration", duration.Seconds()).Warn("Task duration exceeds maximum") + } + + // The claim was lost mid-run; any write-back would miss the claim token, so skip state reconciliation. The outcome + // is discarded during completion. + if errors.Is(context.Cause(claimContext), ErrClaimLost) { + return + } + // If the runner left the task running, reconcile its state based on why the run ended. if tsk.State == task.TaskStateRunning { switch { case context.Cause(ctx) != nil: - // The parent (worker) context was canceled by shutdown; make the task available - // again for retry rather than treating the interruption as a completion. + // The parent (worker) context was canceled by shutdown; make the task available again for retry rather than + // treating the interruption as a completion. tsk.RepeatAvailableAfter(0) case context.Cause(runnerContext) != nil: - // The runner exceeded its timeout but returned; record the cause so the failure is - // attributed to the timeout rather than the generic missing terminal state error. + // The runner exceeded its timeout but returned; record the cause so the failure is attributed to the + // timeout rather than the generic missing terminal state error. lgr.Warn("Task runner exceeded timeout; task will be failed") tsk.AppendError(context.Cause(runnerContext)) RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType(), "recovered").Inc() } } +} - RunDurationSeconds.WithLabelValues(runner.GetRunnerType()).Observe(duration.Seconds()) - if duration > runner.GetRunnerDurationMaximum() { - lgr.WithField("duration", duration.Seconds()).Warn("Task duration exceeds maximum") +// monitorTaskForLostClaim polls until the run's claim on the task is lost - the task is gone (deleted) or holds a +// different claim token (unstuck and possibly re-claimed) - and then returns the reason for the claim loss. If the +// context is canceled before the claim is lost, returns nil. +func (q *Queue) monitorTaskForLostClaim(ctx context.Context, id string, runningClaimToken string) *string { + ticker := time.NewTicker(q.config.MonitorTaskDelay) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if storeClaimToken, exists, err := q.repository.GetTaskClaimToken(ctx, id); err != nil { + if context.Cause(ctx) == nil { + log.LoggerFromContext(ctx).WithError(err).Error("Unable to get task claim token") + } + } else if !exists { + return pointer.From("deleted") + } else if storeClaimToken == nil || *storeClaimToken != runningClaimToken { + return pointer.From("reclaimed") + } + } } } @@ -403,27 +492,27 @@ func (q *Queue) startManager(ctx context.Context) { lgr.Debug("Task queue manager started") - lgr.Debug("Task queue manager initial delay initiated") + lgr.Debug("Task queue start manager delay initiated") // Start at a random future time to help prevent thundering herd problem select { case <-ctx.Done(): lgr.WithError(context.Cause(ctx)).Debug("Task queue manager stopped before dispatching tasks") return - case <-time.After(randomDuration(q.config.DelayInitial)): - lgr.Debug("Task queue manager initial delay complete") + case <-time.After(randomDuration(q.config.StartManagerDelay)): + lgr.Debug("Task queue start manager delay complete") } // Start at a random future time to help prevent thundering herd problem - unstickTime := time.Now().Add(randomDuration(q.config.DelayUnstick)) + unstickTasksTime := time.Now().Add(randomDuration(q.config.UnstickTasksDelay)) for { if err := q.executeManager(ctx); err != nil { lgr.WithError(err).Debug("Task queue manager stopping") - // Complete any remaining tasks concurrently so the total drain time is - // bounded by a single slow completion write rather than the sum across - // all workers' tasks, keeping shutdown within the stop wait timeout. + // Complete any remaining tasks concurrently so the total drain time is bounded by a single slow + // completion write rather than the sum across all workers' tasks, keeping shutdown within the stop wait + // timeout. var completionWaitGroup sync.WaitGroup for tsk := range q.completionChannel { if tsk != nil { @@ -436,9 +525,9 @@ func (q *Queue) startManager(ctx context.Context) { return } - if unstickTime.Before(time.Now()) { + if unstickTasksTime.Before(time.Now()) { q.unstickTasks(ctx) - unstickTime = time.Now().Add(durationWithJitter(q.config.DelayUnstick)) + unstickTasksTime = time.Now().Add(durationWithJitter(q.config.UnstickTasksDelay)) } } }) @@ -461,7 +550,7 @@ func (q *Queue) executeManager(ctx context.Context) error { WorkersAvailable.WithLabelValues(q.name).Set(float64(q.workersAvailable)) q.dispatchTasks(ctx) } - case <-time.After(durationWithJitter(q.config.Delay)): + case <-time.After(durationWithJitter(q.config.DispatchTasksDelay)): q.dispatchTasks(ctx) } @@ -469,7 +558,10 @@ func (q *Queue) executeManager(ctx context.Context) error { } func (q *Queue) unstickTasks(ctx context.Context) { - ids, err := q.repository.UnstickTasks(ctx) + // Delay availability of unstuck tasks to ensure that any still-running tasks have likely been canceled and exited + // before being re-dispatched. + availabilityDelay := q.config.MonitorTaskDelay + q.config.RunnerWatchdogGracePeriod + q.config.UnstickTasksAvailableGracePeriod + ids, err := q.repository.UnstickTasks(ctx, availabilityDelay) if count := len(ids); count > 0 { log.LoggerFromContext(ctx).WithFields(log.Fields{"count": count, "ids": ids}).Info("Unstuck tasks") } @@ -521,7 +613,8 @@ func (q *Queue) dispatchTasks(ctx context.Context) { func (q *Queue) dispatchTask(ctx context.Context, tsk *task.Task) error { ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) - // we don't error here if missing, as the task will be failed during runTask, which persists error to database + // we don't error here if missing, as the task will be failed during runTask and the error persisted to the database + // when the task completes var deadline time.Duration if runner, ok := q.runners[tsk.Type]; ok { deadline = runner.GetRunnerDeadline() @@ -529,8 +622,8 @@ func (q *Queue) dispatchTask(ctx context.Context, tsk *task.Task) error { deadline = TaskDeadlineDefault } - // StartTask completes regardless of context cancellation, so its outcome is definitive: - // a non-nil startedTask means the claim committed with a known state lock. + // StartTask completes regardless of context cancellation, so its outcome is definitive: a non-nil startedTask means + // the claim committed with a known claim token. startedTask, err := q.repository.StartTask(ctx, tsk.ID, tsk.Revision, deadline) if err != nil { return errors.Wrap(err, "unable to start task") @@ -539,12 +632,11 @@ func (q *Queue) dispatchTask(ctx context.Context, tsk *task.Task) error { return nil } - // Hand the task off to a worker. If the queue is shutting down before a worker can - // receive it, revert the task to pending rather than blocking the manager. The revert - // uses the started task state lock so it reliably matches. + // Hand the task off to a worker. If the queue is shutting down before a worker can receive it, revert the task to + // pending rather than blocking the manager. The revert uses the started task claim token so it reliably matches. select { case <-ctx.Done(): - if err := q.repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.StateLock, task.TaskStatePending, nil, nil); err != nil { + if err := q.repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.ClaimToken, task.TaskStatePending, nil, nil); err != nil { return errors.Wrap(err, "unable to revert task to pending") } case q.dispatchChannel <- startedTask: @@ -555,41 +647,46 @@ func (q *Queue) dispatchTask(ctx context.Context, tsk *task.Task) error { return nil } -// completeTask persists the task's completion. It deliberately does not touch -// workersAvailable (the caller accounts for the freed worker where relevant), so it is safe -// to call concurrently during the manager's shutdown drain. +// completeTask persists the task's completion. It deliberately does not touch workersAvailable (the caller accounts for +// the freed worker where relevant), so it is safe to call concurrently during the manager's shutdown drain. func (q *Queue) completeTask(ctx context.Context, tsk *task.Task) { ctx, lgr := log.ContextAndLoggerWithField(ctx, "task", tsk.LogFields()) + // The claim was lost mid-run (task deleted or re-claimed): any write-back would miss the claim token, so discard + // the outcome rather than logging a spurious lost completion. + if tsk.ClaimToken == nil { + lgr.Warn("Task claim lost; task run outcome discarded") + return + } + var duration *time.Duration if tsk.RunTime != nil { - // Clamp to a zero minimum: the run time round-trips through the database without a - // monotonic reading, so a backwards wall clock step could yield a non-positive elapsed - // time, which StopTask would reject, losing the completion. + // Clamp to a zero minimum: the run time round-trips through the database without a monotonic reading, so a + // backwards wall clock step could yield a negative elapsed time, which StopTask would reject, losing the + // completion. duration = pointer.From(max(time.Since(*tsk.RunTime), 0)) } q.computeState(ctx, tsk) - // computeState has already settled the terminal state. A failed task is a genuine problem - // worth an error; a task that errored but will run again (e.g. reverted to pending for - // retry) is an expected, recoverable outcome, so log it at warning to avoid error-level - // noise from routine retries. + // computeState has already settled the terminal state. A failed task is a genuine problem worth an error; a task + // that errored but will run again (e.g. reverted to pending for retry) is an expected, recoverable outcome, so log + // it at warning to avoid error-level noise from routine retries. if err := tsk.GetError(); tsk.State == task.TaskStateFailed { lgr.WithError(err).Error("Task failed while running") } else if err != nil { lgr.WithError(err).Warn("Error occurred while running task that did not fail") } - // Data and Error use non-nil wrappers so that a task whose data or error was cleared during - // the run has the corresponding field unset in the database, rather than left stale (a nil - // wrapper means "leave unchanged" to StopTask). + // Data and Error use non-nil wrappers so that a task whose data or error was cleared during the run has the + // corresponding field unset in the database, rather than left stale (a nil wrapper means "leave unchanged" to + // StopTask). update := &task.TaskUpdate{ Data: pointer.From(tsk.Data), AvailableTime: tsk.AvailableTime, Error: &errors.Serializable{Error: tsk.GetError()}, } - if err := q.repository.StopTask(ctx, tsk.ID, tsk.Revision, tsk.StateLock, tsk.State, duration, update); err != nil { + if err := q.repository.StopTask(ctx, tsk.ID, tsk.Revision, tsk.ClaimToken, tsk.State, duration, update); err != nil { lgr.WithError(err).Error("Unable to complete task") } } @@ -608,8 +705,8 @@ func (q *Queue) computeState(ctx context.Context, tsk *task.Task) { tsk.AvailableTime = pointer.FromTime(now) } case task.TaskStateRunning: - // The runner returned without moving the task out of the running state, violating - // the runner contract; fail the task rather than guessing whether it succeeded. + // The runner returned without moving the task out of the running state, violating the runner contract; fail the + // task rather than guessing whether it succeeded. if tsk.HasError() { tsk.SetFailed() } else { @@ -622,11 +719,10 @@ func (q *Queue) computeState(ctx context.Context, tsk *task.Task) { } } -// validateRunner enforces the runner duration contract, deadline > timeout > duration maximum > 0, -// so that a misconfigured runner fails queue construction rather than causing subtle runtime -// misbehavior (a non-positive deadline prevents the task from ever starting, while a deadline -// that does not exceed the timeout allows the unstick mechanism to reset a task that is still -// running). +// validateRunner enforces the runner duration contract, deadline > timeout > duration maximum > 0, so that a +// misconfigured runner fails queue construction rather than causing subtle runtime misbehavior (a non-positive deadline +// prevents the task from ever starting, while a deadline that does not exceed the timeout allows the unstick mechanism +// to reset a task that is still running). func validateRunner(runner Runner) error { if durationMaximum := runner.GetRunnerDurationMaximum(); durationMaximum <= 0 { return errors.New("runner duration maximum is invalid") @@ -639,9 +735,9 @@ func validateRunner(runner Runner) error { } } -// waitWithTimeout waits for the wait group to complete, returning true if it completed within -// the timeout and false otherwise. On timeout the internal waiter goroutine is left running -// until the wait group eventually completes (or the process exits). +// waitWithTimeout waits for the wait group to complete, returning true if it completed within the timeout and false +// otherwise. On timeout the internal waiter goroutine is left running until the wait group eventually completes (or the +// process exits). func waitWithTimeout(waitGroup *sync.WaitGroup, timeout time.Duration) bool { done := make(chan struct{}) go func() { @@ -679,41 +775,52 @@ var ( Help: "The configured number of task queue workers, sorted by queue", }, []string{"queue"}) - // WorkersAvailable reports the number of available workers, per queue. A value pinned at zero - // indicates a saturated queue or workers wedged in non-cooperative runners. + // WorkersAvailable reports the number of available workers, per queue. A value pinned at zero indicates a saturated + // queue or workers wedged in non-cooperative runners. WorkersAvailable = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "tidepool_task_workers_available", Help: "The number of available task queue workers, sorted by queue", }, []string{"queue"}) - // RunnerNotFoundTotal counts task runs for which no runner is registered for the task's type, - // sorted by type. A non-zero value indicates pending tasks of a type this queue cannot process - // (a configuration problem); such tasks are failed immediately. + // RunnerNotFoundTotal counts task runs for which no runner is registered for the task's type, sorted by type. A + // non-zero value indicates pending tasks of a type this queue cannot process (a configuration problem); such tasks + // are failed immediately. RunnerNotFoundTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "tidepool_task_runner_not_found_total", Help: "The total number of task runs with no registered runner for the task type, sorted by type", }, []string{"type"}) - // RunDurationSeconds observes how long each task run took, sorted by type. Use it to track - // run latency percentiles and to alert when durations approach the runner timeout. + // RunDurationSeconds observes how long each task run took, sorted by type. Use it to track run latency percentiles + // and to alert when durations approach the runner timeout. RunDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "tidepool_task_run_duration_seconds", Help: "The duration of task runs, in seconds, sorted by type", Buckets: prometheus.ExponentialBuckets(0.1, 2, 15), }, []string{"type"}) - // RunnerTimeoutExceededTotal counts task runs that exceeded the runner timeout, sorted by type - // and disposition, where disposition can be: - // - "blocked" - for runs the watchdog caught still running past the grace period (the worker is - // wedged until the runner eventually returns or the process restarts, so this is the severe case) - // - "recovered" - for runs that exceeded the timeout, but returned once their context was - // canceled (the timeout mechanism worked as designed) + // RunnerTimeoutExceededTotal counts task runs that exceeded the runner timeout, sorted by type and disposition, + // where disposition can be: + // - "blocked" - for runs the watchdog caught still running past the grace period (the worker is wedged until the + // runner eventually returns or the process restarts, so this is the severe case) + // - "recovered" - for runs that exceeded the timeout, but returned once their context was canceled (the timeout + // mechanism worked as designed) RunnerTimeoutExceededTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "tidepool_task_runner_timeout_exceeded_total", Help: "The total number of task runs that exceeded the runner timeout, sorted by type and disposition", }, []string{"type", "disposition"}) - // RunPanicTotal counts task runs that panicked and were recovered and failed, sorted by type. + // RunClaimLostTotal counts task runs canceled because the run's claim was lost, sorted by type and reason, where + // reason can be: + // - "deleted" - the task document was deleted mid-run + // - "reclaimed" - the stored claim token no longer matches (the task was unstuck and possibly re-claimed + // elsewhere) + RunClaimLostTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "tidepool_task_run_claim_lost_total", + Help: "The total number of task runs canceled because the task claim was lost, sorted by type and reason", + }, []string{"type", "reason"}) + + // RunPanicTotal counts task runs that panicked and were recovered, sorted by type. The task is then failed, unless + // the runner had already moved it out of running before panicking. RunPanicTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "tidepool_task_run_panic_total", Help: "The total number of task runs that panicked, sorted by type", diff --git a/task/queue/queue_test.go b/task/queue/queue_test.go index a242bac1fa..4957fb50da 100644 --- a/task/queue/queue_test.go +++ b/task/queue/queue_test.go @@ -33,34 +33,42 @@ var _ = Describe("Queue", func() { Expect(taskQueue.WorkersDefault).To(Equal(5)) }) - It("DelayDefault is expected", func() { - Expect(taskQueue.DelayDefault).To(Equal(1 * time.Minute)) + It("StartManagerDelayDefault is expected", func() { + Expect(taskQueue.StartManagerDelayDefault).To(Equal(1 * time.Minute)) }) - It("DelayInitialDefault is expected", func() { - Expect(taskQueue.DelayInitialDefault).To(Equal(1 * time.Minute)) + It("DispatchTasksDelayDefault is expected", func() { + Expect(taskQueue.DispatchTasksDelayDefault).To(Equal(1 * time.Minute)) }) - It("DelayUnstickDefault is expected", func() { - Expect(taskQueue.DelayUnstickDefault).To(Equal(5 * time.Minute)) - }) - - It("StopWaitTimeoutDefault is expected", func() { - Expect(taskQueue.StopWaitTimeoutDefault).To(Equal(10 * time.Second)) + It("MonitorTaskDelayDefault is expected", func() { + Expect(taskQueue.MonitorTaskDelayDefault).To(Equal(1 * time.Minute)) }) It("RunnerWatchdogGracePeriodDefault is expected", func() { Expect(taskQueue.RunnerWatchdogGracePeriodDefault).To(Equal(5 * time.Second)) }) - It("DurationJitterFactor is expected", func() { - Expect(taskQueue.DurationJitterFactor).To(Equal(0.2)) + It("UnstickTasksDelayDefault is expected", func() { + Expect(taskQueue.UnstickTasksDelayDefault).To(Equal(5 * time.Minute)) + }) + + It("UnstickTasksAvailableGracePeriodDefault is expected", func() { + Expect(taskQueue.UnstickTasksAvailableGracePeriodDefault).To(Equal(1 * time.Minute)) + }) + + It("StopWaitTimeoutDefault is expected", func() { + Expect(taskQueue.StopWaitTimeoutDefault).To(Equal(10 * time.Second)) }) It("TaskDeadlineDefault is expected", func() { Expect(taskQueue.TaskDeadlineDefault).To(Equal(1 * time.Minute)) }) + It("DurationJitterFactor is expected", func() { + Expect(taskQueue.DurationJitterFactor).To(Equal(0.2)) + }) + Context("Config", func() { Context("NewConfig", func() { It("returns successfully", func() { @@ -72,11 +80,13 @@ var _ = Describe("Queue", func() { cfg := taskQueue.NewConfig() Expect(cfg).ToNot(BeNil()) Expect(cfg.Workers).To(Equal(taskQueue.WorkersDefault)) - Expect(cfg.Delay).To(Equal(taskQueue.DelayDefault)) - Expect(cfg.DelayInitial).To(Equal(taskQueue.DelayInitialDefault)) - Expect(cfg.DelayUnstick).To(Equal(taskQueue.DelayUnstickDefault)) - Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) + Expect(cfg.StartManagerDelay).To(Equal(taskQueue.StartManagerDelayDefault)) + Expect(cfg.DispatchTasksDelay).To(Equal(taskQueue.DispatchTasksDelayDefault)) + Expect(cfg.MonitorTaskDelay).To(Equal(taskQueue.MonitorTaskDelayDefault)) + Expect(cfg.UnstickTasksDelay).To(Equal(taskQueue.UnstickTasksDelayDefault)) + Expect(cfg.UnstickTasksAvailableGracePeriod).To(Equal(taskQueue.UnstickTasksAvailableGracePeriodDefault)) Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(taskQueue.RunnerWatchdogGracePeriodDefault)) + Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) }) }) @@ -103,24 +113,19 @@ var _ = Describe("Queue", func() { Expect(cfg.Load(configReporter)).To(MatchError("workers is invalid")) }) - It("returns an error when delay is not parsable", func() { - configReporter.Config["delay"] = test.RandomStringFromCharset(test.CharsetAlpha) - Expect(cfg.Load(configReporter)).To(MatchError("delay is invalid")) + It("returns an error when start manager delay is not parsable", func() { + configReporter.Config["start_manager_delay"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("start manager delay is invalid")) }) - It("returns an error when delay initial is not parsable", func() { - configReporter.Config["delay_initial"] = test.RandomStringFromCharset(test.CharsetAlpha) - Expect(cfg.Load(configReporter)).To(MatchError("delay initial is invalid")) + It("returns an error when dispatch tasks delay is not parsable", func() { + configReporter.Config["dispatch_tasks_delay"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("dispatch tasks delay is invalid")) }) - It("returns an error when delay unstick is not parsable", func() { - configReporter.Config["delay_unstick"] = test.RandomStringFromCharset(test.CharsetAlpha) - Expect(cfg.Load(configReporter)).To(MatchError("delay unstick is invalid")) - }) - - It("returns an error when stop wait timeout is not parsable", func() { - configReporter.Config["stop_wait_timeout"] = test.RandomStringFromCharset(test.CharsetAlpha) - Expect(cfg.Load(configReporter)).To(MatchError("stop wait timeout is invalid")) + It("returns an error when monitor task delay is not parsable", func() { + configReporter.Config["monitor_task_delay"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("monitor task delay is invalid")) }) It("returns an error when runner watchdog grace period is not parsable", func() { @@ -128,50 +133,79 @@ var _ = Describe("Queue", func() { Expect(cfg.Load(configReporter)).To(MatchError("runner watchdog grace period is invalid")) }) + It("returns an error when unstick tasks delay is not parsable", func() { + configReporter.Config["unstick_tasks_delay"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("unstick tasks delay is invalid")) + }) + + It("returns an error when unstick tasks available grace period is not parsable", func() { + configReporter.Config["unstick_tasks_available_grace_period"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("unstick tasks available grace period is invalid")) + }) + + It("returns an error when stop wait timeout is not parsable", func() { + configReporter.Config["stop_wait_timeout"] = test.RandomStringFromCharset(test.CharsetAlpha) + Expect(cfg.Load(configReporter)).To(MatchError("stop wait timeout is invalid")) + }) + It("uses existing workers if not set", func() { Expect(cfg.Load(configReporter)).To(Succeed()) Expect(cfg.Workers).To(Equal(taskQueue.WorkersDefault)) }) - It("uses existing delay if not set", func() { + It("uses existing start manager delay if not set", func() { Expect(cfg.Load(configReporter)).To(Succeed()) - Expect(cfg.Delay).To(Equal(taskQueue.DelayDefault)) + Expect(cfg.StartManagerDelay).To(Equal(taskQueue.StartManagerDelayDefault)) }) - It("uses existing delay initial if not set", func() { + It("uses existing dispatch tasks delay if not set", func() { Expect(cfg.Load(configReporter)).To(Succeed()) - Expect(cfg.DelayInitial).To(Equal(taskQueue.DelayInitialDefault)) + Expect(cfg.DispatchTasksDelay).To(Equal(taskQueue.DispatchTasksDelayDefault)) }) - It("uses existing delay unstick if not set", func() { + It("uses existing monitor task delay if not set", func() { Expect(cfg.Load(configReporter)).To(Succeed()) - Expect(cfg.DelayUnstick).To(Equal(taskQueue.DelayUnstickDefault)) + Expect(cfg.MonitorTaskDelay).To(Equal(taskQueue.MonitorTaskDelayDefault)) }) - It("uses existing stop wait timeout if not set", func() { + It("uses existing runner watchdog grace period if not set", func() { Expect(cfg.Load(configReporter)).To(Succeed()) - Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) + Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(taskQueue.RunnerWatchdogGracePeriodDefault)) }) - It("uses existing runner watchdog grace period if not set", func() { + It("uses existing unstick tasks delay if not set", func() { Expect(cfg.Load(configReporter)).To(Succeed()) - Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(taskQueue.RunnerWatchdogGracePeriodDefault)) + Expect(cfg.UnstickTasksDelay).To(Equal(taskQueue.UnstickTasksDelayDefault)) + }) + + It("uses existing unstick tasks available grace period if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.UnstickTasksAvailableGracePeriod).To(Equal(taskQueue.UnstickTasksAvailableGracePeriodDefault)) + }) + + It("uses existing stop wait timeout if not set", func() { + Expect(cfg.Load(configReporter)).To(Succeed()) + Expect(cfg.StopWaitTimeout).To(Equal(taskQueue.StopWaitTimeoutDefault)) }) It("returns successfully and uses values from the config reporter", func() { configReporter.Config["workers"] = "5" - configReporter.Config["delay"] = "30" - configReporter.Config["delay_initial"] = "45" - configReporter.Config["delay_unstick"] = "60" - configReporter.Config["stop_wait_timeout"] = "15" + configReporter.Config["start_manager_delay"] = "45" + configReporter.Config["dispatch_tasks_delay"] = "30" + configReporter.Config["monitor_task_delay"] = "75" configReporter.Config["runner_watchdog_grace_period"] = "20" + configReporter.Config["unstick_tasks_delay"] = "60" + configReporter.Config["unstick_tasks_available_grace_period"] = "10" + configReporter.Config["stop_wait_timeout"] = "15" Expect(cfg.Load(configReporter)).To(Succeed()) Expect(cfg.Workers).To(Equal(5)) - Expect(cfg.Delay).To(Equal(30 * time.Second)) - Expect(cfg.DelayInitial).To(Equal(45 * time.Second)) - Expect(cfg.DelayUnstick).To(Equal(60 * time.Second)) - Expect(cfg.StopWaitTimeout).To(Equal(15 * time.Second)) + Expect(cfg.StartManagerDelay).To(Equal(45 * time.Second)) + Expect(cfg.DispatchTasksDelay).To(Equal(30 * time.Second)) + Expect(cfg.MonitorTaskDelay).To(Equal(75 * time.Second)) Expect(cfg.RunnerWatchdogGracePeriod).To(Equal(20 * time.Second)) + Expect(cfg.UnstickTasksDelay).To(Equal(60 * time.Second)) + Expect(cfg.UnstickTasksAvailableGracePeriod).To(Equal(10 * time.Second)) + Expect(cfg.StopWaitTimeout).To(Equal(15 * time.Second)) }) }) @@ -181,24 +215,19 @@ var _ = Describe("Queue", func() { Expect(cfg.Validate()).To(MatchError("workers is invalid")) }) - It("returns an error when delay is invalid", func() { - cfg.Delay = 0 - Expect(cfg.Validate()).To(MatchError("delay is invalid")) + It("returns an error when start manager delay is invalid", func() { + cfg.StartManagerDelay = 0 + Expect(cfg.Validate()).To(MatchError("start manager delay is invalid")) }) - It("returns an error when delay initial is invalid", func() { - cfg.DelayInitial = 0 - Expect(cfg.Validate()).To(MatchError("delay initial is invalid")) + It("returns an error when dispatch tasks delay is invalid", func() { + cfg.DispatchTasksDelay = 0 + Expect(cfg.Validate()).To(MatchError("dispatch tasks delay is invalid")) }) - It("returns an error when delay unstick is invalid", func() { - cfg.DelayUnstick = 0 - Expect(cfg.Validate()).To(MatchError("delay unstick is invalid")) - }) - - It("returns an error when stop wait timeout is invalid", func() { - cfg.StopWaitTimeout = 0 - Expect(cfg.Validate()).To(MatchError("stop wait timeout is invalid")) + It("returns an error when monitor task delay is invalid", func() { + cfg.MonitorTaskDelay = 0 + Expect(cfg.Validate()).To(MatchError("monitor task delay is invalid")) }) It("returns an error when runner watchdog grace period is invalid", func() { @@ -206,6 +235,21 @@ var _ = Describe("Queue", func() { Expect(cfg.Validate()).To(MatchError("runner watchdog grace period is invalid")) }) + It("returns an error when unstick tasks delay is invalid", func() { + cfg.UnstickTasksDelay = 0 + Expect(cfg.Validate()).To(MatchError("unstick tasks delay is invalid")) + }) + + It("returns an error when unstick tasks available grace period is invalid", func() { + cfg.UnstickTasksAvailableGracePeriod = 0 + Expect(cfg.Validate()).To(MatchError("unstick tasks available grace period is invalid")) + }) + + It("returns an error when stop wait timeout is invalid", func() { + cfg.StopWaitTimeout = 0 + Expect(cfg.Validate()).To(MatchError("stop wait timeout is invalid")) + }) + It("returns successfully", func() { Expect(cfg.Validate()).To(Succeed()) }) @@ -375,14 +419,10 @@ var _ = Describe("Queue", func() { var que *taskQueue.Queue BeforeEach(func() { - cfg = &taskQueue.Config{ - Workers: 2, - Delay: time.Millisecond, - DelayInitial: time.Millisecond, - DelayUnstick: taskQueue.DelayUnstickDefault, - StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, - RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault, - } + cfg = taskQueue.NewConfig() + cfg.Workers = 2 + cfg.StartManagerDelay = time.Millisecond + cfg.DispatchTasksDelay = time.Millisecond }) AfterEach(func() { @@ -480,13 +520,12 @@ var _ = Describe("Queue", func() { lgr.AssertWarn("Database task revision does not match running task revision; Runner contract broken or concurrent update") }) - It("does not complete a task whose state lock changed while it was running", func() { + It("does not complete a task whose claim token changed while it was running", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). WithStub(func(ctx context.Context, tsk *task.Task) { - // Simulate the task being unstuck and re-claimed elsewhere by changing the - // state lock out from under this run. The completion must then miss rather - // than falsely complete another run task. - test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": tsk.ID}, bson.M{"$set": bson.M{"stateLock": ""}})) + // Simulate the task being unstuck and re-claimed elsewhere by changing the claim token out from + // under this run. The completion must then miss rather than falsely complete another run task. + test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": tsk.ID}, bson.M{"$set": bson.M{"claimToken": ""}})) tsk.SetCompleted() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -497,7 +536,7 @@ var _ = Describe("Queue", func() { Eventually(func() bool { defer func() { _ = recover() }() - lgr.AssertError("Unable to stop task; no running task matched the expected condition") + lgr.AssertError("Unable to stop task; no running task matched the id and claim token") return true }, "5s", "100ms").To(BeTrue()) @@ -505,6 +544,154 @@ var _ = Describe("Queue", func() { Expect(actualTask.State).To(Equal(task.TaskStateRunning)) }) + It("does not complete a task was deleted while it was running", func() { + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { + test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": tsk.ID})) + tsk.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertError("Unable to stop task; no running task matched the id and claim token") + return true + }, "5s", "100ms").To(BeTrue()) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask).To(BeNil()) + }) + + It("cancels a running task when the task is deleted mid-run", func() { + cfg.MonitorTaskDelay = time.Millisecond + taskQueue.RunClaimLostTotal.Reset() + + canceled := make(chan error, 1) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { + test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": tsk.ID})) + <-ctx.Done() + canceled <- context.Cause(ctx) + tsk.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(canceled, "5s").Should(Receive(MatchError("task claim lost"))) + + // The outcome is discarded rather than written back as a lost completion. + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertWarn("Task claim lost; task run outcome discarded") + return true + }, "5s", "100ms").To(BeTrue()) + + Expect(test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil))).To(BeNil()) + Expect(testutil.ToFloat64(taskQueue.RunClaimLostTotal.WithLabelValues(runner.GetRunnerType(), "deleted"))).To(Equal(float64(1))) + }) + + It("cancels a running task deleted after earlier claim checks found the claim intact", func() { + cfg.MonitorTaskDelay = 50 * time.Millisecond + taskQueue.RunClaimLostTotal.Reset() + + canceled := make(chan error, 1) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { + // Run past several claim checks before losing the claim, so the monitor must keep checking + // rather than settle after the first. + time.Sleep(5 * cfg.MonitorTaskDelay) + test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": tsk.ID})) + <-ctx.Done() + canceled <- context.Cause(ctx) + tsk.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(canceled, "5s").Should(Receive(MatchError("task claim lost"))) + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertWarn("Task claim lost; task run outcome discarded") + return true + }, "5s", "100ms").To(BeTrue()) + + Expect(test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil))).To(BeNil()) + Expect(testutil.ToFloat64(taskQueue.RunClaimLostTotal.WithLabelValues(runner.GetRunnerType(), "deleted"))).To(Equal(float64(1))) + }) + + It("cancels a running task when the task is re-claimed mid-run", func() { + cfg.MonitorTaskDelay = 100 * time.Millisecond + taskQueue.RunClaimLostTotal.Reset() + + canceled := make(chan error, 1) + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { + // Simulate the task being unstuck and re-claimed elsewhere by changing the claim token out from + // under this run. + test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": tsk.ID}, bson.M{"$set": bson.M{"claimToken": "other-claim-token"}})) + <-ctx.Done() + canceled <- context.Cause(ctx) + tsk.SetCompleted() + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(canceled, "5s").Should(Receive(MatchError("task claim lost"))) + + Eventually(func() bool { + defer func() { _ = recover() }() + lgr.AssertWarn("Task claim lost; task run outcome discarded") + return true + }, "5s", "100ms").To(BeTrue()) + + // The foreign claim remains untouched; this run wrote nothing back. + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.State).To(Equal(task.TaskStateRunning)) + Expect(testutil.ToFloat64(taskQueue.RunClaimLostTotal.WithLabelValues(runner.GetRunnerType(), "reclaimed"))).To(Equal(float64(1))) + }) + + It("does not cancel a running task whose claim is intact", func() { + cfg.MonitorTaskDelay = time.Millisecond + + runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). + WithStub(func(ctx context.Context, tsk *task.Task) { + // Linger across several claim checks; the claim is intact, so the run must not be canceled. + select { + case <-ctx.Done(): + tsk.AppendError(context.Cause(ctx)) + tsk.SetFailed() + case <-time.After(250 * time.Millisecond): + tsk.SetCompleted() + } + }) + que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) + + createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) + + que.Start() + + Eventually(func() string { + return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State + }, "5s", "50ms").To(Equal(task.TaskStateCompleted)) + + actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) + Expect(actualTask.Error).To(BeNil()) + }) + It("cleans up a task that panics during execution", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). WithStub(func(ctx context.Context, tsk *task.Task) { panic("panic test") }) @@ -592,7 +779,12 @@ var _ = Describe("Queue", func() { }) It("unsticks and logs a task left running past its deadline", func() { - cfg.DelayUnstick = time.Millisecond + cfg.UnstickTasksDelay = time.Millisecond + cfg.UnstickTasksAvailableGracePeriod = 50 * time.Millisecond + + // Keep the unstuck task's availability delay short so it is re-dispatched within the wait below. + cfg.MonitorTaskDelay = time.Millisecond + cfg.RunnerWatchdogGracePeriod = 50 * time.Millisecond runner := taskQueueTest.NewCountingRunner(taskTest.RandomType()) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -601,9 +793,9 @@ var _ = Describe("Queue", func() { ID: task.NewID(), Type: runner.GetRunnerType(), State: task.TaskStateRunning, - StateLock: pointer.FromString(taskTest.RandomType()), CreatedTime: time.Now(), Revision: 1, + ClaimToken: pointer.FromString(taskTest.RandomType()), DeadlineTime: pointer.FromTime(time.Now().Add(-time.Minute)), } test.Must(str.GetCollection("tasks").InsertOne(ctx, createdTask)) @@ -636,8 +828,8 @@ var _ = Describe("Queue", func() { return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State }, "5s", "50ms").To(Equal(task.TaskStateRunning)) - // Stopping cancels the worker context; the hanging runner returns leaving the - // task running, so the queue must revert it to pending for a later retry. + // Stopping cancels the worker context; the hanging runner returns leaving the task running, so the + // queue must revert it to pending for a later retry. que.Stop() actualTask := test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)) @@ -751,14 +943,11 @@ var _ = Describe("Queue", func() { var que *taskQueue.Queue BeforeEach(func() { - cfg = &taskQueue.Config{ - Workers: 2, - Delay: time.Millisecond, - DelayInitial: time.Millisecond, - DelayUnstick: taskQueue.DelayUnstickDefault, - StopWaitTimeout: 250 * time.Millisecond, - RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault, - } + cfg = taskQueue.NewConfig() + cfg.Workers = 2 + cfg.StartManagerDelay = time.Millisecond + cfg.DispatchTasksDelay = time.Millisecond + cfg.StopWaitTimeout = 250 * time.Millisecond }) AfterEach(func() { @@ -782,8 +971,8 @@ var _ = Describe("Queue", func() { return test.Must(str.NewTaskRepository().GetTask(ctx, createdTask.ID, nil)).State }, "5s", "50ms").To(Equal(task.TaskStateRunning)) - // Stop must return within roughly the stop timeout even though the runner never - // returns; it abandons the in-flight task rather than blocking forever. + // Stop must return within roughly the stop timeout even though the runner never returns; it abandons + // the in-flight task rather than blocking forever. stopped := make(chan struct{}) go func() { defer GinkgoRecover() @@ -798,9 +987,9 @@ var _ = Describe("Queue", func() { It("logs and counts the run once its timeout elapses", func() { cfg.RunnerWatchdogGracePeriod = 50 * time.Millisecond - // A short duration maximum yields a short runner timeout (3x), and a short grace - // period keeps the watchdog prompt, so the watchdog fires quickly while the runner - // is still blocked, without an unstick reclaiming the task. + // A short duration maximum yields a short runner timeout (2x), and a short grace period keeps the + // watchdog prompt, so the watchdog fires quickly while the runner is still blocked, without an unstick + // reclaiming the task. runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). WithDurationMaximum(20 * time.Millisecond). WithStub(func(ctx context.Context, tsk *task.Task) { select {} }) @@ -847,14 +1036,11 @@ var _ = Describe("Queue", func() { runner = taskQueueTest.NewStubRunner(taskTest.RandomType()). WithStub(func(ctx context.Context, tsk *task.Task) { tsk.State = task.TaskStatePending }) - cfg := &taskQueue.Config{ - Workers: workersCount, - Delay: time.Millisecond, - DelayInitial: time.Millisecond, - DelayUnstick: time.Millisecond, - StopWaitTimeout: taskQueue.StopWaitTimeoutDefault, - RunnerWatchdogGracePeriod: taskQueue.RunnerWatchdogGracePeriodDefault, - } + cfg := taskQueue.NewConfig() + cfg.Workers = workersCount + cfg.StartManagerDelay = time.Millisecond + cfg.DispatchTasksDelay = time.Millisecond + cfg.UnstickTasksDelay = time.Millisecond ques = make([]*taskQueue.Queue, queueCount) for index := range len(ques) { ques[index] = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) diff --git a/task/queue/runner.go b/task/queue/runner.go index 2cf2fb1c4b..84d58fd5bf 100644 --- a/task/queue/runner.go +++ b/task/queue/runner.go @@ -4,101 +4,113 @@ import ( "context" "time" + "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/task" ) -// Runner processes one task type on the queue. This comment is the complete contract; no other -// queue code need be consulted to implement it. +// Runner processes one task type on the queue. This comment is the complete contract; no other queue code need be +// consulted to implement it. // -// Registration: one Runner per task type, registered at queue construction (duplicate types -// fail construction). The queue owns the single instance for the queue's lifetime. +// Registration: one Runner per task type, registered at queue construction (duplicate types fail construction). The +// queue owns the single instance for the queue's lifetime. // -// Concurrency: the queue runs multiple workers (configurable; 5 by default) that share the one Runner instance, -// so Run is called concurrently from multiple goroutines — any state shared across calls must be -// synchronized. Each call's *task.Task is distinct and call-owned: don't retain it past Run or -// touch another call's task. The Get* methods are also called concurrently and must be stable -// (in practice, constants). +// Concurrency: the queue runs multiple workers (configurable; 5 by default) that share the one Runner instance, so Run +// is called concurrently from multiple goroutines - any state shared across calls must be synchronized. Each call's +// *task.Task is distinct and call-owned: don't retain it past Run or touch another call's task. The Get* methods are +// also called concurrently and must be stable (in practice, constants). // // Duration contract, validated at construction (violations fail construction): // // GetRunnerDeadline() > GetRunnerTimeout() > GetRunnerDurationMaximum() > 0 // -// Run's obligation: the task arrives running and already claimed under a unique per-run state -// lock — a token the queue set when it claimed the task; a later write-back to the task only lands -// while that same lock is still in place (see the lost-write note under Delivery). Before -// returning, Run MUST move it out of running by calling exactly one terminal helper on tsk: +// Run's obligation: the task arrives running and already claimed under a unique per-run claim token - a value the queue +// set when it claimed the task; a later write-back to the task only lands while that same token is still in place (see +// the lost-write note under Delivery). Before returning, Run MUST move it out of running by calling exactly one +// terminal helper on tsk: // -// - tsk.SetCompleted() — success; done. -// - tsk.SetFailed() — permanent failure; done. -// - tsk.SetFailedWithError(e) — permanent failure with error; done. -// - tsk.RepeatAvailableAfter(d) — reschedule after duration d. -// - tsk.RepeatAvailableAt(t) — reschedule at time t. +// - tsk.SetCompleted() - success; done. +// - tsk.SetFailed() - permanent failure; done. +// - tsk.SetFailedWithError(e) - permanent failure with error; done. +// - tsk.RepeatAvailableAfter(d) - reschedule after duration d. +// - tsk.RepeatAvailableAt(t) - reschedule at time t. // -// The task arrives with AvailableTime cleared, so reschedule via the Repeat* helpers (which set -// both pending state and a new available time); a reschedule time in the past is clamped to now -// (run ASAP). Setting pending without an available time is a bug the queue only papers over with -// a warning. If Run returns with the task still running, the -// queue: reverts it to pending if interrupted by shutdown (parent context canceled); else fails -// it, recording the timeout as cause if the context was canceled for timeout, otherwise a -// "runner failed to set state" error. +// The task arrives with AvailableTime cleared, so reschedule via the Repeat* helpers (which set both pending state and +// a new available time); a reschedule time in the past is clamped to now (run ASAP). Setting pending without an +// available time is a bug the queue only papers over with a warning. If Run returns with the task still running, the +// queue: reverts it to pending if interrupted by shutdown (parent context canceled); else fails it, recording the +// timeout as cause if the context was canceled for timeout, otherwise a "runner failed to set state" error. // -// Context: canceled (with cause) after GetRunnerTimeout and on shutdown — cooperative runners -// select on ctx.Done() and return promptly. To tell the two apart, inspect context.Cause(ctx): a -// timeout cancels with ErrRunnerTimeout (errors.Is(context.Cause(ctx), ErrRunnerTimeout)), a -// shutdown with context.Canceled. The queue cannot preempt a runner that ignores cancellation; it -// stays blocked until it returns or the process restarts, recovered by the deadline/unstick -// mechanism (a periodic sweep that resets tasks still running past their deadline — see -// GetRunnerDeadline). The logger is on the context (log.LoggerFromContext(ctx)). Wrap must-complete writes -// in context.WithoutCancel so cancellation doesn't abandon them. +// Context: canceled (with cause) after GetRunnerTimeout, on shutdown, and on claim loss - cooperative runners select on +// ctx.Done() and return promptly. To tell them apart, inspect context.Cause(ctx): a normal shutdown with +// context.Canceled; a runner timeout with ErrRunnerTimeoutExceeded, and a lost claim (the task was deleted mid-run, or +// unstuck and its claim token replaced) with ErrClaimLost (use errors.Is in all cases). After a claim loss the run's +// outcome is discarded - no write-back can land - so return promptly and skip any remaining work. Claim loss is +// detected by a periodic check (MonitorTaskDelay, 1 minute by default), not instantly. The queue cannot preempt a +// runner that ignores cancellation; it stays blocked until it returns or the process restarts, recovered by the +// deadline/unstick mechanism (a periodic sweep that resets tasks still running past their deadline - see +// GetRunnerDeadline). The logger is on the context (log.LoggerFromContext(ctx)). So that cancellation doesn't abandon +// them, wrap must-complete writes in: // -// Delivery is at-least-once, so Run must tolerate re-execution of the same logical task: a runner -// that overruns its deadline is unstuck and re-dispatched (possibly while the original run is still -// in flight), and a crash or kill mid-run re-runs the task from the start. Conversely a final write -// can be lost — if the task was unstuck and re-claimed while Run was still working, the queue's -// write-back on return no longer matches the state lock and is dropped (logged and counted, not -// applied). Design terminal effects to be idempotent and safe to repeat. +// ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) +// defer cancel() // -// Errors: accumulate with tsk.AppendError(err), reset with tsk.ClearError() (commonly at run -// start). Setting an error does not change state — pair it with a terminal helper. Persisted with -// the task; logged at error level if failed, warning if rescheduled. A panic is recovered and an -// "unhandled panic" error attached; the task is then failed unless the runner had already moved it -// out of running before panicking (e.g. a task completed before the panic stays completed, with -// the error attached) or a concurrent shutdown reverts it to pending instead. +// Delivery is at-least-once, so Run must tolerate re-execution of the same logical task: a runner that overruns its +// deadline is unstuck and re-dispatched (possibly while the original run is still in flight), and a crash or kill +// mid-run re-runs the task from the start. Conversely a final write can be lost - if the task was unstuck and +// re-claimed while Run was still working, the queue's write-back on return no longer matches the claim token and is +// dropped (logged and counted, not applied). Design terminal effects to be idempotent and safe to repeat. // -// Persistence: on return the queue persists only the terminal state, the available time (written -// only when rescheduling; a terminal task's available time was already cleared when the task was -// claimed), tsk.Data (mutate the map in place, or assign a new map if it arrived nil; the map -// is written as a whole-field replacement, so its final contents are exactly what is stored — keys -// dropped from the map are gone, and a nil map unsets the field), and the accumulated error — -// mutate only these. Run time and duration are recorded by the queue; other fields are -// queue-managed, don't set them. To persist Data mid-run, mutate tsk -// and let the queue write it back on return; a Runner that instead updates its OWN task via -// task.Client must replace tsk with the returned task or the queue's write-back clobbers it (see -// task.Client.UpdateTask). +// Errors: accumulate with tsk.AppendError(err), reset with tsk.ClearError() (commonly at run start). Setting an error +// does not change state - pair it with a terminal helper. Persisted with the task; logged at error level if the task +// failed, warning otherwise (e.g. rescheduled). A panic is recovered and an "unhandled panic" error attached; the task +// is then failed unless the runner had already moved it out of running before panicking (e.g. a task completed before +// the panic stays completed, with the error attached). Unlike a normal return, a panic skips the post-run +// reconciliation, so a panic during shutdown fails the task rather than reverting it to pending. +// +// Persistence: on return the queue persists only the terminal state, the available time (written only when +// rescheduling; a terminal task's available time was already cleared when the task was claimed), tsk.Data (mutate the +// map in place, or assign a new map if it arrived nil; the map is written as a whole-field replacement, so its final +// contents are exactly what is stored - keys dropped from the map are gone, and a nil map unsets the field), and the +// accumulated error - mutate only these. Run time and duration are recorded by the queue; other fields are +// queue-managed, don't set them. To persist Data mid-run, mutate tsk and let the queue write it back on return; a +// Runner that instead updates its OWN task via task.Client must replace tsk with the returned task or the queue's +// write-back clobbers it (see task.Client.UpdateTask). +// +// NOTE: The task system is superseded by the work system. New asynchronous processing implements work.Processor and +// registers a work.ProcessorFactory (see the work/base helpers, and oura/data/work for examples). type Runner interface { - // GetRunnerType returns the task type this runner processes. Unique across a queue's runners; - // must match task.Task.Type of the tasks it handles. + // GetRunnerType returns the task type this runner processes. Unique across a queue's runners; must match + // task.Task.Type of the tasks it handles. GetRunnerType() string - // GetRunnerDeadline returns the duration, measured from when the task starts running, after - // which a still-running task is forcibly reset to pending/available by the unstick mechanism, - // recovering tasks orphaned by crashes, kills, or a runner that never returns. The reset is not - // immediate: unstick runs on a periodic sweep, so a task is recovered on the first sweep after - // its deadline elapses. Must exceed GetRunnerTimeout so a task still within its timeout is never - // unstuck and re-claimed mid-run. + // GetRunnerDeadline returns the duration, measured from when the task starts running, after which a still-running + // task is forcibly reset to pending/available by the unstick mechanism, recovering tasks orphaned by crashes, + // kills, or a runner that never returns. The reset is not immediate: unstick runs on a periodic sweep, so a task is + // recovered on the first sweep after its deadline elapses. Must exceed GetRunnerTimeout so a task still within its + // timeout is never unstuck and re-claimed mid-run. GetRunnerDeadline() time.Duration - // GetRunnerTimeout returns the hard cap on a run; at elapse the Run context is canceled so a - // cooperative runner can return. Must exceed GetRunnerDurationMaximum. + // GetRunnerTimeout returns the hard cap on a run; at elapse the Run context is canceled so a cooperative runner can + // return. Must exceed GetRunnerDurationMaximum. GetRunnerTimeout() time.Duration - // GetRunnerDurationMaximum returns the expected upper bound of a normal run. Advisory: exceeding - // it logs a warning but does not interrupt the run. Must be positive. + // GetRunnerDurationMaximum returns the expected upper bound of a normal run. Advisory: exceeding it logs a warning + // but does not interrupt the run. Must be positive. GetRunnerDurationMaximum() time.Duration - // Run executes tsk within ctx. Before returning it must move tsk out of running (completed, - // failed, or rescheduled via Repeat*); a task left running is failed by the queue, unless - // interrupted by shutdown, in which case it is reverted to pending. ctx is canceled after - // GetRunnerTimeout and on shutdown. Called concurrently for distinct tasks; must be concurrency-safe. + // Run executes tsk within ctx. Before returning it must move tsk out of running (completed, failed, or rescheduled + // via Repeat*); a task left running is failed by the queue, unless interrupted by shutdown, in which case it is + // reverted to pending. ctx is canceled after GetRunnerTimeout, on shutdown, and when the run's claim on the task is + // lost. Called concurrently for distinct tasks; must be concurrency-safe. Run(ctx context.Context, tsk *task.Task) } + +// ErrRunnerTimeoutExceeded is the cancellation cause set on the Run context when a run exceeds the runner timeout. A +// runner distinguishes a timeout from a shutdown with errors.Is(context.Cause(ctx), ErrRunnerTimeoutExceeded); a +// shutdown instead cancels with context.Canceled. +var ErrRunnerTimeoutExceeded = errors.New("task runner timeout exceeded") + +// ErrClaimLost is the cancellation cause set on the Run context when the run's claim on the task was lost mid-run: the +// task was deleted, or it was unstuck and possibly re-claimed (its claim token no longer matches). A runner +// distinguishes it with errors.Is(context.Cause(ctx), ErrClaimLost). +var ErrClaimLost = errors.New("task claim lost") diff --git a/task/store/mongo/mongo.go b/task/store/mongo/mongo.go index 64003cce77..3e8c20d1b3 100644 --- a/task/store/mongo/mongo.go +++ b/task/store/mongo/mongo.go @@ -2,6 +2,7 @@ package mongo import ( "context" + "fmt" "slices" "time" @@ -28,8 +29,8 @@ import ( const ( MaxTaskCreationDuration = 30 * time.Second - // TransitionTimeout bounds a task state-transition write (start or stop) that must - // complete regardless of the caller's context being canceled (e.g. during shutdown). + // TransitionTimeout bounds a task state-transition write (start or stop) that must complete regardless of the + // caller's context being canceled (e.g. during shutdown). TransitionTimeout = 10 * time.Second ) @@ -123,15 +124,15 @@ func (t *TaskRepository) EnsureIndexes() error { Keys: bson.D{{Key: "state", Value: 1}}, }, { - // Used by IteratePending; type equality, then availableTime for range and sort; partial - // on pending since that is the only state it queries. + // Used by IteratePending; type equality, then availableTime for range and sort; partial on pending since + // that is the only state it queries. Keys: bson.D{{Key: "type", Value: 1}, {Key: "availableTime", Value: 1}}, Options: options.Index(). SetPartialFilterExpression(bson.D{{Key: "state", Value: task.TaskStatePending}}), }, { - // Used by UnstickTasks; type equality, then deadlineTime for range and sort; partial - // on running since that is the only state it queries. + // Used by UnstickTasks; type equality, then deadlineTime for range and sort; partial on running since that + // is the only state it queries. Keys: bson.D{{Key: "type", Value: 1}, {Key: "deadlineTime", Value: 1}}, Options: options.Index(). SetPartialFilterExpression(bson.D{{Key: "state", Value: task.TaskStateRunning}}), @@ -389,9 +390,9 @@ func (t *TaskRepository) StartTask(ctx context.Context, id string, revision int, return nil, errors.New("deadline is invalid") } - // Add a timeout, but ignore cancel from the parent context so the claim write completes - // and its outcome is known. A write abandoned mid-flight (e.g. on shutdown) can still - // commit in the database, leaving the task running with no reliable way to revert it. + // Add a timeout, but ignore cancel from the parent context so the claim write completes and its outcome is known. A + // write abandoned mid-flight (e.g. on shutdown) can still commit in the database, leaving the task running with no + // reliable way to revert it. ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), TransitionTimeout) defer cancel() @@ -405,7 +406,7 @@ func (t *TaskRepository) StartTask(ctx context.Context, id string, revision int, "runTime": now, "deadlineTime": now.Add(deadline), "modifiedTime": now, - "stateLock": newStateLock(), + "claimToken": newClaimTokenWithRevision(revision), } unset := bson.M{ "availableTime": 1, @@ -430,7 +431,7 @@ func (t *TaskRepository) StartTask(ctx context.Context, id string, revision int, } // Will only timeout after 10 seconds even if parent context is canceled. -func (t *TaskRepository) StopTask(ctx context.Context, id string, revision int, stateLock *string, state string, duration *time.Duration, update *task.TaskUpdate) error { +func (t *TaskRepository) StopTask(ctx context.Context, id string, revision int, claimToken *string, state string, duration *time.Duration, update *task.TaskUpdate) error { if ctx == nil { return errors.New("context is missing") } @@ -439,10 +440,10 @@ func (t *TaskRepository) StopTask(ctx context.Context, id string, revision int, } else if !task.IsValidID(id) { return errors.New("id is invalid") } - if stateLock == nil { - return errors.New("state lock is missing") - } else if *stateLock == "" { - return errors.New("state lock is invalid") + if claimToken == nil { + return errors.New("claim token is missing") + } else if *claimToken == "" { + return errors.New("claim token is invalid") } if state == "" { return errors.New("state is missing") @@ -462,7 +463,7 @@ func (t *TaskRepository) StopTask(ctx context.Context, id string, revision int, ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), TransitionTimeout) defer cancel() - logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "revision": revision, "stateLock": stateLock, "state": state, "duration": duration, "update": update}) + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id, "revision": revision, "claimToken": claimToken, "state": state, "duration": duration, "update": update}) now := time.Now().UTC() defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("StopTask") }() @@ -471,30 +472,30 @@ func (t *TaskRepository) StopTask(ctx context.Context, id string, revision int, set["modifiedTime"] = now set["state"] = state unset["deadlineTime"] = 1 - unset["stateLock"] = 1 + unset["claimToken"] = 1 if duration != nil { set["duration"] = duration.Truncate(time.Millisecond).Seconds() } else { - // A nil duration means no run actually happened (e.g. a claimed task whose dispatch - // was reverted during shutdown), so also clear the run time recorded by StartTask, - // keeping the runTime/duration pair describing only the last actual run. + // A nil duration means no run actually happened (e.g. a claimed task whose dispatch was reverted during + // shutdown), so also clear the run time recorded by StartTask, keeping the runTime/duration pair describing + // only the last actual run. unset["duration"] = 1 unset["runTime"] = 1 } selector := t.selector(id, nil) selector["state"] = task.TaskStateRunning - selector["stateLock"] = stateLock + selector["claimToken"] = claimToken - tsk := &task.Task{} - err := t.FindOneAndUpdate(ctx, selector, t.ConstructUpdate(set, unset)).Decode(tsk) + partial := &task.Task{} + opts := options.FindOneAndUpdate().SetProjection(bson.M{"_id": 0, "type": 1, "revision": 1}) + err := t.FindOneAndUpdate(ctx, selector, t.ConstructUpdate(set, unset), opts).Decode(partial) if errors.Is(err, mongo.ErrNoDocuments) { - // The compare-and-swap missed: no running task matched the expected state lock - // (it was concurrently modified, unstuck, or deleted since it started). - // The state transition is dropped; the deadline and unstick mechanism will - // recover the task if it was left running. This is logged and counted so lost - // completions are observable rather than silently swallowed. - logger.Error("Unable to stop task; no running task matched the expected condition") + // The compare-and-swap missed: no running task matched the expected claim token (it was concurrently modified, + // unstuck, or deleted since it started). The state transition is dropped; the deadline and unstick mechanism + // will recover the task if it was left running. This is logged and counted so lost completions are observable + // rather than silently swallowed. + logger.Error("Unable to stop task; no running task matched the id and claim token") TypeLostCompletionTotal.WithLabelValues(pointer.Default(t.typeFilter, "")).Inc() return nil } else if err != nil { @@ -503,22 +504,28 @@ func (t *TaskRepository) StopTask(ctx context.Context, id string, revision int, } // If the on-disk task revision does not match the runner task revision, then either: - // - the runner did not follow the Runner contract; i.e. the runner updated the task during - // run, but it did not use the updated task, or, + // - the runner did not follow the Runner contract; i.e. the runner updated the task during run, but it did not + // use the updated task, or, // - the task was concurrently modified outside of the runner while the task was running. - if tsk.Revision != revision { - logger.WithField("revision", log.Fields{"expected": revision, "actual": tsk.Revision}).Warn("Database task revision does not match running task revision; Runner contract broken or concurrent update") - TypeRevisionMismatchTotal.WithLabelValues(tsk.Type).Inc() + if partial.Revision != revision { + logger.WithField("revision", log.Fields{"expected": revision, "actual": partial.Revision}).Warn("Database task revision does not match running task revision; Runner contract broken or concurrent update") + TypeRevisionMismatchTotal.WithLabelValues(partial.Type).Inc() } - TypeStateTotal.WithLabelValues(tsk.Type, state).Inc() + TypeStateTotal.WithLabelValues(partial.Type, state).Inc() return nil } -func (t *TaskRepository) UnstickTasks(ctx context.Context) ([]string, error) { +// UnstickTasks resets tasks still running past their deadline back to pending, making each available again +// availabilityDelay in the future - long enough for the (possibly still running) previous run to observe its claim loss +// and exit before the task is re-dispatched. +func (t *TaskRepository) UnstickTasks(ctx context.Context, availabilityDelay time.Duration) ([]string, error) { if ctx == nil { return nil, errors.New("context is missing") } + if availabilityDelay < 0 { + return nil, errors.New("availability delay is invalid") + } logger := log.LoggerFromContext(ctx) @@ -533,7 +540,9 @@ func (t *TaskRepository) UnstickTasks(ctx context.Context) ([]string, error) { findSelector["type"] = *t.typeFilter } - opts := options.Find().SetSort(bson.M{"deadlineTime": 1}) + opts := options.Find(). + SetProjection(bson.M{"_id": 0, "id": 1, "deadlineTime": 1}). + SetSort(bson.M{"deadlineTime": 1}) cursor, err := t.Find(ctx, findSelector, opts) if err != nil { logger = logger.WithError(err) @@ -543,35 +552,35 @@ func (t *TaskRepository) UnstickTasks(ctx context.Context) ([]string, error) { var ids []string for cursor.Next(ctx) { - tsk := &task.Task{} - if err = cursor.Decode(tsk); err != nil { + partial := &task.Task{} + if err = cursor.Decode(partial); err != nil { logger = logger.WithError(err) log.LoggerFromContext(ctx).WithError(err).Error("Unable to decode task") continue } - // The state clause is defensive: a matching deadline time already implies the same - // running claim (every stop clears the deadline time and any re-claim records a - // strictly later one), but including it keeps the invariant local to this update. + // The state clause is defensive: a matching deadline time already implies the same running claim (every stop + // clears the deadline time and any re-claim records a strictly later one), but including it keeps the invariant + // local to this update. updateSelector := bson.M{ - "id": tsk.ID, + "id": partial.ID, "state": task.TaskStateRunning, - "deadlineTime": tsk.DeadlineTime, + "deadlineTime": partial.DeadlineTime, } set := bson.M{ "state": task.TaskStatePending, - "availableTime": now, + "availableTime": now.Add(availabilityDelay), "modifiedTime": now, } unset := bson.M{ "deadlineTime": 1, - "stateLock": 1, + "claimToken": 1, } if result, updateErr := t.UpdateOne(ctx, updateSelector, t.ConstructUpdate(set, unset)); updateErr != nil { logger = logger.WithError(updateErr) - return ids, updateErr + return ids, errors.Wrap(updateErr, "unable to update task") } else if result.ModifiedCount > 0 { - ids = append(ids, tsk.ID) + ids = append(ids, partial.ID) } } @@ -585,6 +594,32 @@ func (t *TaskRepository) UnstickTasks(ctx context.Context) ([]string, error) { return ids, nil } +func (t *TaskRepository) GetTaskClaimToken(ctx context.Context, id string) (*string, bool, error) { + if ctx == nil { + return nil, false, errors.New("context is missing") + } + if id == "" { + return nil, false, errors.New("id is missing") + } + + logger := log.LoggerFromContext(ctx).WithFields(log.Fields{"id": id}) + + now := time.Now().UTC() + defer func() { logger.WithField("duration", time.Since(now)/time.Microsecond).Debug("GetTaskClaimToken") }() + + partial := &task.Task{} + opts := options.FindOne().SetProjection(bson.M{"_id": 0, "claimToken": 1}) + err := t.FindOne(ctx, t.selector(id, nil), opts).Decode(partial) + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, false, nil + } else if err != nil { + logger = logger.WithError(err) + return nil, false, errors.Wrap(err, "unable to get task claim token") + } + + return partial.ClaimToken, true, nil +} + func (t *TaskRepository) IteratePending(ctx context.Context) (*mongo.Cursor, error) { now := time.Now().UTC() @@ -657,13 +692,16 @@ func (t *TaskRepository) parseUpdate(update *task.TaskUpdate) (bson.M, bson.M) { // assertType return an error if the expected type doesn't match the actual type func (t *TaskRepository) assertType(expected *string, actual *string) error { if expected != nil && actual != nil && *expected != *actual { - return errors.Newf("expected task type %s but got %s", *expected, *actual) + return errors.Newf("expected task type %q, but got %q", *expected, *actual) } return nil } -func newStateLock() string { - return id.Must(id.New(16)) +// newClaimTokenWithRevision embeds the claimed revision so successive claims of the same task can never collide +// (revision is strictly monotonic per document); the random suffix distinguishes claims across tasks. The token is +// opaque everywhere else. +func newClaimTokenWithRevision(revision int) string { + return fmt.Sprintf("%d:%s", revision, id.Must(id.New(16))) } var ( @@ -673,20 +711,18 @@ var ( Help: "The total number of tasks run, sorted by type and state", }, []string{"type", "state"}) - // TypeLostCompletionTotal counts task completions dropped because the compare-and-swap in - // StopTask missed (task state lock was concurrently modified, task unstuck, or task deleted). The task - // is recovered by the deadline/unstick mechanism, but the intended terminal state is lost. The - // type label is populated only when the repository is type-filtered (as it is for each queue in - // a MultiQueue); otherwise it is empty. + // TypeLostCompletionTotal counts task completions dropped because the compare-and-swap in StopTask missed (task + // claim token was concurrently modified, task unstuck, or task deleted). The task is recovered by the + // deadline/unstick mechanism, but the intended terminal state is lost. The type label is populated only when the + // repository is type-filtered (as it is for each queue in a MultiQueue); otherwise it is empty. TypeLostCompletionTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "tidepool_task_type_lost_completion_total", - Help: "The total number of task completions dropped because the state-lock compare-and-swap missed, sorted by type", + Help: "The total number of task completions dropped because the claim-token compare-and-swap missed, sorted by type", }, []string{"type"}) - // TypeRevisionMismatchTotal counts task completions where the task revision does not match - // the task revision in the database. This only occurs if the task runner does not follow the Runner - // contract when updating a task during run, or if there is a concurrent modification outside of - // the expected Runner behavior. + // TypeRevisionMismatchTotal counts task completions where the task revision does not match the task revision in the + // database. This only occurs if the task runner does not follow the Runner contract when updating a task during + // run, or if there is a concurrent modification outside of the expected Runner behavior. TypeRevisionMismatchTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "tidepool_task_type_revision_mismatch_total", Help: "The total number of task revisions that do not match the task revision in the database, sorted by type", diff --git a/task/store/mongo/mongo_test.go b/task/store/mongo/mongo_test.go index 35006ba2eb..f833f22f30 100644 --- a/task/store/mongo/mongo_test.go +++ b/task/store/mongo/mongo_test.go @@ -2,6 +2,7 @@ package mongo_test import ( "context" + "strconv" "time" . "github.com/onsi/ginkgo/v2" @@ -130,7 +131,76 @@ var _ = Describe("Mongo", func() { ctx = log.NewContextWithLogger(context.Background(), logger) }) - Context("with an existing task", func() { + Context("StartTask", func() { + It("embeds the claimed revision in the claim token", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(startedTask.ClaimToken).To(PointTo(HavePrefix(strconv.Itoa(pendingTask.Revision) + ":"))) + }) + + It("generates a distinct claim token for successive claims of the same task", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + firstStartedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(firstStartedTask).ToNot(BeNil()) + Expect(repository.StopTask(ctx, firstStartedTask.ID, firstStartedTask.Revision, firstStartedTask.ClaimToken, task.TaskStatePending, nil, nil)).To(Succeed()) + + stoppedTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": pendingTask.ID}).Decode(stoppedTask)).To(Succeed()) + + secondStartedTask := test.Must(repository.StartTask(ctx, stoppedTask.ID, stoppedTask.Revision, time.Minute)) + Expect(secondStartedTask).ToNot(BeNil()) + Expect(secondStartedTask.ClaimToken).To(PointTo(Not(Equal(*firstStartedTask.ClaimToken)))) + }) + }) + + Context("StopTask", func() { + It("clears the run time and duration when stopping without a duration", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + Expect(pendingTask.AvailableTime).ToNot(BeNil()) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(startedTask.RunTime).ToNot(BeNil()) + Expect(startedTask.AvailableTime).To(BeNil()) + + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.ClaimToken, task.TaskStatePending, nil, nil)).To(Succeed()) + + actualTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) + Expect(actualTask.State).To(Equal(task.TaskStatePending)) + Expect(actualTask.RunTime).To(BeNil()) + Expect(actualTask.Duration).To(BeNil()) + Expect(actualTask.ClaimToken).To(BeNil()) + Expect(actualTask.DeadlineTime).To(BeNil()) + Expect(actualTask.AvailableTime).To(BeNil()) + }) + + It("retains the run time when stopping with a duration", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + Expect(pendingTask.AvailableTime).ToNot(BeNil()) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(startedTask.RunTime).ToNot(BeNil()) + + duration := time.Second + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.ClaimToken, task.TaskStateCompleted, &duration, nil)).To(Succeed()) + + actualTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) + Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) + Expect(actualTask.RunTime).To(PointTo(BeTemporally("~", *startedTask.RunTime, time.Millisecond))) + Expect(actualTask.Duration).To(PointTo(Equal(duration.Seconds()))) + Expect(actualTask.ClaimToken).To(BeNil()) + Expect(actualTask.DeadlineTime).To(BeNil()) + Expect(actualTask.AvailableTime).To(BeNil()) + }) + }) + + Context("UnstickTasks", func() { var tsk *task.Task BeforeEach(func() { @@ -147,119 +217,210 @@ var _ = Describe("Mongo", func() { Expect(err).ToNot(HaveOccurred()) }) - Context("UnstickTasks", func() { - It("returns an error when the context is missing", func() { - unstuckTaskIDs, err := repository.UnstickTasks(context.Context(nil)) - Expect(err).To(MatchError("context is missing")) - Expect(unstuckTaskIDs).To(BeNil()) - }) + It("returns an error when the context is missing", func() { + unstuckTaskIDs, err := repository.UnstickTasks(context.Context(nil), 0) + Expect(err).To(MatchError("context is missing")) + Expect(unstuckTaskIDs).To(BeNil()) + }) - It("returns no ids when there are no stuck tasks", func() { - unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) - Expect(unstuckTaskIDs).To(BeEmpty()) - }) + It("returns an error when the availability delay is negative", func() { + unstuckTaskIDs, err := repository.UnstickTasks(ctx, -time.Second) + Expect(err).To(MatchError("availability delay is invalid")) + Expect(unstuckTaskIDs).To(BeNil()) + }) - It("unsticks a running task with an expired deadline", func() { - stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + It("returns no ids when there are no stuck tasks", func() { + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(BeEmpty()) + }) - unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) - Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) + It("unsticks a running task with an expired deadline", func() { + stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) - actualStuckTask := &task.Task{} - Expect(collection.FindOne(ctx, bson.M{"id": stuckTask.ID}).Decode(actualStuckTask)).To(Succeed()) - Expect(actualStuckTask.State).To(Equal(task.TaskStatePending)) - Expect(actualStuckTask.AvailableTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) - Expect(actualStuckTask.ModifiedTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) - Expect(actualStuckTask.DeadlineTime).To(BeNil()) - }) + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) - It("does not unstick a running task with a deadline in the future", func() { - notStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(time.Hour))) + actualStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": stuckTask.ID}).Decode(actualStuckTask)).To(Succeed()) + Expect(actualStuckTask.State).To(Equal(task.TaskStatePending)) + Expect(actualStuckTask.AvailableTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) + Expect(actualStuckTask.ModifiedTime).To(PointTo(BeTemporally("~", test.Now(), time.Second))) + Expect(actualStuckTask.DeadlineTime).To(BeNil()) + }) - unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) - Expect(unstuckTaskIDs).To(BeEmpty()) + It("offsets the available time by the availability delay", func() { + stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) - actualNotStuckTask := &task.Task{} - Expect(collection.FindOne(ctx, bson.M{"id": notStuckTask.ID}).Decode(actualNotStuckTask)).To(Succeed()) - Expect(actualNotStuckTask).To(Equal(notStuckTask)) - }) + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, time.Minute)) + Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) - It("does not unstick a task that is not running", func() { - notStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, pointer.FromTime(test.Now().Add(-time.Minute))) + actualStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": stuckTask.ID}).Decode(actualStuckTask)).To(Succeed()) + Expect(actualStuckTask.State).To(Equal(task.TaskStatePending)) + Expect(actualStuckTask.AvailableTime).To(PointTo(BeTemporally("~", test.Now().Add(time.Minute), time.Second))) + }) - unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) - Expect(unstuckTaskIDs).To(BeEmpty()) + It("does not unstick a running task with a deadline in the future", func() { + notStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(time.Hour))) - actualNotStuckTask := &task.Task{} - Expect(collection.FindOne(ctx, bson.M{"id": notStuckTask.ID}).Decode(actualNotStuckTask)).To(Succeed()) - Expect(actualNotStuckTask).To(Equal(notStuckTask)) - }) + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(BeEmpty()) + + actualNotStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": notStuckTask.ID}).Decode(actualNotStuckTask)).To(Succeed()) + Expect(actualNotStuckTask).To(Equal(notStuckTask)) + }) - It("unsticks multiple running tasks with expired deadlines, ordered by deadline time", func() { - laterStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) - earlierStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Hour))) + It("does not unstick a task that is not running", func() { + notStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, pointer.FromTime(test.Now().Add(-time.Minute))) - unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx)) - Expect(unstuckTaskIDs).To(Equal([]string{earlierStuckTask.ID, laterStuckTask.ID})) - }) + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(BeEmpty()) - It("only unsticks tasks matching the repository type filter", func() { - stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) - otherTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + actualNotStuckTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": notStuckTask.ID}).Decode(actualNotStuckTask)).To(Succeed()) + Expect(actualNotStuckTask).To(Equal(notStuckTask)) + }) - filteredRepository := str.WithTypeFilter(stuckTask.Type).NewTaskRepository() - unstuckTaskIDs, err := filteredRepository.UnstickTasks(ctx) - Expect(err).ToNot(HaveOccurred()) - Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) + It("unsticks multiple running tasks with expired deadlines, ordered by deadline time", func() { + laterStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + earlierStuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Hour))) - actualOtherTask := &task.Task{} - Expect(collection.FindOne(ctx, bson.M{"id": otherTask.ID}).Decode(actualOtherTask)).To(Succeed()) - Expect(actualOtherTask).To(Equal(otherTask)) - }) + unstuckTaskIDs := test.Must(repository.UnstickTasks(ctx, 0)) + Expect(unstuckTaskIDs).To(Equal([]string{earlierStuckTask.ID, laterStuckTask.ID})) + }) + + It("only unsticks tasks matching the repository type filter", func() { + stuckTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + otherTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStateRunning, pointer.FromTime(test.Now().Add(-time.Minute))) + + filteredRepository := str.WithTypeFilter(stuckTask.Type).NewTaskRepository() + unstuckTaskIDs, err := filteredRepository.UnstickTasks(ctx, 0) + Expect(err).ToNot(HaveOccurred()) + Expect(unstuckTaskIDs).To(ConsistOf(stuckTask.ID)) + + actualOtherTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": otherTask.ID}).Decode(actualOtherTask)).To(Succeed()) + Expect(actualOtherTask).To(Equal(otherTask)) }) }) - Context("StopTask", func() { - It("clears the run time and duration when stopping without a duration", func() { + Context("GetTaskClaimToken", func() { + It("returns an error when the context is missing", func() { + claimToken, exists, err := repository.GetTaskClaimToken(context.Context(nil), taskTest.RandomID()) + Expect(err).To(MatchError("context is missing")) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeFalse()) + }) + + It("returns an error when the id is missing", func() { + claimToken, exists, err := repository.GetTaskClaimToken(ctx, "") + Expect(err).To(MatchError("id is missing")) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeFalse()) + }) + + It("reports the task as not existing when the task does not exist", func() { + claimToken, exists, err := repository.GetTaskClaimToken(ctx, taskTest.RandomID()) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeFalse()) + }) + + It("returns a nil claim token when the task exists, but is not claimed", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + Expect(pendingTask.ClaimToken).To(BeNil()) + + claimToken, exists, err := repository.GetTaskClaimToken(ctx, pendingTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeTrue()) + }) + + It("returns the claim token of a claimed task", func() { pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) - Expect(pendingTask.AvailableTime).ToNot(BeNil()) startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) Expect(startedTask).ToNot(BeNil()) - Expect(startedTask.RunTime).ToNot(BeNil()) - Expect(startedTask.AvailableTime).To(BeNil()) + Expect(startedTask.ClaimToken).ToNot(BeNil()) - Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.StateLock, task.TaskStatePending, nil, nil)).To(Succeed()) + claimToken, exists, err := repository.GetTaskClaimToken(ctx, startedTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(PointTo(Equal(*startedTask.ClaimToken))) + Expect(exists).To(BeTrue()) + }) - actualTask := &task.Task{} - Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) - Expect(actualTask.State).To(Equal(task.TaskStatePending)) - Expect(actualTask.RunTime).To(BeNil()) - Expect(actualTask.Duration).To(BeNil()) - Expect(actualTask.StateLock).To(BeNil()) - Expect(actualTask.DeadlineTime).To(BeNil()) - Expect(actualTask.AvailableTime).To(BeNil()) + It("returns a nil claim token once the task is stopped", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.ClaimToken, task.TaskStateCompleted, nil, nil)).To(Succeed()) + + claimToken, exists, err := repository.GetTaskClaimToken(ctx, startedTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeTrue()) }) - It("retains the run time when stopping with a duration", func() { + It("returns the new claim token once the task is re-claimed", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + firstStartedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(firstStartedTask).ToNot(BeNil()) + Expect(repository.StopTask(ctx, firstStartedTask.ID, firstStartedTask.Revision, firstStartedTask.ClaimToken, task.TaskStatePending, nil, nil)).To(Succeed()) + + stoppedTask := &task.Task{} + Expect(collection.FindOne(ctx, bson.M{"id": pendingTask.ID}).Decode(stoppedTask)).To(Succeed()) + + secondStartedTask := test.Must(repository.StartTask(ctx, stoppedTask.ID, stoppedTask.Revision, time.Minute)) + Expect(secondStartedTask).ToNot(BeNil()) + + claimToken, exists, err := repository.GetTaskClaimToken(ctx, secondStartedTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(PointTo(Equal(*secondStartedTask.ClaimToken))) + Expect(claimToken).To(PointTo(Not(Equal(*firstStartedTask.ClaimToken)))) + Expect(exists).To(BeTrue()) + }) + + It("does not modify the task", func() { pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) - Expect(pendingTask.AvailableTime).ToNot(BeNil()) startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) Expect(startedTask).ToNot(BeNil()) - Expect(startedTask.RunTime).ToNot(BeNil()) - duration := time.Second - Expect(repository.StopTask(ctx, startedTask.ID, startedTask.Revision, startedTask.StateLock, task.TaskStateCompleted, &duration, nil)).To(Succeed()) + _, _, err := repository.GetTaskClaimToken(ctx, startedTask.ID) + Expect(err).ToNot(HaveOccurred()) actualTask := &task.Task{} Expect(collection.FindOne(ctx, bson.M{"id": startedTask.ID}).Decode(actualTask)).To(Succeed()) - Expect(actualTask.State).To(Equal(task.TaskStateCompleted)) - Expect(actualTask.RunTime).To(PointTo(BeTemporally("~", *startedTask.RunTime, time.Millisecond))) - Expect(actualTask.Duration).To(PointTo(Equal(duration.Seconds()))) - Expect(actualTask.StateLock).To(BeNil()) - Expect(actualTask.DeadlineTime).To(BeNil()) - Expect(actualTask.AvailableTime).To(BeNil()) + Expect(actualTask).To(Equal(startedTask)) + }) + + It("returns the claim token of a task matching the repository type filter", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + + filteredRepository := str.WithTypeFilter(startedTask.Type).NewTaskRepository() + claimToken, exists, err := filteredRepository.GetTaskClaimToken(ctx, startedTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(PointTo(Equal(*startedTask.ClaimToken))) + Expect(exists).To(BeTrue()) + }) + + It("reports the task as not existing when it does not match the repository type filter", func() { + pendingTask := insertTaskWithStateAndDeadlineTime(ctx, collection, task.TaskStatePending, nil) + + startedTask := test.Must(repository.StartTask(ctx, pendingTask.ID, pendingTask.Revision, time.Minute)) + Expect(startedTask).ToNot(BeNil()) + + filteredRepository := str.WithTypeFilter(startedTask.Type + "-other").NewTaskRepository() + claimToken, exists, err := filteredRepository.GetTaskClaimToken(ctx, startedTask.ID) + Expect(err).ToNot(HaveOccurred()) + Expect(claimToken).To(BeNil()) + Expect(exists).To(BeFalse()) }) }) diff --git a/task/store/store.go b/task/store/store.go index e2f45fd096..52cea0bfa2 100644 --- a/task/store/store.go +++ b/task/store/store.go @@ -24,10 +24,13 @@ type TaskRepository interface { UpdateTask(ctx context.Context, id string, condition *storeStructured.Condition, update *task.TaskUpdate) (*task.Task, error) DeleteTask(ctx context.Context, id string, condition *storeStructured.Condition) error - UnstickTasks(ctx context.Context) ([]string, error) + // Queue use only below StartTask(ctx context.Context, id string, revision int, deadline time.Duration) (*task.Task, error) - StopTask(ctx context.Context, id string, revision int, stateLock *string, state string, duration *time.Duration, update *task.TaskUpdate) error + StopTask(ctx context.Context, id string, revision int, claimToken *string, state string, duration *time.Duration, update *task.TaskUpdate) error + + UnstickTasks(ctx context.Context, availabilityDelay time.Duration) ([]string, error) + GetTaskClaimToken(ctx context.Context, id string) (*string, bool, error) IteratePending(ctx context.Context) (*mongo.Cursor, error) } diff --git a/task/task.go b/task/task.go index 034448e35e..8783b84c02 100644 --- a/task/task.go +++ b/task/task.go @@ -22,11 +22,6 @@ type Client interface { ListTasks(ctx context.Context, filter *TaskFilter, pagination *page.Pagination) (Tasks, error) CreateTask(ctx context.Context, create *TaskCreate) (*Task, error) GetTask(ctx context.Context, id string, condition *request.Condition) (*Task, error) - // UpdateTask applies the update and returns the resulting task. A runner that updates its - // own running task (for example, to persist progress in Data) must replace its in-memory - // task with the returned task: the update changes fields such as the revision, and the - // queue writes the in-memory task's fields back when it completes the task, so a stale - // in-memory copy would overwrite the update. UpdateTask(ctx context.Context, id string, condition *request.Condition, update *TaskUpdate) (*Task, error) DeleteTask(ctx context.Context, id string, condition *request.Condition) error } @@ -188,8 +183,9 @@ type Task struct { // Database only - // Use to enforce only one state transition at a time. This is a unique value that changes on every update. - StateLock *string `json:"-" bson:"stateLock,omitempty"` + // ClaimToken fences state transitions: a unique value set each time a run claims the task, which a later stop must + // match, so stale writers miss. + ClaimToken *string `json:"-" bson:"claimToken,omitempty"` DeadlineTime *time.Time `json:"-" bson:"deadlineTime,omitempty"` } @@ -292,11 +288,12 @@ func (t *Task) IsFailed() bool { func (t *Task) SetFailed() { t.State = TaskStateFailed + t.AvailableTime = nil } func (t *Task) SetFailedWithError(err error) { - t.State = TaskStateFailed t.AppendError(err) + t.SetFailed() } func (t *Task) IsCompleted() bool { @@ -333,11 +330,11 @@ func (t *Task) ClearError() { func (t *Task) LogFields() log.Fields { return log.Fields{ - "id": t.ID, - "type": t.Type, - "state": t.State, - "revision": t.Revision, - "stateLock": t.StateLock, + "id": t.ID, + "type": t.Type, + "state": t.State, + "revision": t.Revision, + "claimToken": t.ClaimToken, } } diff --git a/task/task_test.go b/task/task_test.go index 4ff3ca0df2..6bd583b448 100644 --- a/task/task_test.go +++ b/task/task_test.go @@ -99,7 +99,7 @@ var _ = Describe("Task", func() { "CreatedTime": BeTemporally("~", time.Now(), time.Second), "ModifiedTime": BeNil(), "Revision": Equal(1), - "StateLock": BeNil(), + "ClaimToken": BeNil(), }))) }) @@ -122,11 +122,11 @@ var _ = Describe("Task", func() { It("returns the id, type, and state as log fields", func() { tsk := taskTest.RandomTask() Expect(tsk.LogFields()).To(Equal(log.Fields{ - "id": tsk.ID, - "type": tsk.Type, - "state": tsk.State, - "revision": tsk.Revision, - "stateLock": tsk.StateLock, + "id": tsk.ID, + "type": tsk.Type, + "state": tsk.State, + "revision": tsk.Revision, + "claimToken": tsk.ClaimToken, })) }) }) From a00b87a7afde2a120107ed3153d0a06eaaf36b1e Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Tue, 28 Jul 2026 11:38:26 -0700 Subject: [PATCH 12/20] Fix intermittent test failures --- Makefile | 2 +- env.sh | 4 +- env.test.sh | 7 ++- task/queue/queue.go | 11 ++--- task/queue/queue_test.go | 92 +++++++++++++++++++-------------------- task/queue/test/runner.go | 4 +- test/test.go | 2 +- 7 files changed, 64 insertions(+), 58 deletions(-) diff --git a/Makefile b/Makefile index d4da95207d..d92862e5bd 100644 --- a/Makefile +++ b/Makefile @@ -240,7 +240,7 @@ vet: tmp @echo "go vet ./..." @cd $(ROOT_DIRECTORY) && \ { [ -z `go env GOWORK` ] || GOWORK_FLAGS=-mod=readonly; } && \ - go vet $${GOWORK_FLAGS:-} ./... > _tmp/govet.out 2>&1 || \ + $(TIMING_CMD) go vet $${GOWORK_FLAGS:-} ./... > _tmp/govet.out 2>&1 || \ (diff .govetignore _tmp/govet.out && exit 1) vet-ignore: diff --git a/env.sh b/env.sh index 13f8a66e31..5dc4e67cb6 100644 --- a/env.sh +++ b/env.sh @@ -1,6 +1,8 @@ +# shellcheck shell=sh + export TIDEPOOL_ENV="local" export TIDEPOOL_LOGGER_LEVEL="debug" -export TIDEPOOL_STORE_ADDRESSES="localhost" +export TIDEPOOL_STORE_ADDRESSES="127.0.0.1" export TIDEPOOL_STORE_DATABASE="tidepool" export TIDEPOOL_STORE_TLS="false" export TIDEPOOL_SERVER_TLS="false" diff --git a/env.test.sh b/env.test.sh index cd71c2c36b..5d893811bd 100644 --- a/env.test.sh +++ b/env.test.sh @@ -1,11 +1,14 @@ +# shellcheck shell=sh + # Clear all TIDEPOOL_* environment variables -unset `env | cut -d'=' -f1 | grep '^TIDEPOOL_' | xargs` +# shellcheck disable=SC2046 +unset $(env | cut -d'=' -f1 | grep '^TIDEPOOL_' | xargs) 2> /dev/null || true export TIDEPOOL_ENV="test" export TIDEPOOL_LOGGER_LEVEL="error" -export TIDEPOOL_STORE_ADDRESSES="localhost" +export TIDEPOOL_STORE_ADDRESSES="127.0.0.1" export TIDEPOOL_STORE_TLS="false" export TIDEPOOL_STORE_DATABASE="tidepool_test" diff --git a/task/queue/queue.go b/task/queue/queue.go index 3997db3cd7..07cfc18744 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -444,18 +444,19 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { return } - // If the runner left the task running, reconcile its state based on why the run ended. + // If the runner left the task running, reconcile its state based on why the run ended. A claim lost after the check + // above matches neither branch; the outcome is discarded during completion. Any other run left running is failed + // with a missing terminal state error during completion. if tsk.State == task.TaskStateRunning { - switch { - case context.Cause(ctx) != nil: + if context.Cause(ctx) != nil { // The parent (worker) context was canceled by shutdown; make the task available again for retry rather than // treating the interruption as a completion. tsk.RepeatAvailableAfter(0) - case context.Cause(runnerContext) != nil: + } else if cause := context.Cause(runnerContext); errors.Is(cause, ErrRunnerTimeoutExceeded) { // The runner exceeded its timeout but returned; record the cause so the failure is attributed to the // timeout rather than the generic missing terminal state error. lgr.Warn("Task runner exceeded timeout; task will be failed") - tsk.AppendError(context.Cause(runnerContext)) + tsk.AppendError(cause) RunnerTimeoutExceededTotal.WithLabelValues(runner.GetRunnerType(), "recovered").Inc() } } diff --git a/task/queue/queue_test.go b/task/queue/queue_test.go index 4957fb50da..f4f377f92e 100644 --- a/task/queue/queue_test.go +++ b/task/queue/queue_test.go @@ -409,9 +409,7 @@ var _ = Describe("Queue", func() { AfterEach(func() { _ = test.Must(str.GetRepository("tasks").DeleteMany(ctx, bson.M{})) - if str != nil { - Expect(str.Terminate(ctx)).To(Succeed()) - } + Expect(str.Terminate(ctx)).To(Succeed()) }) Context("with successful shutdown", func() { @@ -474,9 +472,9 @@ var _ = Describe("Queue", func() { updatedData := metadataTest.RandomMetadataMap() runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { - *tsk = *test.Must(str.NewTaskRepository().UpdateTask(ctx, tsk.ID, nil, &task.TaskUpdate{Data: &updatedData})) - tsk.SetCompleted() + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + *runnerTask = *test.Must(str.NewTaskRepository().UpdateTask(runnerContext, runnerTask.ID, nil, &task.TaskUpdate{Data: &updatedData})) + runnerTask.SetCompleted() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -498,9 +496,9 @@ var _ = Describe("Queue", func() { updatedData := metadataTest.RandomMetadataMap() runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { - test.Must(str.NewTaskRepository().UpdateTask(ctx, tsk.ID, nil, &task.TaskUpdate{Data: &updatedData})) - tsk.SetCompleted() + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + test.Must(str.NewTaskRepository().UpdateTask(runnerContext, runnerTask.ID, nil, &task.TaskUpdate{Data: &updatedData})) + runnerTask.SetCompleted() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -522,11 +520,11 @@ var _ = Describe("Queue", func() { It("does not complete a task whose claim token changed while it was running", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { // Simulate the task being unstuck and re-claimed elsewhere by changing the claim token out from // under this run. The completion must then miss rather than falsely complete another run task. - test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": tsk.ID}, bson.M{"$set": bson.M{"claimToken": ""}})) - tsk.SetCompleted() + test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": runnerTask.ID}, bson.M{"$set": bson.M{"claimToken": ""}})) + runnerTask.SetCompleted() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -546,9 +544,9 @@ var _ = Describe("Queue", func() { It("does not complete a task was deleted while it was running", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { - test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": tsk.ID})) - tsk.SetCompleted() + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": runnerTask.ID})) + runnerTask.SetCompleted() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -572,11 +570,11 @@ var _ = Describe("Queue", func() { canceled := make(chan error, 1) runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { - test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": tsk.ID})) - <-ctx.Done() - canceled <- context.Cause(ctx) - tsk.SetCompleted() + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": runnerTask.ID})) + <-runnerContext.Done() + canceled <- context.Cause(runnerContext) + runnerTask.SetCompleted() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -603,14 +601,14 @@ var _ = Describe("Queue", func() { canceled := make(chan error, 1) runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { // Run past several claim checks before losing the claim, so the monitor must keep checking // rather than settle after the first. time.Sleep(5 * cfg.MonitorTaskDelay) - test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": tsk.ID})) - <-ctx.Done() - canceled <- context.Cause(ctx) - tsk.SetCompleted() + test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": runnerTask.ID})) + <-runnerContext.Done() + canceled <- context.Cause(runnerContext) + runnerTask.SetCompleted() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -636,13 +634,13 @@ var _ = Describe("Queue", func() { canceled := make(chan error, 1) runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { // Simulate the task being unstuck and re-claimed elsewhere by changing the claim token out from // under this run. - test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": tsk.ID}, bson.M{"$set": bson.M{"claimToken": "other-claim-token"}})) - <-ctx.Done() - canceled <- context.Cause(ctx) - tsk.SetCompleted() + test.Must(str.GetCollection("tasks").UpdateOne(ctx, bson.M{"id": runnerTask.ID}, bson.M{"$set": bson.M{"claimToken": "other-claim-token"}})) + <-runnerContext.Done() + canceled <- context.Cause(runnerContext) + runnerTask.SetCompleted() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -668,14 +666,14 @@ var _ = Describe("Queue", func() { cfg.MonitorTaskDelay = time.Millisecond runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { // Linger across several claim checks; the claim is intact, so the run must not be canceled. select { - case <-ctx.Done(): - tsk.AppendError(context.Cause(ctx)) - tsk.SetFailed() + case <-runnerContext.Done(): + runnerTask.AppendError(context.Cause(runnerContext)) + runnerTask.SetFailed() case <-time.After(250 * time.Millisecond): - tsk.SetCompleted() + runnerTask.SetCompleted() } }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -694,7 +692,7 @@ var _ = Describe("Queue", func() { It("cleans up a task that panics during execution", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { panic("panic test") }) + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { panic("panic test") }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) @@ -716,7 +714,7 @@ var _ = Describe("Queue", func() { It("fails a task whose runner leaves it in an unknown state", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { tsk.State = "unknown-state" }) + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { runnerTask.State = "unknown-state" }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) @@ -735,9 +733,9 @@ var _ = Describe("Queue", func() { It("warns and sets the available time for a pending task left without one", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { - tsk.State = task.TaskStatePending - tsk.AvailableTime = nil + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + runnerTask.State = task.TaskStatePending + runnerTask.AvailableTime = nil }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) @@ -759,7 +757,9 @@ var _ = Describe("Queue", func() { It("warns when a pending task available time is significantly in the past", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { tsk.RepeatAvailableAt(time.Now().Add(-2 * time.Minute)) }) + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { + runnerTask.RepeatAvailableAt(time.Now().Add(-2 * time.Minute)) + }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) @@ -816,7 +816,7 @@ var _ = Describe("Queue", func() { It("reverts a task that is still running to pending when the queue is stopped", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { <-ctx.Done() }) + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { <-runnerContext.Done() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) @@ -876,7 +876,7 @@ var _ = Describe("Queue", func() { It("fails a task that exceeds its timeout without setting its own state", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). WithDurationMaximum(100 * time.Millisecond). - WithStub(func(ctx context.Context, tsk *task.Task) { <-ctx.Done() }) + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { <-runnerContext.Done() }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) @@ -959,7 +959,7 @@ var _ = Describe("Queue", func() { It("returns from Stop within the stop timeout when a runner ignores cancellation", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). WithDurationMaximum(time.Minute). - WithStub(func(ctx context.Context, tsk *task.Task) { select {} }) + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { select {} }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) createdTask := test.Must(str.NewTaskRepository().CreateTask(ctx, &task.TaskCreate{Type: runner.GetRunnerType()})) @@ -992,7 +992,7 @@ var _ = Describe("Queue", func() { // reclaiming the task. runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). WithDurationMaximum(20 * time.Millisecond). - WithStub(func(ctx context.Context, tsk *task.Task) { select {} }) + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { select {} }) que = test.Must(taskQueue.New(taskTest.RandomType(), cfg, lgr, str, runner)) runnerType := runner.GetRunnerType() @@ -1034,7 +1034,7 @@ var _ = Describe("Queue", func() { BeforeEach(func() { runner = taskQueueTest.NewStubRunner(taskTest.RandomType()). - WithStub(func(ctx context.Context, tsk *task.Task) { tsk.State = task.TaskStatePending }) + WithStub(func(runnerContext context.Context, runnerTask *task.Task) { runnerTask.State = task.TaskStatePending }) cfg := taskQueue.NewConfig() cfg.Workers = workersCount diff --git a/task/queue/test/runner.go b/task/queue/test/runner.go index b9ddc69ba9..5e0aba14f8 100644 --- a/task/queue/test/runner.go +++ b/task/queue/test/runner.go @@ -103,7 +103,7 @@ var _ queue.Runner = &SleepRunner{} type StubRunner struct { Type string - Stub func(ctx context.Context, tsk *task.Task) + Stub func(runnerContext context.Context, runnerTask *task.Task) deadline *time.Duration timeout *time.Duration durationMaximum *time.Duration @@ -149,7 +149,7 @@ func (s *StubRunner) Run(ctx context.Context, tsk *task.Task) { } } -func (s *StubRunner) WithStub(stub func(ctx context.Context, tsk *task.Task)) *StubRunner { +func (s *StubRunner) WithStub(stub func(runnerContext context.Context, runnerTask *task.Task)) *StubRunner { s.Stub = stub return s } diff --git a/test/test.go b/test/test.go index 652a2c7180..59e0c88612 100644 --- a/test/test.go +++ b/test/test.go @@ -67,6 +67,6 @@ func (m *MockMatcher) String() string { } func Must[T any](value T, err error) T { - gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(err).WithOffset(1).ToNot(gomega.HaveOccurred()) return value } From 4019768968aca41a53f923a41de81c8a888cc53f Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Tue, 28 Jul 2026 12:45:28 -0700 Subject: [PATCH 13/20] Update for broken database tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c6092f786c..c06fd94e6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ before_install: - sudo apt-get install --allow-downgrades -y docker-buildx-plugin mongodb-org=${MONGODB} mongodb-org-database=${MONGODB} mongodb-org-server=${MONGODB} mongodb-mongosh=${MONGOSH} mongodb-org-mongos=${MONGODB} mongodb-org-tools - mkdir -p /var/ramfs/mongodb/data - /usr/bin/mongod --dbpath /var/ramfs/mongodb/data --bind_ip 127.0.0.1 --replSet rs0 --logpath /var/ramfs/mongodb/mongod.log &> /dev/null & - - until nc -z localhost 27017; do echo Waiting for MongoDB; sleep 1; done + - until nc -z 127.0.0.1 27017; do echo Waiting for MongoDB; sleep 1; done - /usr/bin/mongosh --eval 'rs.initiate(); while (rs.status().startupStatus || (rs.status().hasOwnProperty("myState") && rs.status().myState != 1)) { printjson( rs.status() ); sleep(1000); }; printjson( rs.status() );' - echo -e "machine github.com\n login $GITHUB_TOKEN" > ~/.netrc From 4afe97f24e1533e7d35196843d3f63d0949ea474 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Tue, 28 Jul 2026 15:13:23 -0700 Subject: [PATCH 14/20] Minor test updates --- dexcom/fetch/runner_test.go | 7 +++++++ task/store/mongo/mongo_test.go | 16 ---------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/dexcom/fetch/runner_test.go b/dexcom/fetch/runner_test.go index fb6ae562fd..8676303678 100644 --- a/dexcom/fetch/runner_test.go +++ b/dexcom/fetch/runner_test.go @@ -322,6 +322,10 @@ var _ = Describe("Runner", func() { Expect(tsk.HasError()).To(BeFalse()) } + assertDataSourceLastImportTimePresent := func() { + Expect(dataSrc.LastImportTime).ToNot(BeNil()) + } + It("fails if provider session id is missing and update data source returns an error", func() { testErr := errorsTest.RandomError() delete(tsk.Data, dexcom.DataKeyProviderSessionID) @@ -591,6 +595,7 @@ var _ = Describe("Runner", func() { assertTaskDeviceHashesCount(3) assertTaskRetryCountNotPresent() assertTaskAndDataSourceErrorNotPresent() + assertDataSourceLastImportTimePresent() assertProviderSessionRefreshedTimes(6) }) @@ -608,6 +613,7 @@ var _ = Describe("Runner", func() { assertTaskAvailableSoon() assertTaskRetryCountNotPresent() assertTaskAndDataSourceErrorNotPresent() + assertDataSourceLastImportTimePresent() assertProviderSessionRefreshedTimes(6) }) @@ -627,6 +633,7 @@ var _ = Describe("Runner", func() { assertTaskAvailableSoon() assertTaskRetryCountNotPresent() assertTaskError(dexcomFetch.ErrorCodeResourceFailure, "unable to update data source") + assertDataSourceLastImportTimePresent() assertProviderSessionRefreshedTimes(6) }) }) diff --git a/task/store/mongo/mongo_test.go b/task/store/mongo/mongo_test.go index f833f22f30..cb2ce78425 100644 --- a/task/store/mongo/mongo_test.go +++ b/task/store/mongo/mongo_test.go @@ -201,22 +201,6 @@ var _ = Describe("Mongo", func() { }) Context("UnstickTasks", func() { - var tsk *task.Task - - BeforeEach(func() { - var err error - tsk, err = task.NewTask(context.Background(), &task.TaskCreate{ - Name: pointer.FromString("test"), - Type: "fetch", - Data: nil, - AvailableTime: pointer.FromTime(time.Now()), - }) - Expect(err).ToNot(HaveOccurred()) - tsk.State = task.TaskStateRunning - _, err = collection.InsertOne(ctx, tsk) - Expect(err).ToNot(HaveOccurred()) - }) - It("returns an error when the context is missing", func() { unstuckTaskIDs, err := repository.UnstickTasks(context.Context(nil), 0) Expect(err).To(MatchError("context is missing")) From 1503c641dff963d0be9c06d98dacf59b413d7ef0 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Wed, 29 Jul 2026 08:59:10 -0700 Subject: [PATCH 15/20] Updates from code review --- task/queue/queue.go | 6 +++--- task/queue/queue_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/task/queue/queue.go b/task/queue/queue.go index 07cfc18744..e05803aea0 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -180,9 +180,9 @@ func (c *Config) Validate() error { } // The Queue's fields are all immutable after New, except the lifecycle fields, which are guarded by the lifecycle -// mutex, and workersAvailable, which is initialized by Start before the manager exists and thereafter owned exclusively -// by the manager goroutine. The workers and manager therefore read the channels and runners map freely, without -// synchronization. +// mutex, and workersAvailable, which is initialized by Start before the manager exists and is thereafter owned +// exclusively by the manager goroutine. The workers and manager therefore read the channels and runners map freely, +// without synchronization. type Queue struct { name string config *Config diff --git a/task/queue/queue_test.go b/task/queue/queue_test.go index f4f377f92e..2d04f62fc1 100644 --- a/task/queue/queue_test.go +++ b/task/queue/queue_test.go @@ -542,7 +542,7 @@ var _ = Describe("Queue", func() { Expect(actualTask.State).To(Equal(task.TaskStateRunning)) }) - It("does not complete a task was deleted while it was running", func() { + It("does not complete a task that was deleted while it was running", func() { runner := taskQueueTest.NewStubRunner(taskTest.RandomType()). WithStub(func(runnerContext context.Context, runnerTask *task.Task) { test.Must(str.GetCollection("tasks").DeleteOne(ctx, bson.M{"id": runnerTask.ID})) From f698aff696f7b396eb33c49ea8d23c47735bdecc Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Wed, 29 Jul 2026 14:29:15 -0700 Subject: [PATCH 16/20] Fix intermittent test failures --- data/source/store/structured/mongo/mongo_test.go | 1 + dexcom/client/client_test.go | 2 +- store/structured/mongo/test/config.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/data/source/store/structured/mongo/mongo_test.go b/data/source/store/structured/mongo/mongo_test.go index 8092205470..a56b7275ad 100644 --- a/data/source/store/structured/mongo/mongo_test.go +++ b/data/source/store/structured/mongo/mongo_test.go @@ -112,6 +112,7 @@ var _ = Describe("Mongo", func() { store, err = dataSourceStoreStructuredMongo.NewStore(config) Expect(err).ToNot(HaveOccurred()) Expect(store).ToNot(BeNil()) + Expect(store.Ping(context.Background())).ToNot(HaveOccurred()) mongoCollection = store.GetCollection("data_sources") }) diff --git a/dexcom/client/client_test.go b/dexcom/client/client_test.go index bebb889638..17c832ae1d 100644 --- a/dexcom/client/client_test.go +++ b/dexcom/client/client_test.go @@ -1556,7 +1556,7 @@ var _ = Describe("Client", func() { It("does not record a request time metric when the request-time header is not a valid duration", func() { header := http.Header{} - header.Set(dexcomClient.RequestTimeHeaderName, test.RandomString()) + header.Set(dexcomClient.RequestTimeHeaderName, test.RandomStringFromCharset(test.CharsetAlpha)) testRoundTripper.Response = &http.Response{StatusCode: testHttp.NewStatusCode(), Header: header} _ = test.Must(roundTripper.RoundTrip(request)) diff --git a/store/structured/mongo/test/config.go b/store/structured/mongo/test/config.go index 4981ba9798..810294f34a 100644 --- a/store/structured/mongo/test/config.go +++ b/store/structured/mongo/test/config.go @@ -12,6 +12,6 @@ func NewConfig() *storeStructuredMongo.Config { Addresses: []string{Address()}, Database: Database(), CollectionPrefix: NewCollectionPrefix(), - Timeout: 5 * time.Second, + Timeout: 15 * time.Second, } } From 25b1a82e1decb61aaec015b285955fbc25345a2f Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Wed, 29 Jul 2026 16:46:36 -0700 Subject: [PATCH 17/20] Fix PR review issues --- README.md | 6 ++--- client/client.go | 42 +++++++++++++++++++++++++++++--- client/client_test.go | 35 ++++++++++++++++++++++++++ store/structured/mongo/result.go | 13 +++++++--- test/time.go | 28 ++++++++++++--------- 5 files changed, 101 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index a500323fb5..7600c22fd6 100644 --- a/README.md +++ b/README.md @@ -191,8 +191,8 @@ See source files for further details about and usage of each metric. #### Queue -* `tidepool_task_workers_total` - (gauge) - configured number of task queue workers, sorted by queue (5, per config) -* `tidepool_task_workers_available` - (gauge) - number of available task queue workers, sorted by queue (5, per config) +* `tidepool_task_workers_total` - (gauge) - configured number of task queue workers, sorted by queue (per config) +* `tidepool_task_workers_available` - (gauge) - number of available task queue workers, sorted by queue * `tidepool_task_runner_not_found_total` - (counter) - total number of task runs with no registered runner for the task type, sorted by type (ideally zero) * `tidepool_task_run_duration_seconds` - (histogram) - duration of task runs in seconds, sorted by type * `tidepool_task_runner_timeout_exceeded_total` - (counter) - total number of task runs that exceeded the runner timeout, sorted by type and disposition ("blocked", "recovered") (ideally zero) @@ -201,5 +201,5 @@ See source files for further details about and usage of each metric. #### Store * `tidepool_task_type_state_total` - (counter) - total number of tasks run, sorted by type and state -* `tidepool_task_type_lost_completion_total` - (counter) - total number of task completions dropped because the state-lock compare-and-swap missed, sorted by type (ideally low-ish) +* `tidepool_task_type_lost_completion_total` - (counter) - total number of task completions dropped because the claim-token compare-and-swap missed, sorted by type (ideally low-ish) * `tidepool_task_type_revision_mismatch_total` - (counter) - total number of task revisions that do not match the task revision in the database, sorted by type (ideally zero) diff --git a/client/client.go b/client/client.go index 208fae5e38..5e416c3427 100644 --- a/client/client.go +++ b/client/client.go @@ -9,6 +9,7 @@ import ( "net/url" "reflect" "strings" + "time" "unicode/utf8" "github.com/tidepool-org/platform/errors" @@ -76,23 +77,37 @@ func (c *Client) AppendURLQuery(urlString string, query map[string]string) strin } func (c *Client) RequestStreamWithHTTPClient(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody interface{}, inspectors []request.ResponseInspector, httpClient *http.Client) (io.ReadCloser, error) { + if ctx == nil { + return nil, errors.New("context is missing") + } if httpClient == nil { return nil, errors.New("http client is missing") } + // The request must carry a cancelable context for the timeout to reach the transport. A deadline on that context + // cannot be used, though, as it would also abort any read of the returned response body, so instead cancel via a + // timer that is stopped once the response headers arrive. The cause preserves the deadline exceeded error. + ctx, cancel := context.WithCancelCause(ctx) + req, err := c.createRequest(ctx, method, url, mutators, requestBody) if err != nil { + cancel(nil) return nil, err } + var timer *time.Timer if c.config.Timeout > 0 { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, c.config.Timeout) - defer cancel() + timer = time.AfterFunc(c.config.Timeout, func() { cancel(context.DeadlineExceeded) }) } res, err := httpClient.Do(req) + + if timer != nil { + timer.Stop() + } + if err != nil { + cancel(nil) return nil, errors.Wrapf(err, "unable to perform request to %s %s", method, url) } @@ -100,7 +115,16 @@ func (c *Client) RequestStreamWithHTTPClient(ctx context.Context, method string, inspector.InspectResponse(res) } - return c.handleResponse(ctx, res, req) + body, err := c.handleResponse(ctx, res, req) + if body == nil { + cancel(nil) + return nil, err + } + + return &ReadCloserWithCancelCause{ + ReadCloser: body, + CancelCauseFunc: cancel, + }, err } func (c *Client) RequestDataWithHTTPClient(ctx context.Context, method string, url string, mutators []request.RequestMutator, requestBody interface{}, responseBody interface{}, inspectors []request.ResponseInspector, httpClient *http.Client) error { @@ -251,6 +275,16 @@ func drainAndClose(reader io.ReadCloser) { reader.Close() } +type ReadCloserWithCancelCause struct { + io.ReadCloser + context.CancelCauseFunc +} + +func (r *ReadCloserWithCancelCause) Close() error { + defer r.CancelCauseFunc(nil) + return r.ReadCloser.Close() +} + func NewSerializableErrorResponseParser() *SerializableErrorResponseParser { return &SerializableErrorResponseParser{} } diff --git a/client/client_test.go b/client/client_test.go index 03fe31d91a..f59adc5cc6 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -773,6 +774,40 @@ var _ = Describe("Client", func() { Expect(server.ReceivedRequests()).To(HaveLen(1)) }) }) + + Context("with a timeout", func() { + var timeout time.Duration + + BeforeEach(func() { + timeout = 100 * time.Millisecond + config.Timeout = timeout + }) + + It("returns an error if the response is not received within the timeout", func() { + server.AppendHandlers(func(res http.ResponseWriter, req *http.Request) { + time.Sleep(3 * timeout) + }) + + reader, err = clnt.RequestStreamWithHTTPClient(ctx, method, url, mutators, requestBody, inspectors, httpClient) + Expect(errors.Is(errors.Cause(err), context.DeadlineExceeded)).To(BeTrue()) + Expect(reader).To(BeNil()) + }) + + It("does not apply the timeout to reading the response body", func() { + server.AppendHandlers(func(res http.ResponseWriter, req *http.Request) { + res.WriteHeader(http.StatusOK) + res.(http.Flusher).Flush() + time.Sleep(3 * timeout) + res.Write([]byte(responseString)) + }) + + reader, err = clnt.RequestStreamWithHTTPClient(ctx, method, url, mutators, requestBody, inspectors, httpClient) + Expect(err).ToNot(HaveOccurred()) + Expect(reader).ToNot(BeNil()) + Expect(io.ReadAll(reader)).To(Equal([]byte(responseString))) + Expect(server.ReceivedRequests()).To(HaveLen(1)) + }) + }) }) Context("RequestDataWithHTTPClient", func() { diff --git a/store/structured/mongo/result.go b/store/structured/mongo/result.go index 4b03364299..adab4efc36 100644 --- a/store/structured/mongo/result.go +++ b/store/structured/mongo/result.go @@ -97,15 +97,20 @@ func BSONToAny(input any) any { } } +//nolint:contextcheck // Use a background context if the provided context is nil func CloseCursor(ctx context.Context, cursor *mongo.Cursor) { if cursor == nil { return } - if ctx != nil { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer cancel() + + if ctx == nil { + ctx = context.Background() } + + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := cursor.Close(ctx); err != nil { if lgr := log.LoggerFromContext(ctx); lgr != nil { lgr.WithError(err).Warn("Unable to close cursor") diff --git a/test/time.go b/test/time.go index 02db1cbed2..57cc1ab9ba 100644 --- a/test/time.go +++ b/test/time.go @@ -10,23 +10,27 @@ import ( ) func Now() time.Time { - return now + return timeInNormalizedLocation(time.Now().Truncate(time.Millisecond).UTC()) +} + +func NowStable() time.Time { + return nowStable } func PastFarTime() time.Time { - return now.AddDate(-30, 0, 0) + return NowStable().AddDate(-30, 0, 0) } func PastNearTime() time.Time { - return now.AddDate(0, -1, 0) + return NowStable().AddDate(0, -1, 0) } func FutureNearTime() time.Time { - return now.AddDate(0, 1, 0) + return NowStable().AddDate(0, 1, 0) } func FutureFarTime() time.Time { - return now.AddDate(30, 0, 0) + return NowStable().AddDate(30, 0, 0) } func MustTime(value time.Time, err error) time.Time { @@ -37,11 +41,11 @@ func MustTime(value time.Time, err error) time.Time { } func RandomTimeBeforeNow() time.Time { - return RandomTimeBefore(now) + return RandomTimeBefore(Now()) } func RandomTimeAfterNow() time.Time { - return RandomTimeAfter(now) + return RandomTimeAfter(Now()) } func RandomTimeBefore(value time.Time) time.Time { @@ -77,11 +81,11 @@ func RandomTimeFromRange(minimum time.Time, maximum time.Time) time.Time { } func RandomTimeMaximum() time.Time { - return now.Add(RandomDurationMaximum()).Truncate(time.Millisecond) + return NowStable().Add(RandomDurationMaximum()).Truncate(time.Millisecond) } func RandomTimeMinimum() time.Time { - return now.Add(RandomDurationMinimum()).Truncate(time.Millisecond) + return NowStable().Add(RandomDurationMinimum()).Truncate(time.Millisecond) } func NewObjectFromTime(value time.Time, objectFormat ObjectFormat) interface{} { @@ -102,7 +106,7 @@ func PinnedTime(value time.Time) time.Time { } else if value.After(RandomTimeMaximum()) { return RandomTimeMaximum() } else { - return normalizeLocation(value.Truncate(time.Millisecond)) + return timeInNormalizedLocation(value.Truncate(time.Millisecond)) } } @@ -113,11 +117,11 @@ func MatchTime(datum *time.Time) gomegaTypes.GomegaMatcher { return gomegaGstruct.PointTo(gomega.BeTemporally("==", *datum)) } -func normalizeLocation(value time.Time) time.Time { +func timeInNormalizedLocation(value time.Time) time.Time { if value.Location() == time.Local && time.Local.String() == "UTC" { value = value.In(time.UTC) } return value } -var now = normalizeLocation(time.Now().Truncate(time.Millisecond).UTC()) +var nowStable = Now() From ca3d2b0faffea9bc06408065a21c6af7c306c0dd Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Thu, 6 Aug 2026 13:57:48 -0700 Subject: [PATCH 18/20] Update abbott submodule --- private/plugin/abbott | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private/plugin/abbott b/private/plugin/abbott index 1b63c21c2d..d25d5a2e9b 160000 --- a/private/plugin/abbott +++ b/private/plugin/abbott @@ -1 +1 @@ -Subproject commit 1b63c21c2d4a4147fd1692b9a49400727f72d761 +Subproject commit d25d5a2e9bc5f18eb84001fde1e98737fe4068c8 From 889fc23da3b395f7232e61cfe24789f3225a84e5 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Sat, 8 Aug 2026 16:04:21 -0700 Subject: [PATCH 19/20] Fix client duration parsing - Fix client duration parsing - Add generalized duration parsing - Add and update test --- auth/client/external.go | 13 ++-- client/config.go | 19 +++-- client/config_test.go | 131 ++++++++++++++++++++++++++++++++ duration/duration.go | 42 ++++++++++ duration/duration_suite_test.go | 11 +++ duration/duration_test.go | 85 +++++++++++++++++++++ service/server/config.go | 12 ++- task/queue/queue.go | 71 +++++++---------- 8 files changed, 319 insertions(+), 65 deletions(-) create mode 100644 duration/duration.go create mode 100644 duration/duration_suite_test.go create mode 100644 duration/duration_test.go diff --git a/auth/client/external.go b/auth/client/external.go index d3901d2bdb..2292964ac7 100644 --- a/auth/client/external.go +++ b/auth/client/external.go @@ -3,7 +3,6 @@ package client import ( "context" "net/http" - "strconv" "sync" "time" @@ -16,6 +15,7 @@ import ( "github.com/tidepool-org/platform/auth" "github.com/tidepool-org/platform/client" "github.com/tidepool-org/platform/config" + "github.com/tidepool-org/platform/duration" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/permission" @@ -380,13 +380,10 @@ func (l *externalConfigReporterLoader) Load(cfg *ExternalConfig) error { return err } cfg.ServerSessionTokenSecret = l.Reporter.GetWithDefault("server_session_token_secret", "") - if serverSessionTokenTimeoutString, err := l.Reporter.Get("server_session_token_timeout"); err == nil { - var serverSessionTokenTimeoutInteger int64 - serverSessionTokenTimeoutInteger, err = strconv.ParseInt(serverSessionTokenTimeoutString, 10, 0) - if err != nil { - return errors.New("server session token timeout is invalid") - } - cfg.ServerSessionTokenTimeout = time.Duration(serverSessionTokenTimeoutInteger) * time.Second + if serverSessionTokenTimeout, err := duration.Parse(l.Reporter.GetWithDefault("server_session_token_timeout", cfg.ServerSessionTokenTimeout.String()), time.Second); err != nil { + return errors.New("server session token timeout is invalid") + } else { + cfg.ServerSessionTokenTimeout = serverSessionTokenTimeout } return nil diff --git a/client/config.go b/client/config.go index 9a094aeecf..50fe101281 100644 --- a/client/config.go +++ b/client/config.go @@ -2,12 +2,12 @@ package client import ( "net/url" - "strconv" "time" "github.com/kelseyhightower/envconfig" "github.com/tidepool-org/platform/config" + "github.com/tidepool-org/platform/duration" "github.com/tidepool-org/platform/errors" ) @@ -27,7 +27,7 @@ type Config struct { UserAgent string `envconfig:"TIDEPOOL_USER_AGENT"` // Timeout specifies the maximum amount of time a request can take. Zero means no timeout. - Timeout time.Duration + Timeout time.Duration `envconfig:"TIDEPOOL_CLIENT_TIMEOUT"` } func NewConfig() *Config { @@ -41,12 +41,10 @@ func (c *Config) Load(loader ConfigLoader) error { func (c *Config) LoadFromConfigReporter(reporter config.Reporter) error { c.Address = reporter.GetWithDefault("address", c.Address) c.UserAgent = reporter.GetWithDefault("user_agent", c.UserAgent) - if timeoutString, err := reporter.Get("timeout"); err == nil { - if timeout, parseErr := strconv.ParseInt(timeoutString, 10, 0); parseErr != nil { - return errors.New("timeout is invalid") - } else { - c.Timeout = time.Duration(timeout) * time.Second - } + if timeout, parseErr := duration.Parse(reporter.GetWithDefault("timeout", c.Timeout.String()), time.Second); parseErr != nil { + return errors.New("timeout is invalid") + } else { + c.Timeout = timeout } return nil } @@ -85,6 +83,11 @@ func NewConfigReporterLoader(reporter config.Reporter) *configReporterLoader { func (l *configReporterLoader) Load(cfg *Config) error { cfg.Address = l.Reporter.GetWithDefault("address", cfg.Address) cfg.UserAgent = l.Reporter.GetWithDefault("user_agent", cfg.UserAgent) + if timeout, parseErr := duration.Parse(l.Reporter.GetWithDefault("timeout", cfg.Timeout.String()), time.Second); parseErr != nil { + return errors.New("timeout is invalid") + } else { + cfg.Timeout = timeout + } return nil } diff --git a/client/config_test.go b/client/config_test.go index 7d1a1a0d6a..a1e918ade5 100644 --- a/client/config_test.go +++ b/client/config_test.go @@ -1,11 +1,15 @@ package client_test import ( + "strconv" + "time" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/tidepool-org/platform/client" configTest "github.com/tidepool-org/platform/config/test" + "github.com/tidepool-org/platform/test" testHttp "github.com/tidepool-org/platform/test/http" ) @@ -21,17 +25,34 @@ var _ = Describe("Config", func() { Expect(cfg).ToNot(BeNil()) Expect(cfg.Address).To(BeEmpty()) Expect(cfg.UserAgent).To(BeEmpty()) + Expect(cfg.Timeout).To(BeZero()) + }) + }) + + Context("NewConfigReporterLoader", func() { + It("returns successfully", func() { + Expect(client.NewConfigReporterLoader(configTest.NewReporter())).ToNot(BeNil()) + }) + }) + + Context("NewEnvconfigLoader", func() { + It("returns successfully", func() { + Expect(client.NewEnvconfigLoader()).ToNot(BeNil()) }) }) Context("with new config", func() { var address string var userAgent string + var timeoutSeconds int + var timeout time.Duration var cfg *client.Config BeforeEach(func() { address = testHttp.NewAddress() userAgent = testHttp.NewUserAgent() + timeoutSeconds = test.RandomIntFromRange(1, 3600) + timeout = time.Duration(timeoutSeconds) * time.Second cfg = client.NewConfig() Expect(cfg).ToNot(BeNil()) }) @@ -44,6 +65,7 @@ var _ = Describe("Config", func() { configReporter = configTest.NewReporter() configReporter.Config["address"] = address configReporter.Config["user_agent"] = userAgent + configReporter.Config["timeout"] = strconv.Itoa(timeoutSeconds) loader = client.NewConfigReporterLoader(configReporter) }) @@ -54,6 +76,7 @@ var _ = Describe("Config", func() { Expect(cfg.Load(loader)).To(Succeed()) Expect(cfg.Address).To(Equal(existingAddress)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) }) It("uses existing user agent if not set", func() { @@ -63,12 +86,72 @@ var _ = Describe("Config", func() { Expect(cfg.Load(loader)).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(existingUserAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) + }) + + It("uses existing timeout if not set", func() { + existingTimeout := time.Duration(test.RandomIntFromRange(1, 3600)) * time.Second + cfg.Timeout = existingTimeout + delete(configReporter.Config, "timeout") + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Address).To(Equal(address)) + Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(existingTimeout)) + }) + + It("interprets a timeout without units as seconds", func() { + configReporter.Config["timeout"] = "1.5" + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(1500 * time.Millisecond)) + }) + + It("interprets a timeout with units as a duration", func() { + configReporter.Config["timeout"] = "1m30s" + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(90 * time.Second)) + }) + + It("returns an error if the timeout is not a number or a duration", func() { + configReporter.Config["timeout"] = "invalid" + Expect(cfg.Load(loader)).To(MatchError("timeout is invalid")) + }) + + It("returns an error if the timeout is empty", func() { + configReporter.Config["timeout"] = "" + Expect(cfg.Load(loader)).To(MatchError("timeout is invalid")) }) It("returns successfully and uses values from config reporter", func() { Expect(cfg.Load(loader)).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) + }) + }) + + Context("Load with envconfig loader", func() { + var loader client.ConfigLoader + + BeforeEach(func() { + loader = client.NewEnvconfigLoader() + }) + + It("returns successfully and uses the timeout from the environment", func() { + GinkgoT().Setenv("TIDEPOOL_CLIENT_TIMEOUT", timeout.String()) + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(timeout)) + }) + + It("ignores the untagged TIMEOUT environment variable", func() { + GinkgoT().Setenv("TIMEOUT", timeout.String()) + Expect(cfg.Load(loader)).To(Succeed()) + Expect(cfg.Timeout).To(BeZero()) + }) + + // Unlike the config reporter loaders, envconfig requires units - see CLIENT-016. + It("returns an error if the timeout has no units", func() { + GinkgoT().Setenv("TIDEPOOL_CLIENT_TIMEOUT", strconv.Itoa(timeoutSeconds)) + Expect(cfg.Load(loader)).To(MatchError(ContainSubstring("assigning TIDEPOOL_CLIENT_TIMEOUT to Timeout"))) }) }) @@ -79,6 +162,7 @@ var _ = Describe("Config", func() { configReporter = configTest.NewReporter() configReporter.Config["address"] = address configReporter.Config["user_agent"] = userAgent + configReporter.Config["timeout"] = strconv.Itoa(timeoutSeconds) }) It("uses existing address if not set", func() { @@ -88,6 +172,7 @@ var _ = Describe("Config", func() { Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) Expect(cfg.Address).To(Equal(existingAddress)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) }) It("uses existing user agent if not set", func() { @@ -97,12 +182,46 @@ var _ = Describe("Config", func() { Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(existingUserAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) + }) + + It("uses existing timeout if not set", func() { + existingTimeout := time.Duration(test.RandomIntFromRange(1, 3600)) * time.Second + cfg.Timeout = existingTimeout + delete(configReporter.Config, "timeout") + Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) + Expect(cfg.Address).To(Equal(address)) + Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(existingTimeout)) + }) + + It("interprets a timeout without units as seconds", func() { + configReporter.Config["timeout"] = "1.5" + Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(1500 * time.Millisecond)) + }) + + It("interprets a timeout with units as a duration", func() { + configReporter.Config["timeout"] = "1m30s" + Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) + Expect(cfg.Timeout).To(Equal(90 * time.Second)) + }) + + It("returns an error if the timeout is not a number or a duration", func() { + configReporter.Config["timeout"] = "invalid" + Expect(cfg.LoadFromConfigReporter(configReporter)).To(MatchError("timeout is invalid")) + }) + + It("returns an error if the timeout is empty", func() { + configReporter.Config["timeout"] = "" + Expect(cfg.LoadFromConfigReporter(configReporter)).To(MatchError("timeout is invalid")) }) It("returns successfully and uses values from config reporter", func() { Expect(cfg.LoadFromConfigReporter(configReporter)).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) }) }) @@ -110,6 +229,7 @@ var _ = Describe("Config", func() { BeforeEach(func() { cfg.Address = address cfg.UserAgent = userAgent + cfg.Timeout = timeout }) Context("Validate", func() { @@ -123,10 +243,21 @@ var _ = Describe("Config", func() { Expect(cfg.Validate()).To(MatchError("address is invalid")) }) + It("returns an error if the timeout is negative", func() { + cfg.Timeout = -timeout + Expect(cfg.Validate()).To(MatchError("timeout is invalid")) + }) + + It("returns success if the timeout is zero", func() { + cfg.Timeout = 0 + Expect(cfg.Validate()).To(Succeed()) + }) + It("returns success", func() { Expect(cfg.Validate()).To(Succeed()) Expect(cfg.Address).To(Equal(address)) Expect(cfg.UserAgent).To(Equal(userAgent)) + Expect(cfg.Timeout).To(Equal(timeout)) }) }) }) diff --git a/duration/duration.go b/duration/duration.go new file mode 100644 index 0000000000..c5ee713bea --- /dev/null +++ b/duration/duration.go @@ -0,0 +1,42 @@ +package duration + +import ( + "math" + "strconv" + "time" + + "github.com/tidepool-org/platform/errors" +) + +// Parse returns the duration represented by specified value. Value may be a standard Go duration string +// or a number string without units, which is then assumed to be in the specified units. +func Parse(value string, units time.Duration) (time.Duration, error) { + if units <= 0 { + return 0, errors.New("units is invalid") + } + + // Attempt to parse as standard Go duration string first. + if valueParsed, err := time.ParseDuration(value); err == nil { + return valueParsed, nil + } + + // Determine minimum and maximum possible values in the specified units. + valueMinimum := math.MinInt64 / int64(units) + valueMaximum := math.MaxInt64 / int64(units) + + // Attempt to parse as an integer then float without units, which is then assumed to be in the specified + // units. Integer values are inclusive of the minimum and maximum, since the multiply below is exact. Float + // values are exclusive, because float64 rounds the minimum and maximum themselves past the true bounds + // whenever either exceeds 2^53, which happens for units finer than 1024ns. + if valueInt, err := strconv.ParseInt(value, 10, 64); err == nil { + if valueInt >= valueMinimum && valueInt <= valueMaximum { + return time.Duration(valueInt * int64(units)), nil + } + } else if valueFloat, err := strconv.ParseFloat(value, 64); err == nil { + if valueFloat > float64(valueMinimum) && valueFloat < float64(valueMaximum) { + return time.Duration(valueFloat * float64(units)), nil + } + } + + return 0, errors.New("unable to parse duration") +} diff --git a/duration/duration_suite_test.go b/duration/duration_suite_test.go new file mode 100644 index 0000000000..02dc2aa0e6 --- /dev/null +++ b/duration/duration_suite_test.go @@ -0,0 +1,11 @@ +package duration_test + +import ( + "testing" + + "github.com/tidepool-org/platform/test" +) + +func TestSuite(t *testing.T) { + test.Test(t) +} diff --git a/duration/duration_test.go b/duration/duration_test.go new file mode 100644 index 0000000000..1a0aaaa735 --- /dev/null +++ b/duration/duration_test.go @@ -0,0 +1,85 @@ +package duration_test + +import ( + "math" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/tidepool-org/platform/duration" + "github.com/tidepool-org/platform/test" +) + +var _ = Describe("Duration", func() { + Context("Parse", func() { + DescribeTable("returns the expected duration when the value", + func(value string, units time.Duration, expectedDuration time.Duration) { + Expect(test.MustDuration(duration.Parse(value, units))).To(Equal(expectedDuration)) + }, + Entry("has a single unit", "45s", time.Second, 45*time.Second), + Entry("has multiple units", "1m30s", time.Second, 90*time.Second), + Entry("has a fractional unit", "1.5h", time.Second, 90*time.Minute), + Entry("has units, which override the units argument", "45s", time.Hour, 45*time.Second), + Entry("has units and is negative", "-1m", time.Second, -time.Minute), + Entry("has no units, using the units argument", "45", time.Second, 45*time.Second), + Entry("has no units, using non-second units", "45", time.Millisecond, 45*time.Millisecond), + Entry("has no units and is fractional", "1.5", time.Second, 1500*time.Millisecond), + Entry("has no units and is fractional, using non-second units", "1.5", time.Hour, 90*time.Minute), + Entry("has no units and is in exponent notation", "1e3", time.Second, 1000*time.Second), + Entry("has no units and is negative", "-3", time.Second, -3*time.Second), + Entry("has no units and is zero", "0", time.Second, time.Duration(0)), + Entry("has no units and is finer than a nanosecond", "0.0000000001", time.Second, time.Duration(0)), + // The integer bounds are inclusive and the multiply is exact, so the extremes convert without loss. + Entry("has no units and is the largest duration", "9223372036854775807", time.Nanosecond, time.Duration(math.MaxInt64)), + Entry("has no units and is the smallest duration", "-9223372036854775808", time.Nanosecond, time.Duration(math.MinInt64)), + Entry("has no units and is the largest whole number of units", "9223372036", time.Second, 9223372036*time.Second), + Entry("has no units and is the smallest whole number of units", "-9223372036", time.Second, -9223372036*time.Second), + Entry("has no units and is an integer beyond float64 precision", "9007199254740993", time.Nanosecond, time.Duration(9007199254740993)), + // The float bounds are exclusive, and float64 carries only 53 bits, so these two do lose the low bits. + Entry("has no units, is fractional, and is the largest float64 below the largest duration", "9223372036854774784.0", time.Nanosecond, time.Duration(9223372036854774784)), + Entry("has no units, is fractional, and is beyond float64 precision", "9223372035.5", time.Second, time.Duration(9223372035500000256)), + ) + + DescribeTable("returns an error when the value", + func(value string, units time.Duration) { + parsedDuration, err := duration.Parse(value, units) + Expect(err).To(MatchError("unable to parse duration")) + Expect(parsedDuration).To(BeZero()) + }, + Entry("is empty", "", time.Second), + Entry("is not a number or a duration", "invalid", time.Second), + Entry("has a leading space", " 45", time.Second), + Entry("has a trailing space", "45 ", time.Second), + Entry("has an unrecognized unit", "45y", time.Second), + Entry("has an upper case unit", "45S", time.Second), + Entry("has trailing characters after the units", "45sx", time.Second), + Entry("is hexadecimal", "0x10", time.Second), + Entry("is not a number", "NaN", time.Second), + Entry("is not a number in lower case", "nan", time.Second), + Entry("is positive infinity", "Inf", time.Second), + Entry("is negative infinity", "-Inf", time.Second), + Entry("is infinity spelled out", "infinity", time.Second), + Entry("overflows the units in exponent notation", "1e30", time.Second), + Entry("overflows the units", "9223372036854775807", time.Second), + Entry("is one past the largest whole number of units", "9223372037", time.Second), + Entry("is one past the smallest whole number of units", "-9223372037", time.Second), + Entry("is one past the largest duration", "9223372036854775808", time.Nanosecond), + // The float bounds are exclusive, so a fraction above the largest whole number of units is given up. + Entry("is fractional and above the largest whole number of units", "9223372036.5", time.Second), + ) + + DescribeTable("returns an error when the units", + func(value string, units time.Duration) { + parsedDuration, err := duration.Parse(value, units) + Expect(err).To(MatchError("units is invalid")) + Expect(parsedDuration).To(BeZero()) + }, + Entry("is zero", "45", time.Duration(0)), + Entry("is negative", "45", -time.Second), + Entry("is the smallest duration", "45", time.Duration(math.MinInt64)), + Entry("is zero and the value has units", "45s", time.Duration(0)), + Entry("is zero and the value is not parsable", "invalid", time.Duration(0)), + ) + }) +}) diff --git a/service/server/config.go b/service/server/config.go index f050e76a97..2adb673e80 100644 --- a/service/server/config.go +++ b/service/server/config.go @@ -6,6 +6,7 @@ import ( "time" "github.com/tidepool-org/platform/config" + "github.com/tidepool-org/platform/duration" "github.com/tidepool-org/platform/errors" ) @@ -40,13 +41,10 @@ func (c *Config) Load(configReporter config.Reporter) error { } c.TLSCertificateFile = configReporter.GetWithDefault("tls_certificate_file", "") c.TLSKeyFile = configReporter.GetWithDefault("tls_key_file", "") - if timeoutString, err := configReporter.Get("timeout"); err == nil { - var timeout int64 - timeout, err = strconv.ParseInt(timeoutString, 10, 0) - if err != nil { - return errors.New("timeout is invalid") - } - c.Timeout = time.Duration(timeout) * time.Second + if timeout, err := duration.Parse(configReporter.GetWithDefault("timeout", c.Timeout.String()), time.Second); err != nil { + return errors.New("timeout is invalid") + } else { + c.Timeout = timeout } return nil diff --git a/task/queue/queue.go b/task/queue/queue.go index e05803aea0..d8e2d9f509 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -12,6 +12,7 @@ import ( "github.com/tidepool-org/platform/config" "github.com/tidepool-org/platform/crypto" + "github.com/tidepool-org/platform/duration" "github.com/tidepool-org/platform/errors" "github.com/tidepool-org/platform/log" "github.com/tidepool-org/platform/pointer" @@ -98,54 +99,40 @@ func (c *Config) Load(configReporter config.Reporter) error { c.Workers = int(value) } } - if valueString, err := configReporter.Get("start_manager_delay"); err == nil { - if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { - return errors.New("start manager delay is invalid") - } else { - c.StartManagerDelay = time.Duration(value) * time.Second - } + if value, err := duration.Parse(configReporter.GetWithDefault("start_manager_delay", c.StartManagerDelay.String()), time.Second); err != nil { + return errors.New("start manager delay is invalid") + } else { + c.StartManagerDelay = value } - if valueString, err := configReporter.Get("dispatch_tasks_delay"); err == nil { - if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { - return errors.New("dispatch tasks delay is invalid") - } else { - c.DispatchTasksDelay = time.Duration(value) * time.Second - } + if value, err := duration.Parse(configReporter.GetWithDefault("dispatch_tasks_delay", c.DispatchTasksDelay.String()), time.Second); err != nil { + return errors.New("dispatch tasks delay is invalid") + } else { + c.DispatchTasksDelay = value } - if valueString, err := configReporter.Get("monitor_task_delay"); err == nil { - if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { - return errors.New("monitor task delay is invalid") - } else { - c.MonitorTaskDelay = time.Duration(value) * time.Second - } + if value, err := duration.Parse(configReporter.GetWithDefault("monitor_task_delay", c.MonitorTaskDelay.String()), time.Second); err != nil { + return errors.New("monitor task delay is invalid") + } else { + c.MonitorTaskDelay = value } - if valueString, err := configReporter.Get("runner_watchdog_grace_period"); err == nil { - if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { - return errors.New("runner watchdog grace period is invalid") - } else { - c.RunnerWatchdogGracePeriod = time.Duration(value) * time.Second - } + if value, err := duration.Parse(configReporter.GetWithDefault("runner_watchdog_grace_period", c.RunnerWatchdogGracePeriod.String()), time.Second); err != nil { + return errors.New("runner watchdog grace period is invalid") + } else { + c.RunnerWatchdogGracePeriod = value } - if valueString, err := configReporter.Get("unstick_tasks_delay"); err == nil { - if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { - return errors.New("unstick tasks delay is invalid") - } else { - c.UnstickTasksDelay = time.Duration(value) * time.Second - } + if value, err := duration.Parse(configReporter.GetWithDefault("unstick_tasks_delay", c.UnstickTasksDelay.String()), time.Second); err != nil { + return errors.New("unstick tasks delay is invalid") + } else { + c.UnstickTasksDelay = value } - if valueString, err := configReporter.Get("unstick_tasks_available_grace_period"); err == nil { - if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { - return errors.New("unstick tasks available grace period is invalid") - } else { - c.UnstickTasksAvailableGracePeriod = time.Duration(value) * time.Second - } + if value, err := duration.Parse(configReporter.GetWithDefault("unstick_tasks_available_grace_period", c.UnstickTasksAvailableGracePeriod.String()), time.Second); err != nil { + return errors.New("unstick tasks available grace period is invalid") + } else { + c.UnstickTasksAvailableGracePeriod = value } - if valueString, err := configReporter.Get("stop_wait_timeout"); err == nil { - if value, parseErr := strconv.ParseInt(valueString, 10, 0); parseErr != nil { - return errors.New("stop wait timeout is invalid") - } else { - c.StopWaitTimeout = time.Duration(value) * time.Second - } + if value, err := duration.Parse(configReporter.GetWithDefault("stop_wait_timeout", c.StopWaitTimeout.String()), time.Second); err != nil { + return errors.New("stop wait timeout is invalid") + } else { + c.StopWaitTimeout = value } return nil From 355b240bdbcdde5b3dc1630f40baa69e747ac385 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Mon, 10 Aug 2026 20:44:25 -0700 Subject: [PATCH 20/20] Do not update data source or task if task claim was lost --- auth/service/service/client.go | 4 +- dexcom/fetch/runner.go | 19 ++++--- dexcom/fetch/runner_test.go | 92 +++++++++++++++++++++++++++++++++- task/queue/queue.go | 10 ++-- task/queue/runner.go | 9 +--- task/task.go | 5 ++ 6 files changed, 115 insertions(+), 24 deletions(-) diff --git a/auth/service/service/client.go b/auth/service/service/client.go index 074b81483e..a8bca5e9a8 100644 --- a/auth/service/service/client.go +++ b/auth/service/service/client.go @@ -88,8 +88,8 @@ func (c *Client) CreateProviderSession(ctx context.Context, create *auth.Provide if err = prvdr.OnCreate(ctx, providerSession); err != nil { log.LoggerFromContext(ctx).WithError(err).Error("Unable to finalize creation of provider session") - if err := c.deleteProviderSession(ctx, repository, providerSession); err != nil { - log.LoggerFromContext(ctx).WithError(err).Warn("Unable to delete provider session") + if deleteErr := c.deleteProviderSession(ctx, repository, providerSession); deleteErr != nil { + log.LoggerFromContext(ctx).WithError(deleteErr).Warn("Unable to delete provider session") } return nil, err } diff --git a/dexcom/fetch/runner.go b/dexcom/fetch/runner.go index be6f84a2ef..63175915b5 100644 --- a/dexcom/fetch/runner.go +++ b/dexcom/fetch/runner.go @@ -180,14 +180,17 @@ func (t *TaskRunner) Run(ctx context.Context) { t.task.AppendError(err) } - // A permanently failed task is not rescheduled, unless its outcome could not be recorded on the data source, in - // which case run again so a later run can record it - err := t.updateDataSourceWithTaskState() - if err != nil { - t.task.AppendError(err) - } - if err != nil || !t.task.IsFailed() { - t.task.RepeatAvailableAfter(pointer.Default(t.availableAfter, availableAfterDuration())) + // If we didn't lose the claim, then update data source and repeat if not failed + if !errors.Is(context.Cause(t.context), task.ErrClaimLost) { + err := t.updateDataSourceWithTaskState() + if err != nil { + t.task.AppendError(err) + } + if err != nil || !t.task.IsFailed() { + t.task.RepeatAvailableAfter(pointer.Default(t.availableAfter, availableAfterDuration())) + } + } else { + t.logger.Warn("Skipped updating data source and task because the task claim was lost") } } diff --git a/dexcom/fetch/runner_test.go b/dexcom/fetch/runner_test.go index 8676303678..f9a170347b 100644 --- a/dexcom/fetch/runner_test.go +++ b/dexcom/fetch/runner_test.go @@ -187,6 +187,7 @@ var _ = Describe("Runner", func() { Context("with task runner and context", func() { var taskRunner *dexcomFetch.TaskRunner + var logger *logTest.Logger var ctx context.Context BeforeEach(func() { @@ -194,7 +195,8 @@ var _ = Describe("Runner", func() { taskRunner, err = dexcomFetch.NewTaskRunner(provider, tsk) Expect(err).ToNot(HaveOccurred()) Expect(taskRunner).ToNot(BeNil()) - ctx = log.NewContextWithLogger(context.Background(), logTest.NewLogger()) + logger = logTest.NewLogger() + ctx = log.NewContextWithLogger(context.Background(), logger) }) assertTaskState := func(state string) { @@ -364,6 +366,21 @@ var _ = Describe("Runner", func() { assertTaskAndDataSourceError(dexcomFetch.ErrorCodeResourceFailure, "unable to get provider session") }) + It("discards the run outcome if the task claim is lost", func() { + claimContext, claimCancel := context.WithCancelCause(ctx) + defer claimCancel(nil) + testErr := errorsTest.RandomError() + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").DoAndReturn(func(ctx context.Context, id string) (*auth.ProviderSession, error) { + claimCancel(task.ErrClaimLost) + return nil, testErr + }) + taskRunner.Run(claimContext) + assertTaskState(task.TaskStateRunning) + Expect(dataSrc.State).To(Equal(dataSource.StateConnected)) + Expect(dataSrc.HasError()).To(BeFalse()) + logger.AssertWarn("Skipped updating data source and task because the task claim was lost") + }) + It("fails if the provider session is missing", func() { authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(nil, nil) dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) @@ -643,7 +660,78 @@ var _ = Describe("Runner", func() { // deviceHashes - not in data // dataSource.LatestDataTime - not nil (recent) // refresh token - // data ranges multiple 30 day segments + }) + + Context("with provider session and a data range spanning multiple chunks", func() { + var providerSession *auth.ProviderSession + var firstChunkStartTime time.Time + var firstChunkEndTime time.Time + var secondChunkEndTime time.Time + + BeforeEach(func() { + providerSession = &auth.ProviderSession{ + ID: "test-provider-session-id", + UserID: "test-user-id", + OAuthToken: &auth.OAuthToken{ + AccessToken: "test-access-token-1", + TokenType: "Bearer", + RefreshToken: "test-refresh-token-1", + ExpirationTime: time.Now().Add(time.Minute), + }, + } + authClient.EXPECT().GetProviderSession(matchContext(), "test-provider-session-id").Return(providerSession, nil) + authClient.EXPECT().UpdateProviderSession(matchContext(), "test-provider-session-id", matchNotNil()).DoAndReturn(mockAuthClientUpdateProviderSession(providerSession)).AnyTimes() + firstChunkStartTime = time.Now().Add(-45 * Day) + firstChunkEndTime = firstChunkStartTime.AddDate(0, 0, dexcomFetch.DataRangeDaysMaximum) + secondChunkEndTime = time.Now().Add(-3 * Day) + dataRangeResponse := &dexcom.DataRangesResponse{ + Calibrations: &dexcom.DataRange{ + Start: &dexcom.Moment{SystemTime: &dexcom.Time{Time: firstChunkStartTime}}, + End: &dexcom.Moment{SystemTime: &dexcom.Time{Time: secondChunkEndTime}}, + }, + } + dexcomClient.EXPECT().GetDataRange(matchContext(), nil, matchNotNil()).DoAndReturn(mockDexcomClientGetDataRange(nil, dataRangeResponse, nil)) + }) + + // Expects the fetch of a single chunk, all responses empty, invoking onEvents, if any, during the + // final fetch of the chunk + expectFetchChunk := func(startTime time.Time, endTime time.Time, onEvents func()) { + dexcomClient.EXPECT().GetAlerts(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, &dexcom.AlertsResponse{Records: &dexcom.Alerts{}}, nil)) + dexcomClient.EXPECT().GetCalibrations(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, &dexcom.CalibrationsResponse{Records: &dexcom.Calibrations{}}, nil)) + dexcomClient.EXPECT().GetDevices(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, &dexcom.DevicesResponse{Records: &dexcom.Devices{}}, nil)) + dexcomClient.EXPECT().GetEGVs(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData(nil, &dexcom.EGVsResponse{Records: &dexcom.EGVs{}}, nil)) + dexcomClient.EXPECT().GetEvents(matchContext(), startTime, endTime, matchNotNil()).DoAndReturn(func(ctx context.Context, startTime time.Time, endTime time.Time, tokenSource oauth.TokenSource) (*dexcom.EventsResponse, error) { + if onEvents != nil { + onEvents() + } + return &dexcom.EventsResponse{Records: &dexcom.Events{}}, nil + }) + } + + It("fetches every chunk of the data range", func() { + expectFetchChunk(firstChunkStartTime, firstChunkEndTime, nil) + expectFetchChunk(firstChunkEndTime, secondChunkEndTime, nil) + dataSourceClient.EXPECT().Update(matchContext(), "test-data-source-id", matchNil(), matchNotNil()).DoAndReturn(mockDataSourceClientUpdate(dataSrc)) + taskRunner.Run(ctx) + assertTaskAndDataSourceState(task.TaskStatePending) + assertTaskAvailableAfterStandardDuration() + assertTaskRetryCountNotPresent() + assertTaskAndDataSourceErrorNotPresent() + assertDataSourceLastImportTimePresent() + }) + + It("discards the run outcome if the task claim is lost mid-fetch", func() { + claimContext, claimCancel := context.WithCancelCause(ctx) + defer claimCancel(nil) + expectFetchChunk(firstChunkStartTime, firstChunkEndTime, func() { claimCancel(task.ErrClaimLost) }) + // The canceled context fails the next chunk, ending the run + dexcomClient.EXPECT().GetAlerts(matchContext(), firstChunkEndTime, secondChunkEndTime, matchNotNil()).DoAndReturn(mockDexcomClientGetData[dexcom.AlertsResponse](nil, nil, context.Canceled)) + taskRunner.Run(claimContext) + assertTaskState(task.TaskStateRunning) + Expect(dataSrc.State).To(Equal(dataSource.StateConnected)) + Expect(dataSrc.HasError()).To(BeFalse()) + logger.AssertWarn("Skipped updating data source and task because the task claim was lost") + }) }) }) }) diff --git a/task/queue/queue.go b/task/queue/queue.go index d8e2d9f509..a78d3a686f 100644 --- a/task/queue/queue.go +++ b/task/queue/queue.go @@ -34,7 +34,7 @@ const ( DispatchTasksDelayDefault = 1 * time.Minute // MonitorTaskDelayDefault is the default interval between checks that each in-flight task's claim is still held - // (the task exists and its claim token is unchanged); a run whose claim is lost is canceled with ErrClaimLost. + // (the task exists and its claim token is unchanged); a run whose claim is lost is canceled with task.ErrClaimLost. MonitorTaskDelayDefault = 1 * time.Minute // RunnerWatchdogGracePeriodDefault is the extra time beyond the runner timeout that the watchdog waits before @@ -363,7 +363,7 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { return } - // The claim context is canceled with ErrClaimLost by the task claim monitor when the task is deleted or re-claimed + // The claim context is canceled with task.ErrClaimLost by the task claim monitor when the task is deleted or re-claimed // mid-run. Claim loss is irreversible for this run (every claim gets a fresh token), so once canceled the outcome // can never be persisted. claimContext, claimCancel := context.WithCancelCause(ctx) @@ -372,7 +372,7 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { // Clearing the claim token marks the outcome as unpersistable, which completeTask discards. Done in a defer, after // the recover below, so a panicking runner is also reconciled correctly. defer func() { - if errors.Is(context.Cause(claimContext), ErrClaimLost) { + if errors.Is(context.Cause(claimContext), task.ErrClaimLost) { tsk.ClaimToken = nil } }() @@ -408,7 +408,7 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { if reason := q.monitorTaskForLostClaim(claimContext, id, claimToken); reason != nil { log.LoggerFromContext(claimContext).Warnf("Task %s; canceling task run", *reason) RunClaimLostTotal.WithLabelValues(typ, *reason).Inc() - claimCancel(ErrClaimLost) + claimCancel(task.ErrClaimLost) } }(tsk.ID, tsk.Type, *tsk.ClaimToken) @@ -427,7 +427,7 @@ func (q *Queue) runTask(ctx context.Context, tsk *task.Task) { // The claim was lost mid-run; any write-back would miss the claim token, so skip state reconciliation. The outcome // is discarded during completion. - if errors.Is(context.Cause(claimContext), ErrClaimLost) { + if errors.Is(context.Cause(claimContext), task.ErrClaimLost) { return } diff --git a/task/queue/runner.go b/task/queue/runner.go index 84d58fd5bf..f3d4e1009e 100644 --- a/task/queue/runner.go +++ b/task/queue/runner.go @@ -43,8 +43,8 @@ import ( // Context: canceled (with cause) after GetRunnerTimeout, on shutdown, and on claim loss - cooperative runners select on // ctx.Done() and return promptly. To tell them apart, inspect context.Cause(ctx): a normal shutdown with // context.Canceled; a runner timeout with ErrRunnerTimeoutExceeded, and a lost claim (the task was deleted mid-run, or -// unstuck and its claim token replaced) with ErrClaimLost (use errors.Is in all cases). After a claim loss the run's -// outcome is discarded - no write-back can land - so return promptly and skip any remaining work. Claim loss is +// unstuck and its claim token replaced) with task.ErrClaimLost (use errors.Is in all cases). After a claim loss the +// run's outcome is discarded - no write-back can land - so return promptly and skip any remaining work. Claim loss is // detected by a periodic check (MonitorTaskDelay, 1 minute by default), not instantly. The queue cannot preempt a // runner that ignores cancellation; it stays blocked until it returns or the process restarts, recovered by the // deadline/unstick mechanism (a periodic sweep that resets tasks still running past their deadline - see @@ -109,8 +109,3 @@ type Runner interface { // runner distinguishes a timeout from a shutdown with errors.Is(context.Cause(ctx), ErrRunnerTimeoutExceeded); a // shutdown instead cancels with context.Canceled. var ErrRunnerTimeoutExceeded = errors.New("task runner timeout exceeded") - -// ErrClaimLost is the cancellation cause set on the Run context when the run's claim on the task was lost mid-run: the -// task was deleted, or it was unstuck and possibly re-claimed (its claim token no longer matches). A runner -// distinguishes it with errors.Is(context.Cause(ctx), ErrClaimLost). -var ErrClaimLost = errors.New("task claim lost") diff --git a/task/task.go b/task/task.go index 8783b84c02..be3fe458aa 100644 --- a/task/task.go +++ b/task/task.go @@ -348,3 +348,8 @@ func (t Tasks) Sanitize(details request.AuthDetails) error { } return nil } + +// ErrClaimLost is the cancellation cause set on the Run context when the run's claim on the task was lost mid-run: the +// task was deleted, or it was unstuck and possibly re-claimed (its claim token no longer matches). A runner +// distinguishes it with errors.Is(context.Cause(ctx), ErrClaimLost). +var ErrClaimLost = errors.New("task claim lost")