From 7395169508a7ae8121df4cbae5def2270f14f999 Mon Sep 17 00:00:00 2001 From: Tsering Paljor Date: Wed, 1 Jul 2026 16:28:22 +0700 Subject: [PATCH 1/2] feat: Add from_json() and to_json() expressions (#5557) * Add from_json() and to_json() expressions. * Fix rebase mistake * Refactor from/to json runtime formula types to re-use existing json ensurer. * Add frontend counterpart for ensure json * Rename regex var * Simplify formula check. * Improve formula function detection when JSON load fails --- .../integrations/core/service_types.py | 36 +++++- backend/src/baserow/core/apps.py | 4 + .../core/formula/runtime_formula_types.py | 40 ++++++- backend/src/baserow/core/formula/validator.py | 14 ++- .../test_core_http_request_service_type.py | 105 +++++++++++++++++- .../formula/test_runtime_formula_types.py | 95 ++++++++++++++++ .../baserow/core/formula/test_validator.py | 15 +++ ...add_to_json_and_from_json_expressions.json | 9 ++ web-frontend/locales/en.json | 2 + web-frontend/modules/core/plugin.js | 6 +- .../modules/core/runtimeFormulaTypes.js | 82 ++++++++++++++ web-frontend/modules/core/utils/validator.js | 53 +++++++++ .../core/formula/runtimeFormulaTypes.spec.js | 69 ++++++++++++ 13 files changed, 521 insertions(+), 9 deletions(-) create mode 100644 changelog/entries/unreleased/feature/3879_add_to_json_and_from_json_expressions.json diff --git a/backend/src/baserow/contrib/integrations/core/service_types.py b/backend/src/baserow/contrib/integrations/core/service_types.py index 9b68300e59..2925c720b1 100644 --- a/backend/src/baserow/contrib/integrations/core/service_types.py +++ b/backend/src/baserow/contrib/integrations/core/service_types.py @@ -1,6 +1,7 @@ import csv import io import json +import re import socket import uuid from datetime import datetime @@ -57,6 +58,7 @@ ) from baserow.contrib.integrations.core.utils import calculate_next_periodic_run from baserow.contrib.integrations.utils import get_http_request_function +from baserow.core.formula.registries import formula_runtime_function_registry from baserow.core.formula.types import BaserowFormulaObject from baserow.core.formula.validator import ( ensure_array, @@ -83,6 +85,12 @@ from baserow.core.services.types import DispatchResult, FormulaToResolve, ServiceDict from baserow.version import VERSION as BASEROW_VERSION +# Captures potential runtime formula function calls, e.g. `get(`, `concat(`, etc. +# The captured name is checked against the runtime function registry so that +# literal text like `foo(` (which isn't a real function) doesn't match. Function +# names are case-insensitive, so the name is lowercased before the lookup. +RE_FORMULA_FUNCTION = re.compile(r"\b([a-zA-Z_]+)\s*\(") + class CoreServiceType(ServiceType): """ @@ -531,6 +539,23 @@ def formulas_to_resolve( return formulas + def _body_has_formula(self, service: CoreHTTPRequestService) -> bool: + """ + Returns True if the JSON body is built from a formula that interpolates a + runtime value, e.g. a data source via `get(...)` or a function like `now()`. + + This is used to customize the JSON validation error with a more specific + error message. + """ + + formula = (service.body_content or {}).get("formula") or "" + + registered_functions = set(formula_runtime_function_registry.get_types()) + return any( + match.group(1).lower() in registered_functions + for match in RE_FORMULA_FUNCTION.finditer(formula) + ) + def dispatch_data( self, service: CoreHTTPRequestService, @@ -554,9 +579,14 @@ def dispatch_data( json.loads(body_content, strict=False) if body_content else None ) except json.JSONDecodeError as e: - raise ServiceImproperlyConfiguredDispatchException( - "The body is not a valid JSON" - ) from e + message = "The body is not a valid JSON" + if self._body_has_formula(service): + message += ( + ". If the body includes a value from a formula, wrap " + "it with to_json() so it is correctly quoted and " + "escaped, e.g. concat('{\"value\": ', to_json(get('...')), '}')." + ) + raise ServiceImproperlyConfiguredDispatchException(message) from e elif service.body_type == BODY_TYPE.FORM: # Form multipart payload body_dict["data"] = { f.key: resolved_values[f"form_data_{f.id}"] diff --git a/backend/src/baserow/core/apps.py b/backend/src/baserow/core/apps.py index 74728168e8..005904d5d9 100755 --- a/backend/src/baserow/core/apps.py +++ b/backend/src/baserow/core/apps.py @@ -49,6 +49,7 @@ def ready(self): RuntimeDivide, RuntimeDurationFormat, RuntimeEqual, + RuntimeFromJson, RuntimeGenerateUUID, RuntimeGet, RuntimeGetProperty, @@ -88,6 +89,7 @@ def ready(self): RuntimeToDatetime, RuntimeToday, RuntimeToDuration, + RuntimeToJson, RuntimeUpper, RuntimeYear, ) @@ -141,6 +143,8 @@ def ready(self): formula_runtime_function_registry.register(RuntimeAt()) formula_runtime_function_registry.register(RuntimeToArray()) formula_runtime_function_registry.register(RuntimeRange()) + formula_runtime_function_registry.register(RuntimeToJson()) + formula_runtime_function_registry.register(RuntimeFromJson()) formula_runtime_function_registry.register(RuntimeNull()) formula_runtime_function_registry.register(RuntimeNumberFormat()) formula_runtime_function_registry.register(RuntimeToDuration()) diff --git a/backend/src/baserow/core/formula/runtime_formula_types.py b/backend/src/baserow/core/formula/runtime_formula_types.py index 64fb04a687..0282aefd9c 100644 --- a/backend/src/baserow/core/formula/runtime_formula_types.py +++ b/backend/src/baserow/core/formula/runtime_formula_types.py @@ -1,3 +1,4 @@ +import json import random import uuid from datetime import datetime, timedelta @@ -33,7 +34,13 @@ from baserow.core.formula.registries import RuntimeFormulaFunction from baserow.core.formula.types import FormulaArg, FormulaArgs, FormulaContext from baserow.core.formula.utils.date import convert_date_format_moment_to_python -from baserow.core.formula.validator import ensure_array, ensure_datetime, ensure_string +from baserow.core.formula.validator import ( + ensure_array, + ensure_datetime, + ensure_deserialized_json, + ensure_json_serializable, + ensure_string, +) from baserow.core.utils import to_path @@ -784,6 +791,37 @@ def execute(self, context: FormulaContext, args: FormulaArgs): return list(result) +class RuntimeToJson(RuntimeFormulaFunction): + type = "to_json" + + args = [AnyBaserowRuntimeFormulaArgumentType()] + + def execute(self, context: FormulaContext, args: FormulaArgs): + try: + serializable = ensure_json_serializable(args[0]) + except ValidationError as exc: + raise BaserowFormulaSyntaxError( + "The 'to_json' function received a value that cannot be " + "converted to JSON." + ) from exc + # Compact separators are used to match the frontend's `JSON.stringify` output. + return json.dumps(serializable, separators=(",", ":")) + + +class RuntimeFromJson(RuntimeFormulaFunction): + type = "from_json" + + args = [TextBaserowRuntimeFormulaArgumentType()] + + def execute(self, context: FormulaContext, args: FormulaArgs): + # Parse a JSON-encoded string back into a value. Returns `None` when the + # input isn't valid JSON. + try: + return ensure_deserialized_json(args[0], strict=True) + except ValidationError: + return None + + class RuntimeNull(RuntimeFormulaFunction): type = "null" diff --git a/backend/src/baserow/core/formula/validator.py b/backend/src/baserow/core/formula/validator.py index b099af8397..5d3a4cf87f 100644 --- a/backend/src/baserow/core/formula/validator.py +++ b/backend/src/baserow/core/formula/validator.py @@ -324,20 +324,28 @@ def default(self, obj): return super().default(obj) -def ensure_deserialized_json(value: Any) -> Any: +def ensure_deserialized_json(value: Any, strict: bool = False) -> Any: """ Decode a JSON string if possible, otherwise return the value unchanged. + Values that are not strings are already deserialized and are returned as-is, + even when `strict` is True. + :param value: The value to decode if it is a valid JSON string. + :param strict: If True, raise a ValidationError when `value` is a string that + cannot be decoded as JSON, instead of returning it unchanged. :return: The decoded JSON value if `value` is a valid JSON string, otherwise `value`. + :raises ValidationError: If `strict` is True and `value` is a string that is + not valid JSON. """ if isinstance(value, str): try: return json.loads(value) - except (TypeError, JSONDecodeError): - pass + except (TypeError, JSONDecodeError) as exc: + if strict: + raise ValidationError("Value is not valid JSON.") from exc return value diff --git a/backend/tests/baserow/contrib/integrations/core/test_core_http_request_service_type.py b/backend/tests/baserow/contrib/integrations/core/test_core_http_request_service_type.py index ffd038b562..7405374a1c 100644 --- a/backend/tests/baserow/contrib/integrations/core/test_core_http_request_service_type.py +++ b/backend/tests/baserow/contrib/integrations/core/test_core_http_request_service_type.py @@ -7,7 +7,10 @@ from baserow.contrib.integrations.core.models import BODY_TYPE, HTTP_METHOD from baserow.contrib.integrations.core.service_types import CoreHTTPRequestServiceType -from baserow.core.services.exceptions import UnexpectedDispatchException +from baserow.core.services.exceptions import ( + ServiceImproperlyConfiguredDispatchException, + UnexpectedDispatchException, +) from baserow.core.services.handler import ServiceHandler from baserow.test_utils.helpers import AnyInt, AnyStr from baserow.test_utils.pytest_conftest import FakeDispatchContext @@ -197,6 +200,106 @@ def test_core_http_request_basic_body_json( ) +@pytest.mark.django_db +def test_core_http_request_body_json_invalid_static_body(data_fixture): + """ + A static body that isn't valid JSON should fail with the plain error + message, without the extra hint. + """ + + service = data_fixture.create_core_http_request_service( + url="'http://example.notexist/'", + # `{"value1": }` is missing a value, so it's invalid JSON. + body_content="""'{"value1": }'""", + body_type=BODY_TYPE.JSON, + ) + service_type = service.get_type() + + with pytest.raises(ServiceImproperlyConfiguredDispatchException) as exc: + service_type.dispatch(service, FakeDispatchContext()) + + assert str(exc.value) == "The body is not a valid JSON" + + +@pytest.mark.django_db +def test_core_http_request_body_json_invalid_with_data_provider_hint(data_fixture): + """ + When the body interpolates a formula value without `to_json(...)` and + the result isn't valid JSON, the error should hint at wrapping the value. + """ + + service = data_fixture.create_core_http_request_service( + url="'http://example.notexist/'", + # The value isn't wrapped in quotes/`to_json`, so a string value + # produces invalid JSON, e.g. `{"value1": hello}`. + body_content="""concat('{"value1": ', get('page_parameter.id'), '}')""", + body_type=BODY_TYPE.JSON, + ) + service_type = service.get_type() + dispatch_context = FakeDispatchContext(context={"page_parameter": {"id": "hello"}}) + + with pytest.raises(ServiceImproperlyConfiguredDispatchException) as exc: + service_type.dispatch(service, dispatch_context) + + assert "The body is not a valid JSON" in str(exc.value) + assert "to_json(" in str(exc.value) + + +@pytest.mark.django_db +def test_core_http_request_body_json_invalid_with_formula_function_hint(data_fixture): + """ + The hint should also be shown for formula functions as well (e.g. `now()`). + """ + + service = data_fixture.create_core_http_request_service( + url="'http://example.notexist/'", + # `now()` produces an unquoted value, e.g. `{"id": 2026-06-22 03:52:07.130000+00:00}`. + body_content="""concat('{"id": ', now(), '}')""", + body_type=BODY_TYPE.JSON, + ) + service_type = service.get_type() + + with pytest.raises(ServiceImproperlyConfiguredDispatchException) as exc: + service_type.dispatch(service, FakeDispatchContext()) + + assert "The body is not a valid JSON" in str(exc.value) + assert "wrap it with to_json()" in str(exc.value) + + +@pytest.mark.django_db +def test_core_http_request_body_json_with_to_json_escapes_data_source(data_fixture): + """ + Wrapping a data source value with `to_json(...)` produces a valid JSON body + even when the value contains characters that would otherwise break it. + """ + + service = data_fixture.create_core_http_request_service( + url="'http://example.notexist/'", + body_content=( + """concat('{"value1": ', to_json(get('page_parameter.id')), '}')""" + ), + body_type=BODY_TYPE.JSON, + ) + service_type = service.get_type() + dispatch_context = FakeDispatchContext( + context={"page_parameter": {"id": 'foo "bar"'}} + ) + + with mock_advocate_request({"foo": "bar"}) as mock_request: + service_type.dispatch(service, dispatch_context) + + mock_request.assert_called_once_with( + **{ + "headers": {"user-agent": AnyStr()}, + "json": {"value1": 'foo "bar"'}, + "method": HTTP_METHOD.GET, + "params": {}, + "timeout": 30, + "url": "http://example.notexist/", + } + ) + + @pytest.mark.django_db @pytest.mark.parametrize("control_char", ["\n", "\t", "\r"]) def test_core_http_request_basic_body_json_with_control_characters( diff --git a/backend/tests/baserow/core/formula/test_runtime_formula_types.py b/backend/tests/baserow/core/formula/test_runtime_formula_types.py index 04210f00dd..3b45e1d2fb 100644 --- a/backend/tests/baserow/core/formula/test_runtime_formula_types.py +++ b/backend/tests/baserow/core/formula/test_runtime_formula_types.py @@ -20,6 +20,7 @@ RuntimeDivide, RuntimeDurationFormat, RuntimeEqual, + RuntimeFromJson, RuntimeGenerateUUID, RuntimeGet, RuntimeGetProperty, @@ -59,6 +60,7 @@ RuntimeToDatetime, RuntimeToday, RuntimeToDuration, + RuntimeToJson, RuntimeUpper, RuntimeYear, ) @@ -2510,6 +2512,99 @@ def test_runtime_range_validate_number_of_args(args, expected): assert result is expected +@pytest.mark.parametrize( + "args,expected", + [ + (["foo"], '"foo"'), + ([123], "123"), + ([True], "true"), + ([None], "null"), + ([{"a": 1}], '{"a":1}'), + ([[1, 2]], "[1,2]"), + # Quotes, backslashes and control characters are escaped so the result + # is safe to embed inside a larger JSON document. + ([r'foo "bar"'], r'"foo \"bar\""'), + (["a\nb"], r'"a\nb"'), + ], +) +def test_runtime_to_json_execute(args, expected): + parsed_args = RuntimeToJson().parse_args(args) + result = RuntimeToJson().execute({}, parsed_args) + assert result == expected + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), object()]) +def test_runtime_to_json_raises_for_non_serializable_value(value): + parsed_args = RuntimeToJson().parse_args([value]) + with pytest.raises(BaserowFormulaSyntaxError) as exc_info: + RuntimeToJson().execute({}, parsed_args) + assert "cannot be" in str(exc_info.value) + + +@pytest.mark.parametrize( + "args,expected", + [ + ([123], []), + (["foo"], []), + ([{"a": 1}], []), + ], +) +def test_runtime_to_json_validate_type_of_args(args, expected): + result = RuntimeToJson().validate_type_of_args(args) + assert result == expected + + +@pytest.mark.parametrize( + "args,expected", + [ + ([], False), + (["foo"], True), + (["foo", "bar"], False), + ], +) +def test_runtime_to_json_validate_number_of_args(args, expected): + result = RuntimeToJson().validate_number_of_args(args) + assert result is expected + + +@pytest.mark.parametrize( + "args,expected", + [ + (["[1, 2, 3]"], [1, 2, 3]), + (['{"a": 1}'], {"a": 1}), + (['"foo"'], "foo"), + (["123"], 123), + # Invalid JSON degrades gracefully to None instead of raising. + (["not json"], None), + ([""], None), + ], +) +def test_runtime_from_json_execute(args, expected): + parsed_args = RuntimeFromJson().parse_args(args) + result = RuntimeFromJson().execute({}, parsed_args) + assert result == expected + + +def test_runtime_to_json_from_json_round_trip(): + value = {"value1": 'foo "bar"\nbaz'} + dumped = RuntimeToJson().execute({}, RuntimeToJson().parse_args([value])) + result = RuntimeFromJson().execute({}, RuntimeFromJson().parse_args([dumped])) + assert result == value + + +@pytest.mark.parametrize( + "args,expected", + [ + ([], False), + (["foo"], True), + (["foo", "bar"], False), + ], +) +def test_runtime_from_json_validate_number_of_args(args, expected): + result = RuntimeFromJson().validate_number_of_args(args) + assert result is expected + + def test_runtime_null_execute(): parsed_args = RuntimeNull().parse_args([]) result = RuntimeNull().execute({}, parsed_args) diff --git a/backend/tests/baserow/core/formula/test_validator.py b/backend/tests/baserow/core/formula/test_validator.py index 6da2d15dd6..a9d9952781 100644 --- a/backend/tests/baserow/core/formula/test_validator.py +++ b/backend/tests/baserow/core/formula/test_validator.py @@ -225,6 +225,21 @@ def test_ensure_deserialized_json_returns_value_unchanged_when_json_is_invalid() assert ensure_deserialized_json({"name": "Ada"}) == {"name": "Ada"} +def test_ensure_deserialized_json_strict_raises_for_invalid_json_strings(): + with pytest.raises(ValidationError) as exc: + ensure_deserialized_json("not json", strict=True) + assert exc.value.args[0] == "Value is not valid JSON." + + with pytest.raises(ValidationError): + ensure_deserialized_json("", strict=True) + + +def test_ensure_deserialized_json_strict_keeps_non_strings_unchanged(): + assert ensure_deserialized_json({"name": "Ada"}, strict=True) == {"name": "Ada"} + assert ensure_deserialized_json(42, strict=True) == 42 + assert ensure_deserialized_json(None, strict=True) is None + + @pytest.mark.parametrize( "value,expected", [ diff --git a/changelog/entries/unreleased/feature/3879_add_to_json_and_from_json_expressions.json b/changelog/entries/unreleased/feature/3879_add_to_json_and_from_json_expressions.json new file mode 100644 index 0000000000..18c6d0b1b8 --- /dev/null +++ b/changelog/entries/unreleased/feature/3879_add_to_json_and_from_json_expressions.json @@ -0,0 +1,9 @@ +{ + "type": "feature", + "message": "Add to_json() and from_json() expressions.", + "issue_origin": "github", + "issue_number": 3879, + "domain": "core", + "bullet_points": [], + "created_at": "2026-06-22" +} diff --git a/web-frontend/locales/en.json b/web-frontend/locales/en.json index b8a222f570..6d76e3711a 100644 --- a/web-frontend/locales/en.json +++ b/web-frontend/locales/en.json @@ -701,6 +701,8 @@ "atDescription": "Returns the item in the first argument at the index specified by the second argument.", "toArrayDescription": "Converts a comma-delimited string into an array.", "rangeDescription": "Returns an array of numbers. Provide range(stop), range(start, stop), or range(start, stop, step).", + "toJsonDescription": "Serializes a value into a JSON-encoded string, escaping quotes, backslashes and control characters so it can be safely embedded inside a JSON document.", + "fromJsonDescription": "Parses a JSON-encoded string into a value such as an object, array or number. Returns null if the string is not valid JSON.", "nullDescription": "Returns null, which can be used as a value or for comparison", "numberFormatDescription": "Formats a number as a string with configurable decimal places, thousand separator, and decimal separator.", "toDatetimeDescription": "Parses a string into a datetime object. An optional second argument specifies the format (e.g. 'DD/MM/YYYY'). If omitted, ISO 8601 format is assumed.", diff --git a/web-frontend/modules/core/plugin.js b/web-frontend/modules/core/plugin.js index 78ce5f0c92..95e1984ba5 100644 --- a/web-frontend/modules/core/plugin.js +++ b/web-frontend/modules/core/plugin.js @@ -119,8 +119,10 @@ import { RuntimeSum, RuntimeAvg, RuntimeAt, - RuntimeToArray, RuntimeRange, + RuntimeToArray, + RuntimeToJson, + RuntimeFromJson, RuntimeToDatetime, } from '@baserow/modules/core/runtimeFormulaTypes' @@ -303,6 +305,8 @@ export default defineNuxtPlugin({ registry.register('runtimeFormulaFunction', new RuntimeAt(context)) registry.register('runtimeFormulaFunction', new RuntimeToArray(context)) registry.register('runtimeFormulaFunction', new RuntimeRange(context)) + registry.register('runtimeFormulaFunction', new RuntimeToJson(context)) + registry.register('runtimeFormulaFunction', new RuntimeFromJson(context)) registry.register('runtimeFormulaFunction', new RuntimeNull(context)) registry.register( 'runtimeFormulaFunction', diff --git a/web-frontend/modules/core/runtimeFormulaTypes.js b/web-frontend/modules/core/runtimeFormulaTypes.js index 1e81ea5490..b36b492e89 100644 --- a/web-frontend/modules/core/runtimeFormulaTypes.js +++ b/web-frontend/modules/core/runtimeFormulaTypes.js @@ -28,6 +28,8 @@ import { ensureString, ensureArray, ensureDateTime, + ensureJsonSerializable, + ensureDeserializedJson, } from '@baserow/modules/core/utils/validator' import { formatValueWithDurationFormat, @@ -2780,6 +2782,86 @@ export class RuntimeRange extends RuntimeFormulaFunction { } } +export class RuntimeToJson extends RuntimeFormulaFunction { + static getType() { + return 'to_json' + } + + static getFormulaType() { + return FORMULA_TYPE.FUNCTION + } + + static getCategoryType() { + return FORMULA_CATEGORY.UTILITY + } + + get args() { + return [new AnyBaserowRuntimeFormulaArgumentType()] + } + + execute(context, [arg]) { + try { + return JSON.stringify(ensureJsonSerializable(arg)) + } catch { + return null + } + } + + getDescription() { + const { $i18n: i18n } = this.app + return i18n.t('runtimeFormulaTypes.toJsonDescription') + } + + getExamples() { + return [ + { + formula: "concat('{\"value\": ', to_json('foo \"bar\"'), '}')", + result: '\'{"value": "foo \\"bar\\""}\'', + }, + ] + } +} + +export class RuntimeFromJson extends RuntimeFormulaFunction { + static getType() { + return 'from_json' + } + + static getFormulaType() { + return FORMULA_TYPE.FUNCTION + } + + static getCategoryType() { + return FORMULA_CATEGORY.UTILITY + } + + get args() { + return [new TextBaserowRuntimeFormulaArgumentType()] + } + + execute(context, [arg]) { + try { + return ensureDeserializedJson(arg, { strict: true }) + } catch { + return null + } + } + + getDescription() { + const { $i18n: i18n } = this.app + return i18n.t('runtimeFormulaTypes.fromJsonDescription') + } + + getExamples() { + return [ + { + formula: "from_json('[1, 2, 3]')", + result: '[1, 2, 3]', + }, + ] + } +} + export class RuntimeNull extends RuntimeFormulaFunction { static getType() { return 'null' diff --git a/web-frontend/modules/core/utils/validator.js b/web-frontend/modules/core/utils/validator.js index 4f1473d10e..4c2bad3f2f 100644 --- a/web-frontend/modules/core/utils/validator.js +++ b/web-frontend/modules/core/utils/validator.js @@ -322,3 +322,56 @@ export const ensureDuration = (value) => { throw new TypeError('Value cannot be converted to a duration.') } + +/** + * Normalizes a value into a JSON-serializable counterpart, mirroring the + * backend `ensure_json_serializable`. Special runtime types are converted to + * the same representation the backend uses, so `to_json` produces identical + * output on both sides: a `Timedelta` becomes its number of seconds. Arrays and + * plain objects are normalized recursively; every other value is returned + * unchanged so native JSON types pass through untouched. + * + * @param {*} value - The value to normalize. + * @returns {*} The JSON-serializable value. + */ +export const ensureJsonSerializable = (value) => { + if (value instanceof Timedelta) { + return Math.floor(value.ms / 1000) + } + + if (Array.isArray(value)) { + return value.map((item) => ensureJsonSerializable(item)) + } + + if (_.isPlainObject(value)) { + return _.mapValues(value, (item) => ensureJsonSerializable(item)) + } + + return value +} + +/** + * Decodes a JSON string if possible, otherwise returns the value unchanged, + * mirroring the backend `ensure_deserialized_json`. Values that are not strings + * are already deserialized and are returned as-is, even when `strict` is true. + * + * @param {*} value - The value to decode if it is a valid JSON string. + * @param {Boolean} strict - If true, throw when `value` is a string that cannot + * be decoded as JSON, instead of returning it unchanged. + * @returns {*} The decoded JSON value, or `value` when it is not a JSON string. + * @throws {Error} If `strict` is true and `value` is a string that is not valid + * JSON. + */ +export const ensureDeserializedJson = (value, { strict = false } = {}) => { + if (typeof value === 'string') { + try { + return JSON.parse(value) + } catch (e) { + if (strict) { + throw new Error('Value is not valid JSON.', { cause: e }) + } + } + } + + return value +} diff --git a/web-frontend/test/unit/core/formula/runtimeFormulaTypes.spec.js b/web-frontend/test/unit/core/formula/runtimeFormulaTypes.spec.js index 76af3f782c..403c572e8f 100644 --- a/web-frontend/test/unit/core/formula/runtimeFormulaTypes.spec.js +++ b/web-frontend/test/unit/core/formula/runtimeFormulaTypes.spec.js @@ -50,6 +50,8 @@ import { RuntimeAt, RuntimeToArray, RuntimeRange, + RuntimeToJson, + RuntimeFromJson, RuntimeNull, RuntimeNumberFormat, RuntimeToDatetime, @@ -2231,6 +2233,73 @@ describe('RuntimeRange', () => { }) }) +describe('RuntimeToJson', () => { + test.each([ + { args: ['foo'], expected: '"foo"' }, + { args: [123], expected: '123' }, + { args: [true], expected: 'true' }, + { args: [null], expected: 'null' }, + { args: [{ a: 1 }], expected: '{"a":1}' }, + { args: [[1, 2]], expected: '[1,2]' }, + // Quotes and control characters are escaped so the result is safe to embed + // inside a larger JSON document. + { args: ['foo "bar"'], expected: '"foo \\"bar\\""' }, + { args: ['a\nb'], expected: '"a\\nb"' }, + // A Timedelta is normalized to its number of seconds, matching the + // backend's `to_json` output (e.g. `3600`, not `{"ms":3600000}`). + { args: [new Timedelta(3600 * 1000)], expected: '3600' }, + { args: [new Timedelta(5400 * 1000)], expected: '5400' }, + // Timedeltas nested inside arrays and objects are normalized recursively. + { args: [[new Timedelta(1800 * 1000)]], expected: '[1800]' }, + { + args: [{ a: new Timedelta(3600 * 1000), b: [1, 2] }], + expected: '{"a":3600,"b":[1,2]}', + }, + ])('execute returns expected value', ({ args, expected }) => { + const formulaType = new RuntimeToJson() + const parsedArgs = formulaType.parseArgs(args) + const result = formulaType.execute({}, parsedArgs) + expect(result).toEqual(expected) + }) + + test.each([ + { args: [], expected: false }, + { args: ['foo'], expected: true }, + { args: ['foo', 'bar'], expected: false }, + ])('validates number of args', ({ args, expected }) => { + const formulaType = new RuntimeToJson() + const result = formulaType.validateNumberOfArgs(args) + expect(result).toStrictEqual(expected) + }) +}) + +describe('RuntimeFromJson', () => { + test.each([ + { args: ['[1, 2, 3]'], expected: [1, 2, 3] }, + { args: ['{"a": 1}'], expected: { a: 1 } }, + { args: ['"foo"'], expected: 'foo' }, + { args: ['123'], expected: 123 }, + // Invalid JSON degrades gracefully to null instead of throwing. + { args: ['not json'], expected: null }, + { args: [''], expected: null }, + ])('execute returns expected value', ({ args, expected }) => { + const formulaType = new RuntimeFromJson() + const parsedArgs = formulaType.parseArgs(args) + const result = formulaType.execute({}, parsedArgs) + expect(result).toEqual(expected) + }) + + test.each([ + { args: [], expected: false }, + { args: ['foo'], expected: true }, + { args: ['foo', 'bar'], expected: false }, + ])('validates number of args', ({ args, expected }) => { + const formulaType = new RuntimeFromJson() + const result = formulaType.validateNumberOfArgs(args) + expect(result).toStrictEqual(expected) + }) +}) + describe('RuntimeNull', () => { test('execute returns null', () => { const formulaType = new RuntimeNull() From caca908d5c4e7153dc5444a730258509dbca3b5f Mon Sep 17 00:00:00 2001 From: Davide Silvestri <75379892+silvestrid@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:13:18 +0200 Subject: [PATCH 2/2] fix(web-frontend): refresh access token on websocket reconnect (#5623) A backgrounded tab makes no HTTP calls, so the axios interceptor that refreshes the short-lived access token never runs and the token expires. On refocus the socket reconnected with the stale token, the server rejected it, and the reconnect latched into a permanent 'please refresh' error instead of recovering. connect() now refreshes an expiring/expired token before reconnecting, and an auth rejection forces one refresh-and-retry (bounded so a persistently rejected token can't loop) before surfacing the failure. --- ...es_when_returning_to_an_idle_tab_inst.json | 9 + .../modules/core/plugins/realTimeHandler.js | 75 +++++- .../test/unit/core/realTimeHandler.spec.js | 217 ++++++++++++++++++ 3 files changed, 295 insertions(+), 6 deletions(-) create mode 100644 changelog/entries/unreleased/bug/restores_realtime_updates_when_returning_to_an_idle_tab_inst.json diff --git a/changelog/entries/unreleased/bug/restores_realtime_updates_when_returning_to_an_idle_tab_inst.json b/changelog/entries/unreleased/bug/restores_realtime_updates_when_returning_to_an_idle_tab_inst.json new file mode 100644 index 0000000000..bf3a65ddb9 --- /dev/null +++ b/changelog/entries/unreleased/bug/restores_realtime_updates_when_returning_to_an_idle_tab_inst.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Restores real-time updates when returning to an idle tab instead of asking to refresh", + "issue_origin": "github", + "issue_number": null, + "domain": "core", + "bullet_points": [], + "created_at": "2026-07-01" +} diff --git a/web-frontend/modules/core/plugins/realTimeHandler.js b/web-frontend/modules/core/plugins/realTimeHandler.js index 1390b91742..3d86fe378b 100644 --- a/web-frontend/modules/core/plugins/realTimeHandler.js +++ b/web-frontend/modules/core/plugins/realTimeHandler.js @@ -10,6 +10,9 @@ const RECONNECT_BASE_DELAY = 1000 const RECONNECT_MAX_DELAY = 30000 const RECONNECT_MAX_ATTEMPTS = 10 const RECONNECT_JITTER = 1000 +// The handshake resets ``attempts`` before the auth result arrives, so the +// backoff cap can't bound an auth-rejection loop; bound the refreshes instead. +const MAX_TOKEN_REFRESH_RETRIES = 1 export class RealTimeHandler { constructor(context) { @@ -31,6 +34,11 @@ export class RealTimeHandler { this.lastSeenEventId = FIRST_CONNECT_CURSOR this.replayEnabled = false + this.connecting = false + // Set on a rejected token so the next reconnect refreshes before retrying. + this.forceTokenRefresh = false + this.tokenRefreshRetries = 0 + this.registerCoreEvents() this._onPageHide = () => { @@ -67,7 +75,7 @@ export class RealTimeHandler { * Creates a new connection with to the web socket so that real time updates can be * received. */ - connect(reconnect = true, anonymous = false) { + async connect(reconnect = true, anonymous = false) { if (!import.meta.client) { return } @@ -75,9 +83,6 @@ export class RealTimeHandler { this.reconnect = reconnect this.anonymous = anonymous - const jwtToken = this.context.store.getters['auth/token'] - const token = anonymous ? jwtToken || 'anonymous' : jwtToken - if ( this.socket && (this.socket.readyState === WebSocket.CONNECTING || @@ -86,11 +91,42 @@ export class RealTimeHandler { return } + // A backgrounded tab makes no HTTP calls, so the axios refresh interceptor + // never runs and the access token silently expires; reconnecting with it + // would be rejected as a permanent auth failure. Only this branch awaits, + // keeping the common reconnect path synchronous. + if (this._tokenRefreshNeeded(anonymous)) { + if (this.connecting) { + return + } + this.connecting = true + let transientRefreshFailure = false + try { + await this.context.store.dispatch('auth/refresh') + this.forceTokenRefresh = false + } catch (error) { + // A 401 means the refresh token itself expired: the session is gone + // and the guards below surface it. Anything else is transient (e.g. a + // network blip), so retry with backoff instead of failing, keeping + // forceTokenRefresh set for the next attempt. + transientRefreshFailure = error?.response?.status !== 401 + } finally { + this.connecting = false + } + if (transientRefreshFailure) { + this.delayedReconnect() + return + } + } + if (this.socket) { this.socket.onclose = null this.socket = null } + const jwtToken = this.context.store.getters['auth/token'] + const token = anonymous ? jwtToken || 'anonymous' : jwtToken + // "Failed — refresh" is only for genuine auth problems; transient // network failures keep retrying with capped backoff. const noToken = !token @@ -232,9 +268,21 @@ export class RealTimeHandler { return typeof navigator === 'undefined' || navigator.onLine !== false } + _tokenRefreshNeeded(anonymous) { + if (anonymous) { + return false + } + const store = this.context.store + if (!store.getters['auth/isAuthenticated']) { + return false + } + return this.forceTokenRefresh || store.getters['auth/shouldRefreshToken']() + } + _retryReconnectNow() { clearTimeout(this.reconnectTimeout) this.attempts = 0 + this.tokenRefreshRetries = 0 this.context.store.dispatch('toast/setFailedConnecting', false) // Keep the "Reconnecting" toast up until ``onopen`` clears it; flickering // it off here just confuses the user during the retry round-trip. @@ -335,6 +383,9 @@ export class RealTimeHandler { this.reconnect = false this.attempts = 0 this.connected = false + this.connecting = false + this.forceTokenRefresh = false + this.tokenRefreshRetries = 0 this.lastSeenEventId = FIRST_CONNECT_CURSOR // Reset until the next auth message confirms replay is enabled. this.replayEnabled = false @@ -391,8 +442,20 @@ export class RealTimeHandler { this.lastSeenEventId = NO_REPLAY_AVAILABLE } - if (data.success && this._canReplayEvents()) { - this._sendReplayEventsRequest() + if (data.success) { + this.forceTokenRefresh = false + this.tokenRefreshRetries = 0 + if (this._canReplayEvents()) { + this._sendReplayEventsRequest() + } + } else if ( + !this.anonymous && + this.tokenRefreshRetries < MAX_TOKEN_REFRESH_RETRIES + ) { + // A rejected token is usually just expired: refresh and retry rather + // than failing. Once the cap is hit, let connect()'s guard surface it. + this.tokenRefreshRetries++ + this.forceTokenRefresh = true } }) diff --git a/web-frontend/test/unit/core/realTimeHandler.spec.js b/web-frontend/test/unit/core/realTimeHandler.spec.js index 3ad634ff31..c8a5206366 100644 --- a/web-frontend/test/unit/core/realTimeHandler.spec.js +++ b/web-frontend/test/unit/core/realTimeHandler.spec.js @@ -790,3 +790,220 @@ describe('RealTimeHandler presence events', () => { ).toBe(true) }) }) + +describe('RealTimeHandler token refresh on reconnect', () => { + function makeRefreshStore({ shouldRefresh = false } = {}) { + const dispatched = [] + const store = { + getters: { + 'auth/token': 'stale-token', + 'auth/webSocketId': 'ws-id', + 'auth/isAuthenticated': true, + 'auth/shouldRefreshToken': () => shouldRefresh, + }, + dispatch(name, value) { + dispatched.push([name, value]) + if (name === 'auth/refresh') { + const refreshCount = dispatched.filter( + ([n]) => n === 'auth/refresh' + ).length + store.getters['auth/token'] = `fresh-token-${refreshCount}` + } + return Promise.resolve() + }, + subscribe() {}, + _dispatched: dispatched, + } + return store + } + + function makeHandlerWith(store) { + const context = { store, app: { router: {} } } + return { handler: new RealTimeHandler(context), store } + } + + beforeEach(() => { + Object.defineProperty(document, 'visibilityState', { + value: 'visible', + writable: true, + configurable: true, + }) + }) + + test('an expiring access token is refreshed before the socket opens', async () => { + const store = makeRefreshStore({ shouldRefresh: true }) + const { handler } = makeHandlerWith(store) + + await handler.connect(true, false) + + expect(store._dispatched.some(([n]) => n === 'auth/refresh')).toBe(true) + expect(handler.lastToken).toBe('fresh-token-1') + expect( + store._dispatched.some( + ([n, v]) => n === 'toast/setFailedConnecting' && v === true + ) + ).toBe(false) + }) + + test('a fresh-looking token is not refreshed on a normal reconnect', async () => { + const store = makeRefreshStore({ shouldRefresh: false }) + const { handler } = makeHandlerWith(store) + + await handler.connect(true, false) + + expect(store._dispatched.some(([n]) => n === 'auth/refresh')).toBe(false) + expect(handler.lastToken).toBe('stale-token') + }) + + test('anonymous reconnects never refresh the token', async () => { + const store = makeRefreshStore({ shouldRefresh: true }) + // An anonymous session has no refresh token to spend. + store.getters['auth/isAuthenticated'] = false + const { handler } = makeHandlerWith(store) + + await handler.connect(true, true) + + expect(store._dispatched.some(([n]) => n === 'auth/refresh')).toBe(false) + }) + + test('auth rejection forces a token refresh on the next reconnect', async () => { + // The token looks fresh to the client (shouldRefresh=false) but the server + // rejects it. The reconnect must still refresh instead of giving up. + const store = makeRefreshStore({ shouldRefresh: false }) + const { handler } = makeHandlerWith(store) + handler.reconnect = true + handler.anonymous = false + + await handler.connect(true, false) + expect(handler.lastToken).toBe('stale-token') + + fire(handler, 'authentication', { success: false }) + expect(handler.forceTokenRefresh).toBe(true) + + await handler.connect(true, false) + + expect(store._dispatched.some(([n]) => n === 'auth/refresh')).toBe(true) + expect(handler.lastToken).toBe('fresh-token-1') + expect(handler.forceTokenRefresh).toBe(false) + expect( + store._dispatched.some( + ([n, v]) => n === 'toast/setFailedConnecting' && v === true + ) + ).toBe(false) + }) + + test('a successful reconnect clears the forced-refresh flag', async () => { + const store = makeRefreshStore({ shouldRefresh: false }) + const { handler } = makeHandlerWith(store) + handler.forceTokenRefresh = true + + fire(handler, 'authentication', { success: true, replay_enabled: false }) + + expect(handler.forceTokenRefresh).toBe(false) + }) + + test('stops refreshing once a freshly refreshed token is also rejected', async () => { + const store = makeRefreshStore({ shouldRefresh: false }) + const { handler } = makeHandlerWith(store) + handler.reconnect = true + handler.anonymous = false + + fire(handler, 'authentication', { success: false }) + expect(handler.forceTokenRefresh).toBe(true) + expect(handler.tokenRefreshRetries).toBe(1) + await handler.connect(true, false) + expect(handler.lastToken).toBe('fresh-token-1') + + fire(handler, 'authentication', { success: false }) + expect(handler.forceTokenRefresh).toBe(false) + + // The next reconnect reuses the same rejected token and surfaces failure. + const refreshesBefore = store._dispatched.filter( + ([n]) => n === 'auth/refresh' + ).length + await handler.connect(true, false) + const refreshesAfter = store._dispatched.filter( + ([n]) => n === 'auth/refresh' + ).length + expect(refreshesAfter).toBe(refreshesBefore) + expect( + store._dispatched.some( + ([n, v]) => n === 'toast/setFailedConnecting' && v === true + ) + ).toBe(true) + }) + + test('a 401 refresh clears the session and surfaces the failure toast', async () => { + const dispatched = [] + const store = { + getters: { + 'auth/token': 'stale-token', + 'auth/webSocketId': 'ws-id', + 'auth/isAuthenticated': true, + 'auth/shouldRefreshToken': () => true, + }, + dispatch(name, value) { + dispatched.push([name, value]) + if (name === 'auth/refresh') { + store.getters['auth/token'] = null + const error = new Error('session expired') + error.response = { status: 401 } + return Promise.reject(error) + } + return Promise.resolve() + }, + subscribe() {}, + _dispatched: dispatched, + } + const { handler } = makeHandlerWith(store) + + await handler.connect(true, false) + + expect(dispatched.some(([n]) => n === 'auth/refresh')).toBe(true) + expect( + dispatched.some( + ([n, v]) => n === 'toast/setFailedConnecting' && v === true + ) + ).toBe(true) + }) + + test('a transient refresh failure retries with backoff instead of failing', async () => { + vi.useFakeTimers() + const dispatched = [] + const store = { + getters: { + 'auth/token': 'stale-token', + 'auth/webSocketId': 'ws-id', + 'auth/isAuthenticated': true, + 'auth/shouldRefreshToken': () => true, + }, + dispatch(name, value) { + dispatched.push([name, value]) + if (name === 'auth/refresh') { + // No ``response`` — a network-level error that leaves the token intact. + return Promise.reject(new Error('Network Error')) + } + return Promise.resolve() + }, + subscribe() {}, + _dispatched: dispatched, + } + const { handler } = makeHandlerWith(store) + handler.reconnect = true + + await handler.connect(true, false) + + expect( + dispatched.some( + ([n, v]) => n === 'toast/setFailedConnecting' && v === true + ) + ).toBe(false) + expect( + dispatched.some(([n, v]) => n === 'toast/setReconnecting' && v === true) + ).toBe(true) + expect(handler.connecting).toBe(false) + + clearTimeout(handler.reconnectTimeout) + vi.useRealTimers() + }) +})