diff --git a/.gitignore b/.gitignore index 7b21fe823..c72519acd 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,6 @@ snapshot-results/ .nx/workspace-data .nx/cache -*.local.code-workspace \ No newline at end of file +*.local.code-workspace +__pycache__/ +*.py[cod] diff --git a/docker-compose.docs-snapshots.yml b/docker-compose.docs-snapshots.yml index 5903f3bb2..3ad278390 100644 --- a/docker-compose.docs-snapshots.yml +++ b/docker-compose.docs-snapshots.yml @@ -8,15 +8,17 @@ services: [ 'sh', '-c', - 'node extract.cjs --directory /extract/plotly-express --output /results/plotly-express; node extract.cjs --directory /extract/ui --output /results/ui', + 'node extract.cjs --directory /extract/plotly-express --output /results/plotly-express; node extract.cjs --directory /extract/ui --output /results/ui; node extract.cjs --directory /extract/tradingview-lightweight --output /results/tradingview-lightweight', ] volumes: - ./plugins/ui/docs:/extract/ui - ./plugins/plotly-express/docs:/extract/plotly-express + - ./plugins/tradingview-lightweight/docs:/extract/tradingview-lightweight # exclude doc build directories by overriding with empty volumes # We could extract from the build directories, but we need to output the snapshots to the build directory itself for Salmon to pick up... - /extract/plotly-express/build/ - /extract/ui/build/ + - /extract/tradingview-lightweight/build/ - test-sets:/results # Read the test sets from the results directory and run them against the test server, taking a snapshot of the results. @@ -33,7 +35,7 @@ services: [ 'sh', '-c', - 'node snapshot.cjs --directory /test-sets/plotly-express --output /results/plotly-express --errors /errors/plotly-express; node snapshot.cjs --directory /test-sets/ui --output /results/ui --errors /errors/ui', + 'node snapshot.cjs --directory /test-sets/plotly-express --output /results/plotly-express --errors /errors/plotly-express; node snapshot.cjs --directory /test-sets/ui --output /results/ui --errors /errors/ui; node snapshot.cjs --directory /test-sets/tradingview-lightweight --output /results/tradingview-lightweight --errors /errors/tradingview-lightweight', ] volumes: - /var/run/docker.sock:/var/run/docker.sock # Allows the snapshotter to start its own docker containers for the test server @@ -42,8 +44,110 @@ services: # Map all the results back to the snapshots directory for those docs - ./plugins/ui/docs/snapshots:/results/ui - ./plugins/plotly-express/docs/snapshots:/results/plotly-express + - ./plugins/tradingview-lightweight/docs/snapshots:/results/tradingview-lightweight - ./snapshot-results/errors:/errors + # Pass 2: capture widget PNGs for the tradingview-lightweight docs. Runs + # AFTER salmon's Pass 1 snapshotter has finished and torn its server down. + # Brings up its own python-snapshotter compose via the mounted docker + # socket so it can join the shared snapshot network. The image-snapshotter + # tool itself is plugin-agnostic; per-plugin settings come from + # SNAPSHOTTER_* env vars below. + deephaven-plugins-docs-image-snapshotter-tvl: + build: + context: ./tools/image-snapshotter + container_name: deephaven-plugins-docs-image-snapshotter-tvl + depends_on: + deephaven-plugins-docs-snapshotter: + condition: service_completed_successfully + environment: + SNAPSHOTTER_BASE_URL: 'http://server:10000/ide/' + SNAPSHOTTER_PLUGIN_ROOT: '/work' + SNAPSHOTTER_PLUGIN: 'tradingview-lightweight' + SNAPSHOTTER_TARGET_SELECTOR: '.dh-tvl-chart' + SNAPSHOTTER_WIDGET_TYPE: 'deephaven.plot.tradingview_lightweight.TvlChart' + SNAPSHOTTER_APP_ID: 'tvl.docs.examples' + SNAPSHOTTER_APP_NAME: 'TVL Docs Examples' + SNAPSHOTTER_COMPOSE_FILE: '/workspace/docker/python-snapshotter/docker-compose.yml' + SNAPSHOTTER_SERVER_CONTAINER: 'tvl-snapshotter-server' + # Pass through the host's SNAPSHOTTER_FORCE so callers can opt in to + # cache-busting recapture without editing this file. + SNAPSHOTTER_FORCE: ${SNAPSHOTTER_FORCE:-} + CI: 'true' + # The Pass 2 entrypoint brings up the python-snapshotter test server + # via the mounted docker socket. That compose file bind-mounts the + # generated app.d, which is interpreted against the HOST filesystem + # (the socket is the host's). `${PWD}` is evaluated by compose at + # compose-up time against the user's shell, so HOST_PWD inside the + # container always carries the host's working directory. + HOST_PWD: ${PWD} + volumes: + # Allows the snapshotter to start its own server container alongside us. + - /var/run/docker.sock:/var/run/docker.sock + # docker/python-snapshotter/docker-compose.yml is resolved from the + # host's view of the socket — mount the repo at a stable path inside + # the container so SNAPSHOTTER_COMPOSE_FILE resolves correctly. + - ${PWD}:/workspace + # The spec walks docs/*.md (read-only) and writes PNGs + JSON envelopes + # into docs/snapshots. PLUGIN_ROOT resolves to /work, so /work/docs is + # the docs tree and /work/docs/snapshots is the output dir. + - ./plugins/tradingview-lightweight/docs:/work/docs:ro + - ./plugins/tradingview-lightweight/docs/snapshots:/work/docs/snapshots + + # Surface snapshot errors to the user. Compose's `depends_on` conditions + # only fire on success (`service_completed_successfully`); there's no + # "completed-with-any-exit" condition. And dependency stdout doesn't + # stream to the terminal when you `docker compose run ` — only + # the target's own stdout does. So the salmon snapshotter's errors were + # silently written to /errors and never shown. + # + # Workaround: this service has no depends_on. The npm `update-doc-snapshots` + # script invokes it as its own `compose run` target right after the main + # chain — so its stdout streams to the user, regardless of whether the + # snapshotter passed or failed. Bind-mounts the same errors dir the + # snapshotter writes to. Exits non-zero if any errors are present. + # Also acts as the post-run chown step. The salmon snapshotter and the + # Pass-2 image-snapshotter both run as root, so the PNGs + JSON envelopes + # they write into docs/snapshots land owned by root and the host user + # can't edit them. Bind-mounting those dirs here (rw) and chowning them + # to SNAPSHOT_UID:SNAPSHOT_GID (exported by tools/run_docker.sh) makes + # them editable on the host. Runs whether or not snapshots succeeded. + deephaven-plugins-docs-error-reporter: + image: alpine:3.20 + environment: + SNAPSHOT_UID: ${SNAPSHOT_UID:-0} + SNAPSHOT_GID: ${SNAPSHOT_GID:-0} + volumes: + - ./snapshot-results/errors:/errors + - ./plugins/ui/docs/snapshots:/snapshots/ui + - ./plugins/plotly-express/docs/snapshots:/snapshots/plotly-express + - ./plugins/tradingview-lightweight/docs/snapshots:/snapshots/tradingview-lightweight + command: + - sh + - -c + - | + if [ "$$SNAPSHOT_UID" != "0" ]; then + chown -R "$$SNAPSHOT_UID:$$SNAPSHOT_GID" /errors /snapshots 2>/dev/null || true + fi + had=0 + for plugin in plotly-express ui tradingview-lightweight; do + f="/errors/$$plugin/errors.txt" + if [ -s "$$f" ]; then + had=1 + echo "" + echo "==================== $$plugin ($$f) ====================" + head -n 200 "$$f" + n=$$(wc -l < "$$f") + if [ "$$n" -gt 200 ]; then + echo "... ($$n total lines; full file at snapshot-results/errors/$$plugin/errors.txt)" + fi + fi + done + if [ "$$had" -eq 0 ]; then + echo "[error-reporter] no snapshot errors recorded." + fi + exit $$had + # Validate MDX and the snapshots that were written are valid deephaven-plugins-docs-validator: image: ghcr.io/deephaven/salmon-validator @@ -51,6 +155,7 @@ services: volumes: - ./plugins/ui/docs/build/markdown:/validate/ui - ./plugins/plotly-express/docs/build/markdown:/validate/plotly-express + - ./plugins/tradingview-lightweight/docs/build/markdown:/validate/tradingview-lightweight - ./docker/build/validator-results:/results volumes: diff --git a/docker-compose.docs.yml b/docker-compose.docs.yml index 7d8cb008d..bbf35f02f 100644 --- a/docker-compose.docs.yml +++ b/docker-compose.docs.yml @@ -9,3 +9,4 @@ services: volumes: - ./plugins/ui/docs${BUILT+/build/markdown}:/salmon/core/ui/docs - ./plugins/plotly-express/docs${BUILT+/build/markdown}:/salmon/core/plotly/docs + - ./plugins/tradingview-lightweight/docs${BUILT+/build/markdown}:/salmon/core/tradingview-lightweight/docs diff --git a/docker/python-snapshotter/docker-compose.yml b/docker/python-snapshotter/docker-compose.yml new file mode 100644 index 000000000..d1138b542 --- /dev/null +++ b/docker/python-snapshotter/docker-compose.yml @@ -0,0 +1,43 @@ +# Pass 2 (image-snapshotter) test server. Plugin-agnostic. +# +# Reuses the python-server image already built by salmon's Pass 1 +# (../python/docker-compose.yml). The only differences vs. that compose: +# +# - We mount a generated app.d directory at /app.d so the docs examples +# are pre-loaded as Deephaven panels (the Pass 2 server is otherwise +# headless — no code is ever sent over the console). +# - START_OPTS adds -Ddeephaven.application.dir=/app.d to enable app +# mode against that directory. +# - DEEPHAVEN_PLUGINS_STATIC_DATA=1 forces tvl.data / dx.data into +# static mode so screenshots are deterministic. +# +# image-snapshotter's entrypoint sets PASS2_APPD_HOST_PATH to the host +# filesystem path of the generated app.d (bind mounts traverse the host +# because the docker socket is the host's). The `:?` makes compose fail +# fast if the entrypoint forgot to export it. +# +# SNAPSHOTTER_SERVER_CONTAINER lets parallel plugin invocations produce +# unique container names. Serial Pass 2 invocations (the default +# docs-snapshots compose flow) can rely on the default. +# +# The image tag matches what `docker/python/docker-compose.yml` builds — +# `image: python-server:latest` rather than a `build:` block because Pass 1 +# always runs first and produces the image. On a clean run with Pass 1 +# skipped the image won't exist locally and `up` will fail; that's the +# documented contract. +services: + server: + image: python-server:latest + container_name: ${SNAPSHOTTER_SERVER_CONTAINER:-deephaven-snapshotter-server} + expose: + - 10000 + environment: + - START_OPTS=-Xmx4g -DAuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler -Ddeephaven.console.type=python -Ddeephaven.application.dir=/app.d + - DEEPHAVEN_PLUGINS_STATIC_DATA=1 + volumes: + - ${PASS2_APPD_HOST_PATH:?PASS2_APPD_HOST_PATH must be set}:/app.d:ro + +networks: + default: + name: deephaven-plugins-docs-snapshot-network + external: true diff --git a/docker/python/docker-compose.yml b/docker/python/docker-compose.yml index dd393c530..b88d1660a 100644 --- a/docker/python/docker-compose.yml +++ b/docker/python/docker-compose.yml @@ -8,6 +8,11 @@ services: - 10000 environment: - START_OPTS=-Xmx4g -DAuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler -Ddeephaven.console.type=python + # Force tvl.data / dx.data into static mode so docs snapshots are + # reproducible AND so doc blocks that chain `.update(["X = (double)ii"])` + # onto a generator output don't blow up on the refreshing-formula check + # (the merged static+time_table isn't classified as append-only). + - DEEPHAVEN_PLUGINS_STATIC_DATA=1 networks: default: diff --git a/package-lock.json b/package-lock.json index e3654d001..40fd1a21f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -105,7 +105,6 @@ "node_modules/@adobe/react-spectrum": { "version": "3.47.0", "license": "Apache-2.0", - "peer": true, "dependencies": { "@internationalized/date": "^3.12.1", "@react-types/shared": "^3.34.0", @@ -170,7 +169,6 @@ "version": "7.29.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -729,7 +727,6 @@ "version": "7.28.6", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.28.6" }, @@ -1541,7 +1538,6 @@ "version": "7.28.6", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-annotate-as-pure": "^7.27.3", "@babel/helper-module-imports": "^7.28.6", @@ -1998,6 +1994,7 @@ "version": "2.0.8", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cacheable/utils": "^2.4.0", "@keyv/bigmap": "^1.3.1", @@ -2009,6 +2006,7 @@ "version": "1.3.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "hashery": "^1.4.0", "hookified": "^1.15.0" @@ -2033,6 +2031,7 @@ "version": "2.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "hashery": "^1.5.1", "keyv": "^5.6.0" @@ -2042,6 +2041,7 @@ "version": "5.6.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@keyv/serialize": "^1.1.1" } @@ -2070,6 +2070,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=20.19.0" }, @@ -2114,6 +2115,7 @@ } ], "license": "MIT-0", + "peer": true, "peerDependencies": { "css-tree": "^3.2.1" }, @@ -2156,6 +2158,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=20.19.0" }, @@ -2178,6 +2181,7 @@ } ], "license": "MIT-0", + "peer": true, "engines": { "node": ">=20.19.0" }, @@ -2199,6 +2203,7 @@ } ], "license": "MIT-0", + "peer": true, "engines": { "node": ">=20.19.0" }, @@ -2506,7 +2511,9 @@ } }, "node_modules/@deephaven/dashboard-core-plugins": { - "version": "1.26.0", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@deephaven/dashboard-core-plugins/-/dashboard-core-plugins-1.26.1.tgz", + "integrity": "sha512-jWPqxwOuSUCaQ0Q6eotUY+T4p1dmXb3Hvy8nMv+qSCOh3l+i06/yWQEltfPzNIh+prnHeqTvyMHjISAouBdq8g==", "license": "Apache-2.0", "dependencies": { "@deephaven/chart": "^1.26.0", @@ -2518,13 +2525,13 @@ "@deephaven/golden-layout": "^1.24.0", "@deephaven/grid": "^1.25.0", "@deephaven/icons": "^1.2.0", - "@deephaven/iris-grid": "^1.26.0", + "@deephaven/iris-grid": "^1.26.1", "@deephaven/jsapi-bootstrap": "^1.23.0", - "@deephaven/jsapi-components": "^1.23.0", + "@deephaven/jsapi-components": "^1.26.1", "@deephaven/jsapi-types": "^1.0.0-dev0.40.4", "@deephaven/jsapi-utils": "^1.23.0", "@deephaven/log": "^1.8.0", - "@deephaven/plugin": "^1.26.0", + "@deephaven/plugin": "^1.26.1", "@deephaven/react-hooks": "^1.21.1", "@deephaven/redux": "^1.23.0", "@deephaven/storage": "^1.8.0", @@ -2695,7 +2702,9 @@ } }, "node_modules/@deephaven/iris-grid": { - "version": "1.26.0", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@deephaven/iris-grid/-/iris-grid-1.26.1.tgz", + "integrity": "sha512-yzuOWIA6SpxX+48VF5zmuH+7feEz0ArKrOuFheoEXwZuMnCHLwYmncULL6uC/+GHMr/Nf409yoyJaxKYz4Q+3Q==", "license": "Apache-2.0", "dependencies": { "@deephaven/components": "^1.22.1", @@ -2703,7 +2712,7 @@ "@deephaven/filters": "^1.1.0", "@deephaven/grid": "^1.25.0", "@deephaven/icons": "^1.2.0", - "@deephaven/jsapi-components": "^1.23.0", + "@deephaven/jsapi-components": "^1.26.1", "@deephaven/jsapi-types": "^1.0.0-dev0.40.4", "@deephaven/jsapi-utils": "^1.23.0", "@deephaven/log": "^1.8.0", @@ -2792,6 +2801,10 @@ "resolved": "plugins/theme-pack/src/js", "link": true }, + "node_modules/@deephaven/js-plugin-tradingview-lightweight": { + "resolved": "plugins/tradingview-lightweight/src/js", + "link": true + }, "node_modules/@deephaven/js-plugin-ui": { "resolved": "plugins/ui/src/js", "link": true @@ -2815,7 +2828,9 @@ } }, "node_modules/@deephaven/jsapi-components": { - "version": "1.23.0", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@deephaven/jsapi-components/-/jsapi-components-1.26.1.tgz", + "integrity": "sha512-6I2vavKuCkwJ+opeu2n38ex4tIGNlxRDby+x4hcuBTZzUqYI4+RqgbnkaJGsaxq4OYDR6E0Zq5W0Au5wuMFeYQ==", "license": "Apache-2.0", "dependencies": { "@deephaven/components": "^1.22.1", @@ -2885,7 +2900,9 @@ } }, "node_modules/@deephaven/plugin": { - "version": "1.26.0", + "version": "1.26.1", + "resolved": "https://registry.npmjs.org/@deephaven/plugin/-/plugin-1.26.1.tgz", + "integrity": "sha512-3KQwwVmv+Af2ZmFtNFO+CYKf5uTGAEu94Ptkjh7AhAyIO7n8eoKbLQW4aPp3ILwPviJi5BtN5qv1u41lNvn4kg==", "license": "Apache-2.0", "dependencies": { "@deephaven/components": "^1.22.1", @@ -2893,7 +2910,7 @@ "@deephaven/golden-layout": "^1.24.0", "@deephaven/grid": "^1.25.0", "@deephaven/icons": "^1.2.0", - "@deephaven/iris-grid": "^1.26.0", + "@deephaven/iris-grid": "^1.26.1", "@deephaven/jsapi-types": "^1.0.0-dev0.40.4", "@deephaven/log": "^1.8.0", "@deephaven/react-hooks": "^1.21.1", @@ -3060,7 +3077,6 @@ "node_modules/@dnd-kit/core": { "version": "6.3.1", "license": "MIT", - "peer": true, "dependencies": { "@dnd-kit/accessibility": "^3.1.1", "@dnd-kit/utilities": "^3.2.2", @@ -3634,7 +3650,6 @@ "node_modules/@fortawesome/fontawesome-svg-core": { "version": "6.7.2", "license": "MIT", - "peer": true, "dependencies": { "@fortawesome/fontawesome-common-types": "6.7.2" }, @@ -3645,7 +3660,6 @@ "node_modules/@fortawesome/react-fontawesome": { "version": "0.2.6", "license": "MIT", - "peer": true, "dependencies": { "prop-types": "^15.8.1" }, @@ -3700,8 +3714,7 @@ }, "node_modules/@hello-pangea/dnd/node_modules/redux": { "version": "5.0.1", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@humanwhocodes/config-array": { "version": "0.13.0", @@ -4071,7 +4084,9 @@ } }, "node_modules/@internationalized/date": { - "version": "3.12.1", + "version": "3.12.2", + "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.2.tgz", + "integrity": "sha512-FY1Y+H64NDs+HAF6omlnWxm3mEpfgaCSWtL5l551ZZfImA+kGjPFgrnJrGjH6lfmLL0g8Z/mBu1R3kufeCp6Jw==", "license": "Apache-2.0", "dependencies": { "@swc/helpers": "^0.5.0" @@ -4086,14 +4101,18 @@ } }, "node_modules/@internationalized/number": { - "version": "3.6.6", + "version": "3.6.7", + "resolved": "https://registry.npmjs.org/@internationalized/number/-/number-3.6.7.tgz", + "integrity": "sha512-3ji1fcrT+FPAK86UqEhB/psHixYo6niWPJtt7+qRaYFynt/BaJG8GhAPimtWUpEiVSTq8ZM8L5psMxGquiB/Vg==", "license": "Apache-2.0", "dependencies": { "@swc/helpers": "^0.5.0" } }, "node_modules/@internationalized/string": { - "version": "3.2.8", + "version": "3.2.9", + "resolved": "https://registry.npmjs.org/@internationalized/string/-/string-3.2.9.tgz", + "integrity": "sha512-kzP/M/mbQxODlmOt4bIQZ2SBVUWUSqMLXooXixnX7noche8WHaQcA+nwFN1K2KCF/cp+LDUhcJsCicwkvhD1pg==", "license": "Apache-2.0", "dependencies": { "@swc/helpers": "^0.5.0" @@ -4649,7 +4668,8 @@ "node_modules/@keyv/serialize": { "version": "1.1.1", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@mapbox/geojson-rewind": { "version": "0.5.2", @@ -5658,7 +5678,6 @@ "version": "5.2.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.1.0", @@ -6117,6 +6136,7 @@ "version": "0.1.2", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^12.20.0 || ^14.18.0 || >=16.0.0" }, @@ -6993,6 +7013,7 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -7680,7 +7701,8 @@ "node_modules/@types/aria-query": { "version": "5.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/babel__core": { "version": "7.20.5", @@ -7922,7 +7944,6 @@ "version": "20.19.39", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -8110,7 +8131,6 @@ "version": "5.62.0", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -8367,7 +8387,6 @@ "node_modules/acorn": { "version": "8.16.0", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8801,6 +8820,7 @@ "version": "2.0.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8" } @@ -9289,7 +9309,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", @@ -9472,6 +9491,7 @@ "version": "2.3.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cacheable/memory": "^2.0.8", "@cacheable/utils": "^2.4.0", @@ -9484,6 +9504,7 @@ "version": "5.6.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@keyv/serialize": "^1.1.1" } @@ -9874,7 +9895,8 @@ "node_modules/colord": { "version": "2.9.3", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/columnify": { "version": "1.6.0", @@ -10239,6 +10261,7 @@ "version": "3.3.3", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" } @@ -10961,6 +10984,7 @@ "version": "5.20.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.0" @@ -11709,7 +11733,6 @@ "version": "8.57.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -11802,7 +11825,6 @@ "version": "8.3.0", "dev": true, "license": "MIT", - "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -11881,6 +11903,7 @@ "version": "3.5.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "debug": "^4.3.4", "enhanced-resolve": "^5.10.0", @@ -11905,6 +11928,7 @@ "version": "13.2.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "dir-glob": "^3.0.1", "fast-glob": "^3.3.0", @@ -11923,6 +11947,7 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -11994,7 +12019,6 @@ "version": "2.32.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -12069,7 +12093,6 @@ "version": "6.10.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "aria-query": "^5.3.2", "array-includes": "^3.1.8", @@ -12106,6 +12129,7 @@ "version": "5.5.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "prettier-linter-helpers": "^1.0.1", "synckit": "^0.11.12" @@ -12135,6 +12159,7 @@ "version": "0.2.9", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^12.20.0 || ^14.18.0 || >=16.0.0" }, @@ -12146,6 +12171,7 @@ "version": "0.11.12", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@pkgr/core": "^0.2.9" }, @@ -12160,7 +12186,6 @@ "version": "7.37.5", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "array-includes": "^3.1.8", "array.prototype.findlast": "^1.2.5", @@ -12192,7 +12217,6 @@ "version": "4.6.2", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -12281,6 +12305,7 @@ "version": "2.1.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "eslint-visitor-keys": "^1.1.0" }, @@ -12295,6 +12320,7 @@ "version": "1.3.0", "dev": true, "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=4" } @@ -12558,6 +12584,12 @@ "node": ">=0.4.0" } }, + "node_modules/fancy-canvas": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fancy-canvas/-/fancy-canvas-2.1.0.tgz", + "integrity": "sha512-nifxXJ95JNLFR2NgRV4/MxVP45G9909wJTEKz5fg/TZS20JJZA6hfgRVh/bC9bwl2zBtBNcYPjiBE4njQHVBwQ==", + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "license": "MIT" @@ -12565,7 +12597,8 @@ "node_modules/fast-diff": { "version": "1.3.0", "dev": true, - "license": "Apache-2.0" + "license": "Apache-2.0", + "peer": true }, "node_modules/fast-glob": { "version": "3.3.3", @@ -12633,6 +12666,7 @@ "version": "1.0.16", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">= 4.9.1" } @@ -13010,6 +13044,7 @@ "version": "1.5.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -13183,6 +13218,7 @@ "version": "4.14.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "resolve-pkg-maps": "^1.0.0" }, @@ -13347,6 +13383,7 @@ "version": "2.0.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "global-prefix": "^3.0.0" }, @@ -13358,6 +13395,7 @@ "version": "3.0.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "ini": "^1.3.5", "kind-of": "^6.0.2", @@ -13371,6 +13409,7 @@ "version": "1.3.1", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "isexe": "^2.0.0" }, @@ -13429,7 +13468,8 @@ "node_modules/globjoin": { "version": "0.1.4", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/glsl-inject-defines": { "version": "1.0.3", @@ -13783,6 +13823,7 @@ "version": "1.5.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "hookified": "^1.15.0" }, @@ -13886,7 +13927,8 @@ "node_modules/hookified": { "version": "1.15.1", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/hosted-git-info": { "version": "9.0.2", @@ -13926,6 +13968,7 @@ "version": "5.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=20.10" }, @@ -14111,6 +14154,7 @@ "version": "4.2.0", "dev": true, "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -14585,6 +14629,7 @@ "version": "5.0.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -14892,7 +14937,6 @@ "version": "29.7.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -16137,8 +16181,7 @@ }, "node_modules/jquery": { "version": "3.7.1", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/js-cookie": { "version": "3.0.5", @@ -16853,6 +16896,15 @@ "immediate": "~3.0.5" } }, + "node_modules/lightweight-charts": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/lightweight-charts/-/lightweight-charts-5.2.0.tgz", + "integrity": "sha512-ey3Vas8UhV06ni+LT9TA1nEe4y8So4Mi6CL/oarNHFMyTktz/xy8e8+oh04Q//eO3t6etvFXgayz2fClyFQb5w==", + "license": "Apache-2.0", + "dependencies": { + "fancy-canvas": "2.1.0" + } + }, "node_modules/lines-and-columns": { "version": "2.0.3", "dev": true, @@ -16936,7 +16988,8 @@ "node_modules/lodash.truncate": { "version": "4.4.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/log-symbols": { "version": "4.1.0", @@ -17005,6 +17058,7 @@ "version": "1.5.0", "dev": true, "license": "MIT", + "peer": true, "bin": { "lz-string": "bin/bin.js" } @@ -17269,6 +17323,7 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -17490,7 +17545,8 @@ "node_modules/mdn-data": { "version": "2.27.1", "dev": true, - "license": "CC0-1.0" + "license": "CC0-1.0", + "peer": true }, "node_modules/memoize-one": { "version": "5.2.1", @@ -19105,7 +19161,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@napi-rs/wasm-runtime": "0.2.4", "@yarnpkg/lockfile": "^1.1.0", @@ -20165,7 +20220,6 @@ "node_modules/plotly.js": { "version": "3.5.0", "license": "MIT", - "peer": true, "dependencies": { "@plotly/d3": "3.8.2", "@plotly/d3-sankey": "0.7.2", @@ -20237,7 +20291,6 @@ "node_modules/popper.js": { "version": "1.16.1", "license": "MIT", - "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -20268,7 +20321,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -20296,6 +20348,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18.0" }, @@ -20307,7 +20360,6 @@ "version": "7.1.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -20319,7 +20371,8 @@ "node_modules/postcss-value-parser": { "version": "4.2.0", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/potpack": { "version": "1.0.2", @@ -20337,7 +20390,6 @@ "version": "3.0.0", "dev": true, "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -20352,6 +20404,7 @@ "version": "1.0.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-diff": "^1.1.2" }, @@ -20363,6 +20416,7 @@ "version": "27.5.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -20376,6 +20430,7 @@ "version": "5.2.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -20548,6 +20603,7 @@ "version": "0.9.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "hookified": "^2.1.1" }, @@ -20558,7 +20614,8 @@ "node_modules/qified/node_modules/hookified": { "version": "2.1.1", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/querystringify": { "version": "2.2.0", @@ -20609,7 +20666,6 @@ "node_modules/react": { "version": "18.3.1", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -20655,7 +20711,6 @@ "node_modules/react-dom": { "version": "18.3.1", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -20666,8 +20721,7 @@ }, "node_modules/react-is": { "version": "17.0.2", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/react-markdown": { "version": "8.0.7", @@ -20716,7 +20770,6 @@ "node_modules/react-redux": { "version": "7.2.9", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.15.4", "@types/react-redux": "^7.1.20", @@ -21022,7 +21075,6 @@ "node_modules/redux": { "version": "4.2.1", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.9.2" } @@ -21093,6 +21145,7 @@ "version": "3.2.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8" }, @@ -21346,6 +21399,7 @@ "version": "1.0.0", "dev": true, "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } @@ -21769,7 +21823,6 @@ "version": "1.99.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "chokidar": "^4.0.0", "immutable": "^5.1.5", @@ -22001,6 +22054,7 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", @@ -22489,6 +22543,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@csstools/css-calc": "^3.1.1", "@csstools/css-parser-algorithms": "^4.0.0", @@ -22538,6 +22593,7 @@ "version": "6.2.2", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -22549,6 +22605,7 @@ "version": "9.0.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -22574,6 +22631,7 @@ "version": "11.1.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "flat-cache": "^6.1.20" } @@ -22582,6 +22640,7 @@ "version": "6.1.22", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "cacheable": "^2.3.4", "flatted": "^3.4.2", @@ -22592,6 +22651,7 @@ "version": "16.2.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@sindresorhus/merge-streams": "^4.0.0", "fast-glob": "^3.3.3", @@ -22611,6 +22671,7 @@ "version": "7.0.5", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">= 4" } @@ -22619,6 +22680,7 @@ "version": "4.0.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -22630,6 +22692,7 @@ "version": "14.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=20" }, @@ -22641,6 +22704,7 @@ "version": "4.1.0", "dev": true, "license": "ISC", + "peer": true, "engines": { "node": ">=14" }, @@ -22652,6 +22716,7 @@ "version": "5.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=14.16" }, @@ -22663,6 +22728,7 @@ "version": "8.2.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "get-east-asian-width": "^1.5.0", "strip-ansi": "^7.1.2" @@ -22678,6 +22744,7 @@ "version": "7.2.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "ansi-regex": "^6.2.2" }, @@ -22692,6 +22759,7 @@ "version": "7.0.1", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "signal-exit": "^4.0.1" }, @@ -22729,6 +22797,7 @@ "version": "4.4.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "has-flag": "^5.0.1", "supports-color": "^10.2.2" @@ -22744,6 +22813,7 @@ "version": "5.0.1", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -22755,6 +22825,7 @@ "version": "10.2.2", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -22806,7 +22877,8 @@ }, "node_modules/svg-tags": { "version": "1.0.0", - "dev": true + "dev": true, + "peer": true }, "node_modules/symbol-tree": { "version": "3.2.4", @@ -22816,6 +22888,7 @@ "version": "0.8.8", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@pkgr/core": "^0.1.0", "tslib": "^2.6.2" @@ -22831,6 +22904,7 @@ "version": "6.9.0", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "ajv": "^8.0.1", "lodash.truncate": "^4.4.2", @@ -22843,9 +22917,12 @@ } }, "node_modules/tapable": { - "version": "2.3.2", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6" }, @@ -23018,7 +23095,6 @@ "version": "4.0.4", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -23333,7 +23409,6 @@ "version": "5.9.3", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -23416,6 +23491,7 @@ "version": "0.4.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=20" }, @@ -23699,7 +23775,6 @@ "version": "5.4.21", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", @@ -24558,6 +24633,36 @@ "@deephaven/plugin": "^1.18.0" } }, + "plugins/tradingview-lightweight/src/js": { + "name": "@deephaven/js-plugin-tradingview-lightweight", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "@deephaven/components": "^1.22.1", + "@deephaven/dashboard": "^1.24.0", + "@deephaven/dashboard-core-plugins": "^1.24.0", + "@deephaven/icons": "^1.2.0", + "@deephaven/jsapi-bootstrap": "^1.23.0", + "@deephaven/log": "^1.8.0", + "@deephaven/plugin": "^1.24.0", + "@deephaven/redux": "^1.23.0", + "@deephaven/utils": "^1.10.0", + "lightweight-charts": "^5.2.0", + "react-redux": "^7.2.9" + }, + "devDependencies": { + "@deephaven/jsapi-types": "^1.0.0-dev0.39.6", + "@deephaven/test-utils": "^1.8.0", + "@types/react": "^18.0.0", + "react": "^18.0.0", + "react-dom": "^18.0.0", + "typescript": "^5.9.3" + }, + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, "plugins/ui/src/js": { "name": "@deephaven/js-plugin-ui", "version": "0.40.2", diff --git a/package.json b/package.json index 5ef8b570d..b2aeaa312 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "update-dh-packages": "lerna run --concurrency 1 update-dh-packages", "update-dh-packages:ui": "npm run update-dh-packages -- --scope=@deephaven/js-plugin-ui --", "validate-marketplace": "node tools/validate-marketplace.mjs", - "update-doc-snapshots": "./tools/run_docker.sh ./docker-compose.docs-snapshots.yml deephaven-plugins-docs-snapshotter", + "update-doc-snapshots": "docker compose -f ./docker/python/docker-compose.yml build server && (./tools/run_docker.sh ./docker-compose.docs-snapshots.yml deephaven-plugins-docs-image-snapshotter-tvl; ec=$?; ./tools/run_docker.sh ./docker-compose.docs-snapshots.yml deephaven-plugins-docs-error-reporter || true; exit $ec)", "validate-doc-snapshots": "./tools/run_docker.sh ./docker-compose.docs-snapshots.yml deephaven-plugins-docs-validator" }, "devDependencies": { diff --git a/plugins/ag-grid/src/js/src/components/AgGridView.tsx b/plugins/ag-grid/src/js/src/components/AgGridView.tsx index d470dbceb..8ca72d1f0 100644 --- a/plugins/ag-grid/src/js/src/components/AgGridView.tsx +++ b/plugins/ag-grid/src/js/src/components/AgGridView.tsx @@ -57,6 +57,7 @@ export function AgGridView({ const [isVisible, setIsVisible] = useState(false); const [isFirstDataRendered, setIsFirstDataRendered] = useState(false); + const [isFirstDataLoaded, setIsFirstDataLoaded] = useState(false); log.debug('AgGridView rendering', table); @@ -121,9 +122,33 @@ export function AgGridView({ [datasource] ); + /** + * The viewport row model creates blank row nodes for the visible range and + * fills them as Deephaven viewport updates arrive, so firstDataRendered can + * fire while some displayed rows still have no data. Auto-sizing at that + * point measures a partial column (e.g. before any negative value has + * arrived) and the resulting widths depend on update chunk timing — column + * widths would differ run to run. Only treat data as loaded once every + * displayed row actually has data; the viewport model dispatches + * modelUpdated after each setRowData, so this is re-checked as data lands. + */ + const checkFirstDataLoaded = () => { + const gridApi = gridApiRef.current; + if (!gridApi) return; + const first = gridApi.getFirstDisplayedRowIndex(); + const last = gridApi.getLastDisplayedRowIndex(); + for (let i = first; i <= last; i += 1) { + if (gridApi.getDisplayedRowAtIndex(i)?.data == null) { + return; + } + } + setIsFirstDataLoaded(true); + }; + const handleFirstDataRendered = (event: FirstDataRenderedEvent) => { log.debug('handleFirstDataRendered', event); setIsFirstDataRendered(true); + checkFirstDataLoaded(); }; const handleGridSizeChanged = (event: GridSizeChangedEvent) => { @@ -131,11 +156,17 @@ export function AgGridView({ setIsVisible(event.clientHeight > 0 && event.clientWidth > 0); }; + const handleModelUpdated = () => { + if (!isFirstDataLoaded) { + checkFirstDataLoaded(); + } + }; + useEffect(() => { - if (isVisible && isFirstDataRendered) { + if (isVisible && isFirstDataRendered && isFirstDataLoaded) { autoSizeAllColumns(); } - }, [isVisible, isFirstDataRendered]); + }, [isVisible, isFirstDataRendered, isFirstDataLoaded]); const getRowId = useCallback( (params: GetRowIdParams): string => { @@ -173,6 +204,7 @@ export function AgGridView({ onGridReady={handleGridReady} onFirstDataRendered={handleFirstDataRendered} onGridSizeChanged={handleGridSizeChanged} + onModelUpdated={handleModelUpdated} autoGroupColumnDef={autoGroupColumnDef} columnDefs={colDefs} dataTypeDefinitions={formatter.cellDataTypeDefinitions} diff --git a/plugins/tradingview-lightweight/.gitignore b/plugins/tradingview-lightweight/.gitignore new file mode 100644 index 000000000..054da38f1 --- /dev/null +++ b/plugins/tradingview-lightweight/.gitignore @@ -0,0 +1,10 @@ +build/ +dist/ +.venv/ +/venv +*.egg-info/ +.idea +.DS_store +__pycache__/ +docs/build/ +app.d/docs_examples/ diff --git a/plugins/tradingview-lightweight/AGENTS.md b/plugins/tradingview-lightweight/AGENTS.md new file mode 100644 index 000000000..3973aa0c4 --- /dev/null +++ b/plugins/tradingview-lightweight/AGENTS.md @@ -0,0 +1,434 @@ +# Local Development & Testing + +tradingview-lightweight charts I call tvl for short. + +## Quick Start + +Use the root `tools/plugin_builder.py` to build the JS bundle + Python wheel +and bring up a Deephaven server with the plugin installed. From the repo root: + +```bash +python tools/plugin_builder.py --js --reinstall --server tradingview-lightweight +``` + +The plugin name is a positional argument — there is no `--plugin` flag. +`--js` builds the JS bundle, `--reinstall` rebuilds and force-reinstalls the +wheel (needed when the version number hasn't changed), and `--server` (`-s`) +starts the Deephaven server. After code changes, re-run the same command to +rebuild and restart. Run `python tools/plugin_builder.py --help` for all flags. + +App.d fixtures for local dev/snapshot capture live under `app.d/` at the +plugin root (`disconnect_test.py`, `downsample_compare.py`). Point the +Deephaven server at this directory with `-Ddeephaven.application.dir=$(pwd)/app.d` +when you need the fixtures pre-loaded. + +## Testing with agent-browser + +After the server is running: + +```bash +# Open the IDE and set viewport +agent-browser open http://localhost:10000/ide/ +agent-browser wait --load networkidle +agent-browser wait 5000 +agent-browser set viewport 1920 1080 +``` + +### Running code in the DH console + +The DH IDE opens panels for **new variable assignments**, not bare expressions. +Typing `tvl_line` alone will not open a panel. You must assign to a new name: + +```bash +# 1. Find the console editor +agent-browser snapshot -i # look for the textbox ref +agent-browser click @e15 # click the editor (ref may vary) + +# 2. Type a variable assignment and execute +agent-browser keyboard type "my_chart = tvl_line" +agent-browser press Escape # dismiss autocomplete +agent-browser wait 100 +agent-browser press Enter # execute (Enter works; Ctrl+Enter may not) +agent-browser wait 5000 # wait for panel to open and data to load + +# 3. Move mouse away and screenshot +agent-browser mouse move 0 0 +agent-browser wait 500 +agent-browser screenshot /tmp/tvl_test.png +``` + +Multi-statement commands work the same way: + +```bash +agent-browser keyboard type "from deephaven.plot import tradingview_lightweight as tvl" +agent-browser press Escape && agent-browser wait 100 && agent-browser press Enter +agent-browser wait 2000 + +agent-browser keyboard type "my_by = tvl.line(_by_table, time='Timestamp', value='Price', by='Sym')" +agent-browser press Escape && agent-browser wait 100 && agent-browser press Enter +agent-browser wait 5000 +``` + +To verify a panel opened, check the Golden Layout tabs: + +```bash +agent-browser eval "JSON.stringify(Array.from(document.querySelectorAll('.lm_tab')).map(e => e.textContent).filter(t => t.length > 0))" +# Should include your variable name, e.g. ["Console","Log","Command History","File Explorer","my_chart"] +``` + +### Opening widgets from the Panels menu (alternative) + +The Panels menu lists all exported variables. This approach does NOT +reliably open widget panels — it works for tables but may silently +fail for plugin widgets. Prefer the console assignment method above. + +```bash +agent-browser snapshot -i -c +agent-browser click @e3 # "Panels" button — ref may vary +agent-browser wait 500 +agent-browser snapshot -i -c # find the search box ref +agent-browser fill @e7 "tvl_candlestick" +agent-browser wait 500 +agent-browser snapshot -i -c # find the button ref +agent-browser click @e11 # ref may vary +agent-browser wait 3000 +``` + +### Screenshotting an individual chart + +After opening a chart panel, isolate the chart and screenshot: + +```bash +# Move mouse off chart to avoid hover tooltips +agent-browser mouse move 0 0 +agent-browser wait 500 + +# Screenshot the full page (chart will be visible in the panel area) +agent-browser screenshot /tmp/tvl_chart.png + +# Or check for chart elements +agent-browser eval "document.querySelectorAll('.dh-tvl-chart').length" +``` + +View the screenshot with the `Read` tool on the image path. Always screenshot into the local directory so the user can also verify if needed. + +### Important Notes + +- **Refs change between snapshots.** Always run `agent-browser snapshot -i` before clicking to get fresh refs. +- **Charts render on canvas.** Use screenshots to verify chart content — DOM queries won't see rendered lines/candles. +- **Use variable assignments** to open widget panels (e.g., `my_chart = tvl_line`). Bare expressions evaluate but don't open panels. +- **Enter, not Ctrl+Enter.** Plain `Enter` executes single-line commands. `Ctrl+Enter` may insert a newline instead. +- **Dismiss autocomplete** with `agent-browser press Escape` before pressing Enter, otherwise it may select a completion instead of executing. +- **Zombie processes.** If the server seems stale (serving old JS), kill any + lingering `deephaven_server` processes (`pkill -f deephaven_server`) and + re-run `tools/plugin_builder.py`. + +## How It Works + +`tools/plugin_builder.py --js --reinstall --server tradingview-lightweight`: + +1. Builds the JS bundle via `npm run build` in `src/js/`. +2. Builds the Python wheel and installs it into a Deephaven server venv. +3. Starts the DH server on port 10000. + +The DH web client loads the JS plugin via `/js-plugins/manifest.json`. The +CJS bundle uses `require()` for modules the DH client provides (react, +`@deephaven/plugin`, etc.) — these are resolved by the client's built-in +require shim. + +## API Reference Notes + +For TradingView Lightweight Charts v5.2 API documentation, fetch the +upstream docs at https://tradingview.github.io/lightweight-charts/docs (the +local notes/api-reference/ snapshot has been removed to keep the plugin +tree clean). + +## Architecture Overview + +### JS Component Hierarchy + +``` +TradingViewPlugin (plugin registration) +├── component: TradingViewChart — for inline/embedded use +└── panelComponent: TradingViewChartPanel — for standalone panels + └── WidgetPanel (@deephaven/dashboard-core-plugins) + ├── Session disconnect/reconnect detection + ├── LoadingOverlay (spinner + error + disconnect) + └── TradingViewChart + ├── TradingViewChartModel — data pipeline, table subscriptions + └── TradingViewChartRenderer — LWC chart instance, series management +``` + +### Key Files + +| File | Role | +| --------------------------------------------- | ----------------------------------------------------------------- | +| `src/js/src/TradingViewChartPanel.tsx` | WidgetPanel wrapper — session disconnect, loading overlay | +| `src/js/src/TradingViewChart.tsx` | Main component — init, data updates, zoom/pan, downsample UX | +| `src/js/src/TradingViewChartModel.ts` | Model — widget messages, table subscriptions, autobin/EVENT | +| `src/js/src/TradingViewChartRenderer.ts` | LWC wrapper — chart creation, series CRUD, markers, price lines | +| `src/js/src/TradingViewEventPayload.ts` | Builds the press-event payload sent to Python (hit test, series) | +| `src/js/src/TradingViewChart.css` | Downsample scrim/status bar styles (inlined via `?inline` import) | +| `src/deephaven/.../auto_bin.py` | Server-side time-bin aggregation for Histogram/Candlestick/Bar | +| `src/deephaven/.../events.py` | Press-event payloads + handler plumbing (`wrap_callable`) | +| `src/deephaven/.../communication/listener.py` | Message handler — RETRIEVE/AUTOBIN_ZOOM/AUTOBIN_RESET/EVENT | + +### CSS Injection + +Plugin CSS files aren't loaded by the DH client. TVL uses Vite's `?inline` import to embed CSS as a string, injected via a ` + {/* lightweight-charts mounts into this absolutely-positioned host so its + explicitly-sized element stays out of the outer flex item's flow — + see chartHostRef. */} +
+ {pendingDs && ( + <> +
+
+
+ Downsampling data… +
+
+ + )} +
+ {debugInfo} +
+
+ ); +} + +export default TradingViewChart; diff --git a/plugins/tradingview-lightweight/src/js/src/TradingViewChartModel.ts b/plugins/tradingview-lightweight/src/js/src/TradingViewChartModel.ts new file mode 100644 index 000000000..32e84d699 --- /dev/null +++ b/plugins/tradingview-lightweight/src/js/src/TradingViewChartModel.ts @@ -0,0 +1,1579 @@ +import type { dh as DhType } from '@deephaven/jsapi-types'; +import Log from '@deephaven/log'; +import type { + AutoBinFigureMessage, + TvlAutoBinMeta, + TvlChartType, + TvlDownsampleMeta, + TvlFigureData, + TvlSeriesConfig, + ModelEvent, + ModelEventListener, + NewFigureMessage, +} from './TradingViewTypes'; +import { + getAllColumnsForTable, + convertTime, + unconvertTime, +} from './TradingViewUtils'; + +const log = Log.module('TradingViewChartModel'); + +const DOWNSAMPLE_THRESHOLD = 1000; + +/** + * Manages the data flow between Deephaven tables and the chart renderer. + * Uses table.subscribe() and ChartData for efficient delta-based updates, + * matching the pattern used by PlotlyExpressChartModel. + * + * Downsampling is performed entirely in JS via + * dh.plot.Downsample.runChartDownsample (same approach as plotly-express). + */ +class TradingViewChartModel { + private dh: typeof DhType; + + private widget: DhType.Widget; + + private listeners: Set = new Set(); + + private figureData: TvlFigureData | null = null; + + /** Tables currently subscribed to (may be original or downsampled). */ + private tables: Map = new Map(); + + /** + * Active subscriptions, keyed by tableId. All paths (downsample, + * autobin, and direct) use a full table.subscribe() — autobin tables + * are server-side scoped to a body+anchors aggregation that's already + * small enough to subscribe to wholesale. + */ + private tableSubscriptionMap: Map = + new Map(); + + /** ChartData objects that handle delta updates efficiently. */ + private chartDataMap: Map = new Map(); + + /** Full column data arrays, updated incrementally via ChartData. */ + private tableDataMap: Map> = new Map(); + + /** Cleanup functions for event listeners. */ + private subscriptionCleanupMap: Map void>> = new Map(); + + private widgetListenerCleanup: (() => void) | null = null; + + private revision = 0; + + /** Track whether initial data has loaded for fitContent. */ + private initialLoadComplete = false; + + /** Set to true when close() is called; prevents stale async callbacks. */ + private closed = false; + + /** Next table ID for dynamically added partition tables. */ + private nextTableId = 0; + + /** Per-template partition watcher state, keyed by template series id. */ + private partitionWatchers: Map< + string, + { + partitionedTable: DhType.PartitionedTable; + cleanup: () => void; + seenKeys: Set; + } + > = new Map(); + + /** IANA timezone string (e.g. "America/New_York") for time column conversion. */ + private timeZone = ''; + + getTimeZone(): string { + return this.timeZone; + } + + /** Chart type — determines whether time columns need TZ conversion. */ + private chartType: TvlChartType = 'standard'; + + // ---- JS-side downsample state ---- + + /** Original (full) tables stored for re-downsampling on zoom/pan. */ + private originalTableMap: Map = new Map(); + + /** Current live downsampled tables (replaced on each re-downsample). */ + private downsampledTableMap: Map = new Map(); + + /** Table IDs that are JS-downsampled. */ + private jsDownsampledTableIds: Set = new Set(); + + /** Metadata from Python about which tables are eligible. */ + private downsampleMeta: Record = {}; + + /** True while waiting for a downsample operation to complete. */ + pendingDownsample = false; + + /** If a new zoom was requested while waiting, store it here. */ + private pendingZoomParams: { + range: [number, number] | null; + width: number; + } | null = null; + + // ---- Server-side auto-bin state ---- + + /** Tables that were auto-binned server-side. */ + private autoBinnedTableIds: Set = new Set(); + + /** Per-table auto-bin metadata from the server. */ + private autoBinMeta: Record = {}; + + /** + * Per-table currently-scoped body range in UTC nanoseconds, or null when + * the full source is in use. Updated when AUTOBIN_ZOOM/RESET is sent so + * tests and debug overlays can read the current scope. + */ + private autoBinBodyRange: Record = {}; + + /** True while waiting for an AUTOBIN_FIGURE response from the server. */ + pendingAutoBin = false; + + /** + * True if the in-flight auto-bin request was triggered by a RESET + * (double-click), not a zoom/pan. The server's AUTOBIN_FIGURE response + * doesn't distinguish, so the model carries the flag forward to plumb + * isResetView into the resulting DATA_UPDATED event. + */ + private autoBinPendingIsReset = false; + + /** + * Monotonic counter incremented every time a resample request is issued + * (downsample or auto-bin). Tests use this to assert race-condition + * invariants ("N rapid zooms produce exactly N seq increments"). + */ + resampleSeq = 0; + + /** + * Monotonic counter incremented on every subscription update delivered to + * the chart, whatever the shape of the delta (added / modified / removed). + * + * This is the signal for "data is still flowing". Row counts are not: on a + * downsampled or auto-binned chart the rendered row count is capped by the + * target bin count, so it saturates and can even shrink as bins merge, and + * the rendered time extent only advances when a bin boundary is crossed + * (~100 source ticks on the ticking fixtures). Ticks that merely move a + * bin's extremes arrive as modifies, which move neither. + */ + dataUpdateSeq = 0; + + /** If a new auto-bin zoom was requested while pending, store it here. */ + private pendingAutoBinParams: { + range: [number, number] | null; + width?: number; + } | null = null; + + /** Tables that should trigger fitContent on next DATA_UPDATED. */ + private resetPendingForTable: Set = new Set(); + + /** + * Tables that have just been re-subscribed after a downsample and + * are awaiting their first DATA_UPDATED. Used to distinguish a + * bulk data swap from a normal tick update. + */ + private freshDownsampleTables: Set = new Set(); + + /** + * Table IDs whose current subscription has delivered at least one update, + * i.e. its initial Barrage snapshot has arrived. Cancelling a subscription + * (or releasing its table's export) before that point races the server's + * snapshot delivery: the stream gets errored, and the queued + * BarrageMessageProducer.propagateSnapshotForSubscription then logs + * "IllegalStateException: Stream was terminated by error". See + * retireSubscription. + */ + private deliveredTableIds: Set = new Set(); + + /** + * Settle callbacks for retirements that are waiting on their + * subscription's initial snapshot before releasing. Each entry removes + * itself when it settles (first update or timeout). + */ + private drainingRetirements: Set<() => void> = new Set(); + + /** + * Set by close(): release the widget once the last draining retirement + * settles (a widget close releases every export it owns at once, which + * must not race in-flight snapshots either). + */ + private widgetCloseWhenDrained = false; + + /** + * Upper bound on how long a retirement may wait for its snapshot. This is + * a stuck-subscription backstop, not an expected path: under load a fresh + * aggregation's snapshot can legitimately take many seconds, and releasing + * before it lands recreates the cancel-mid-snapshot race this machinery + * exists to avoid. Keep it generous. + */ + private static readonly RETIRE_TIMEOUT_MS = 60000; + + /** Debug callback for overlay. */ + private debugFn: ((msg: string) => void) | null = null; + + /** + * Set pendingDownsample and emit a DOWNSAMPLE_PENDING event + * so the view layer can show/hide the loading scrim. + */ + private setPendingDownsample(pending: boolean): void { + if (this.pendingDownsample === pending) return; + this.pendingDownsample = pending; + this.emit({ type: 'DOWNSAMPLE_PENDING', pending }); + } + + /** Same UX signal as setPendingDownsample but for the auto-bin path. */ + private setPendingAutoBin(pending: boolean): void { + if (this.pendingAutoBin === pending) return; + this.pendingAutoBin = pending; + this.emit({ type: 'DOWNSAMPLE_PENDING', pending }); + } + + /** + * Stable translator for value columns. ChartData caches per function + * identity, so this must be a fixed reference (not a new lambda per call). + */ + private readonly valueTranslator = TradingViewChartModel.unwrapValue; + + /** + * Stable translator for time columns. Produces TZ-adjusted epoch seconds + * directly, so the view layer never needs to call convertTime. + */ + private readonly timeTranslator = (val: unknown): unknown => { + const unwrapped = TradingViewChartModel.unwrapValue(val); + if (unwrapped == null || typeof unwrapped !== 'number') return 0; + // Numeric-scale charts (yieldCurve, options) use raw x values + if (this.chartType === 'yieldCurve' || this.chartType === 'options') { + return unwrapped; + } + // Standard charts: convert millis → TZ-adjusted epoch seconds + return convertTime(unwrapped, this.timeZone); + }; + + constructor(dh: typeof DhType, widget: DhType.Widget) { + this.dh = dh; + this.widget = widget; + } + + subscribe(listener: ModelEventListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + private emit(event: ModelEvent): void { + this.listeners.forEach(listener => { + try { + listener(event); + } catch (e) { + log.error('Error in model listener', e); + } + }); + } + + /** + * Set the timezone used for time column conversion. + * + * Called once before init with the user's Deephaven timezone setting, and + * again whenever that setting changes. Before init (no live subscriptions) + * the value is simply stored and picked up when tables are first + * subscribed. After init, every active table is re-subscribed so its time + * columns re-convert through timeTranslator in the new timezone — mirroring + * PlotlyExpressChartModel.fireTimeZoneUpdated, which re-subscribes tables + * on a timezone change rather than tearing the whole chart down. + */ + setTimeZone(tz: string): void { + const next = tz ?? ''; + if (next === this.timeZone) return; + this.timeZone = next; + if (this.tableSubscriptionMap.size === 0) return; + this.resubscribeForTimeZone(); + } + + /** + * Tear down and re-create every active table subscription so time columns + * are re-extracted through timeTranslator using the current timezone. Each + * table is flagged as a fresh data swap so the view replaces (rather than + * appends) its series data and can re-anchor the viewport. The currently + * subscribed table (original, downsampled, or auto-binned) is reused, so + * the existing downsample / auto-bin scope is preserved across the change. + */ + private resubscribeForTimeZone(): void { + const tableIds = Array.from(this.tableSubscriptionMap.keys()); + tableIds.forEach(tableId => { + const table = this.tables.get(tableId); + if (!table) return; + // Same table is re-subscribed below, so no table release here. + this.retireSubscription(tableId); + this.chartDataMap.delete(tableId); + this.tableDataMap.delete(tableId); + this.freshDownsampleTables.add(tableId); + this.subscribeTable(tableId, table); + }); + } + + /** + * Set the chart type (standard, yieldCurve, options). Determines whether + * time columns receive TZ conversion or are passed through as raw numbers. + */ + setChartType(ct: TvlChartType): void { + this.chartType = ct; + } + + /** + * Collect the set of column names that serve as time/x-axis columns + * for any series or marker spec on the given table. + */ + private getTimeColumnsForTable(tableId: number): Set { + const timeCols = new Set(); + this.figureData?.series.forEach(s => { + if (s.dataMapping.tableId === tableId) { + timeCols.add(s.dataMapping.columns.time); + } + if (s.markerSpec?.tableId === tableId && s.markerSpec.columns.time) { + timeCols.add(s.markerSpec.columns.time); + } + }); + return timeCols; + } + + /** + * Initialize the model with widget data from fetch(). + */ + async init( + exportedObjects: DhType.WidgetExportedObject[], + dataString: string + ): Promise { + const message: NewFigureMessage = JSON.parse(dataString); + + if (message.type !== 'NEW_FIGURE') { + log.error('Unexpected initial message type:', message.type); + return; + } + + this.figureData = message.figure; + this.revision = message.revision; + + // Read downsample metadata from Python + if (this.figureData.downsampleMeta) { + this.downsampleMeta = this.figureData.downsampleMeta; + } + // Read auto-bin metadata from Python + if (this.figureData.autoBinMeta) { + this.autoBinMeta = this.figureData.autoBinMeta; + Object.keys(this.autoBinMeta).forEach(refStr => { + this.autoBinnedTableIds.add(Number(refStr)); + }); + } + + // Collect partition refs and source table refs used only by partition + // templates. A large `by=` chart should not subscribe/downsample the raw + // source table directly; it only needs per-key constituent tables. + const partitionRefIndices = new Set(); + const partitionTemplateTableIds = new Set(); + const directSeriesTableIds = new Set(); + this.figureData.series.forEach(s => { + if (s.partition?.refIndex != null) { + partitionRefIndices.add(s.partition.refIndex); + } + if (s.partition != null) { + partitionTemplateTableIds.add(s.dataMapping.tableId); + } else { + directSeriesTableIds.add(s.dataMapping.tableId); + } + if (s.markerSpec?.tableId != null) { + directSeriesTableIds.add(s.markerSpec.tableId); + } + }); + + // Fetch all referenced tables (skip PartitionedTable refs) + const fetchedRefs = new Set(); + const tablePromises: Promise[] = []; + message.new_references.forEach(refIdx => { + if (partitionRefIndices.has(refIdx)) return; // handled below + if ( + partitionTemplateTableIds.has(refIdx) && + !directSeriesTableIds.has(refIdx) && + this.downsampleMeta[String(refIdx)] != null + ) { + return; + } + if (refIdx < exportedObjects.length) { + fetchedRefs.add(refIdx); + const exported = exportedObjects[refIdx]; + tablePromises.push( + exported.fetch().then((table: unknown) => { + this.tables.set(refIdx, table as DhType.Table); + }) + ); + } + }); + + // PartitionedTable refs are fetched by setupPartitionWatcher below. + partitionRefIndices.forEach(ref => { + if (ref < exportedObjects.length) fetchedRefs.add(ref); + }); + + // Close exports we never fetch (e.g. the raw source behind a large + // partition template): unfetched exports must be closed, per the + // WidgetExportedObject contract, or they pin server-side resources + // for the life of the widget. + TradingViewChartModel.closeExportedObjects( + exportedObjects, + ...Array.from(fetchedRefs) + ); + + await Promise.all(tablePromises); + + // Set nextTableId past the highest used ref index to avoid collisions + this.nextTableId = + message.new_references.length > 0 + ? Math.max(...message.new_references) + 1 + : 0; + + // Determine which tables need JS-side downsampling + const downsamplePromises: Promise[] = []; + this.tables.forEach((table, tableId) => { + const meta = this.downsampleMeta[String(tableId)]; + if (meta != null) { + // Store original for re-downsample on zoom/pan + this.originalTableMap.set(tableId, table); + this.jsDownsampledTableIds.add(tableId); + // Initial full-range downsample + downsamplePromises.push( + this.downsampleTable(tableId).catch(err => { + log.warn('Initial downsample failed for table', tableId, err); + // Fall back to subscribing to original table directly + this.jsDownsampledTableIds.delete(tableId); + this.originalTableMap.delete(tableId); + this.subscribeTable(tableId, table); + }) + ); + } else { + // Non-downsampled (including server-side autobin): subscribe directly. + // Autobin tables are scoped server-side to body + anchors so the + // full aggregation is small enough for a regular subscription. + this.subscribeTable(tableId, table); + } + }); + await Promise.all(downsamplePromises); + + // For each series that's a partition template, fetch its + // PartitionedTable and start watching for keys. + const partitionPromises: Promise[] = []; + this.figureData.series.forEach(template => { + const ref = template.partition?.refIndex; + if (ref != null && ref < exportedObjects.length) { + partitionPromises.push( + this.setupPartitionWatcher(template, exportedObjects[ref]) + ); + } + }); + await Promise.all(partitionPromises); + + // Listen for widget config updates + this.widgetListenerCleanup = this.listenToWidget(); + + // Emit initial figure config + this.emit({ + type: 'FIGURE_UPDATED', + figure: this.figureData, + tables: Array.from(this.tables.values()), + }); + } + + // ---- JS-side downsample API ---- + + /** Whether JS-side downsampling is active for any table. */ + isDownsampled(): boolean { + return this.jsDownsampledTableIds.size > 0; + } + + /** Whether server-side auto-bin is active for any table. */ + isAutoBinned(): boolean { + return this.autoBinnedTableIds.size > 0; + } + + /** Whether any resampling path is active (downsample or auto-bin). */ + isResampling(): boolean { + return this.isDownsampled() || this.isAutoBinned(); + } + + /** + * True when nothing is in flight server-side on this chart's behalf: no + * resample pending or queued, no retirement draining, and every active + * subscription has delivered its initial snapshot. + * + * This is the "safe to tear the page down" signal. A client that vanishes + * while a snapshot is propagating makes the server race its own cleanup + * and log "Stream was terminated by error" — noise that lands in the + * console history and can bleed into unrelated tests' screenshots. Tests + * should wait for quiescence (via the `quiescent` field of the + * data-tvl-state attribute) before ending. + */ + isQuiescent(): boolean { + if ( + this.pendingDownsample || + this.pendingAutoBin || + this.pendingZoomParams != null || + this.pendingAutoBinParams != null || + this.drainingRetirements.size > 0 + ) { + return false; + } + return Array.from(this.tableSubscriptionMap.keys()).every(tableId => + this.deliveredTableIds.has(tableId) + ); + } + + /** Get the auto-bin metadata from Python. */ + getAutoBinMeta(): Record { + return this.autoBinMeta; + } + + /** + * Currently-scoped body range (UTC ns) for the given auto-binned table, + * or null when at full source. Returns null when the table is unknown. + */ + getAutoBinBodyRange(tableRef: number): [number, number] | null { + return this.autoBinBodyRange[String(tableRef)] ?? null; + } + + /** Get the downsample metadata from Python. */ + getDownsampleMeta(): Record { + return this.downsampleMeta; + } + + /** Set debug callback for overlay output. */ + setDebugFn(fn: (msg: string) => void): void { + this.debugFn = fn; + } + + private dbg(msg: string): void { + this.debugFn?.(msg); + } + + /** + * Downsample a single table using dh.plot.Downsample.runChartDownsample. + * Closes old downsampled table, installs new one, subscribes. + * + * @param tableId The table ID to downsample + * @param range Optional [fromSec, toSec] in TZ-shifted epoch seconds. Null = full range. + * @param width Optional chart width in pixels for target output size. + * @param isReset True if this is a reset (double-click) — triggers fitContent on data arrival. + */ + private async downsampleTable( + tableId: number, + range?: [number, number] | null, + width?: number, + isReset = false + ): Promise { + const meta = this.downsampleMeta[String(tableId)]; + if (meta == null) return; + + const originalTable = this.originalTableMap.get(tableId); + if (!originalTable) return; + + // Convert TZ-shifted seconds to DateWrapper range + let dsRange: DhType.DateWrapper[] | undefined; + if (range != null) { + const fromUtcSec = unconvertTime(range[0], this.timeZone); + const toUtcSec = unconvertTime(range[1], this.timeZone); + dsRange = [ + this.dh.DateWrapper.ofJsDate(new Date(fromUtcSec * 1000)), + this.dh.DateWrapper.ofJsDate(new Date(toUtcSec * 1000)), + ]; + } + + const targetWidth = width ?? 1000; + + this.dbg( + `downsampleTable tid=${tableId} range=${ + range ? `[${range[0]},${range[1]}]` : 'null' + } w=${targetWidth} reset=${isReset}` + ); + + const newTable = await this.dh.plot.Downsample.runChartDownsample( + originalTable, + meta.timeCol, + meta.valueCols, + targetWidth, + dsRange + ); + + if (this.closed) { + try { + newTable.close(); + } catch { + // ignore + } + return; + } + + this.dbg(`downsampleTable tid=${tableId} result: ${newTable.size} rows`); + + // Retire the old subscription FIRST — this prevents the old + // subscription from firing ticks that consume the reset flag. The old + // downsampled table is a client-created export (runChartDownsample), so + // the client must close it — but only once its snapshot has landed; + // retireSubscription defers the release when necessary. + const oldDs = this.downsampledTableMap.get(tableId); + this.downsampledTableMap.delete(tableId); + this.retireSubscription(tableId, oldDs); + this.chartDataMap.delete(tableId); + this.tableDataMap.delete(tableId); + + // Install new downsampled table + this.downsampledTableMap.set(tableId, newTable); + this.tables.set(tableId, newTable); + + // Set flags AFTER cleanup, BEFORE subscribe — race-free. + // The old subscription is gone, so it can't consume these. + this.freshDownsampleTables.add(tableId); + if (isReset) { + this.resetPendingForTable.add(tableId); + } + + // Subscribe — first EVENT_UPDATED will see the flags above + this.subscribeTable(tableId, newTable); + } + + /** + * Perform a downsample operation for all JS-downsampled tables. + * Called by the view on zoom/pan/reset. + * + * @param range [fromSec, toSec] in TZ-shifted epoch seconds, or null for full range (reset). + * @param width Chart width in pixels. + */ + async performDownsample( + range: [number, number] | null, + width: number + ): Promise { + if (!this.isDownsampled()) return; + + // Queue if already pending + if (this.pendingDownsample) { + this.pendingZoomParams = { range, width }; + return; + } + + this.setPendingDownsample(true); + this.resampleSeq += 1; + + const isReset = range == null; + + try { + const promises: Promise[] = []; + this.jsDownsampledTableIds.forEach(tableId => { + promises.push( + this.downsampleTable(tableId, range, width, isReset).catch(err => { + log.warn('Re-downsample failed for table', tableId, err); + // On failure, fall back to original table + const orig = this.originalTableMap.get(tableId); + if (orig) { + const oldDs = this.downsampledTableMap.get(tableId); + this.downsampledTableMap.delete(tableId); + this.retireSubscription(tableId, oldDs); + this.chartDataMap.delete(tableId); + this.tableDataMap.delete(tableId); + this.tables.set(tableId, orig); + this.subscribeTable(tableId, orig); + } + }) + ); + }); + await Promise.all(promises); + } finally { + this.setPendingDownsample(false); + } + + // Drain pending queue + if (this.pendingZoomParams != null) { + const p = this.pendingZoomParams; + this.pendingZoomParams = null; + this.performDownsample(p.range, p.width); + } + } + + // ---- Server-side auto-bin API ---- + + /** + * Request a re-aggregation for the visible range. Sends AUTOBIN_ZOOM + * (or AUTOBIN_RESET if range is null) to the server. The server + * responds asynchronously with AUTOBIN_FIGURE which is handled in + * listenToWidget. + */ + performAutoBin(range: [number, number] | null, width?: number): void { + if (!this.isAutoBinned()) return; + + if (this.pendingAutoBin) { + this.pendingAutoBinParams = { range, width }; + return; + } + + this.setPendingAutoBin(true); + this.resampleSeq += 1; + this.autoBinPendingIsReset = range == null; + + // Round chart width up to the nearest 1000 px so the server's derived + // bin width lands on a small set of common values across sessions — + // makes the engine's `upperBin(time, w)` results cache-friendly. The + // raw width is sent alongside as `actualWidthPx` so the server can + // floor `target_bins` to keep each bar at least MIN_BAR_PX wide, + // regardless of how much the rounding overshoots. + const actualWidthPx = + width != null && width > 0 ? Math.round(width) : undefined; + const widthPx = + actualWidthPx != null + ? Math.max(1000, Math.ceil(actualWidthPx / 1000) * 1000) + : undefined; + + this.autoBinnedTableIds.forEach(tableRef => { + if (range == null) { + this.autoBinBodyRange[String(tableRef)] = null; + this.sendWidgetMessage({ + type: 'AUTOBIN_RESET', + tableRef, + widthPx, + actualWidthPx, + }); + return; + } + // Range comes in as TZ-shifted epoch seconds (matching the chart's + // visible range). Convert to UTC nanoseconds for the server. + const fromUtcSec = unconvertTime(range[0], this.timeZone); + const toUtcSec = unconvertTime(range[1], this.timeZone); + const fromNs = Math.floor(fromUtcSec * 1e9); + const toNs = Math.floor(toUtcSec * 1e9); + // atLiveEdge: visible range's right edge is at or past the source's + // full extent. Server then extends the body's right bound past the + // tail anchor so live ticks land in the body's last bin. + const meta = this.autoBinMeta[String(tableRef)]; + const atLiveEdge = meta != null && toNs >= meta.fullRangeNs[1]; + this.autoBinBodyRange[String(tableRef)] = [fromNs, toNs]; + this.sendWidgetMessage({ + type: 'AUTOBIN_ZOOM', + tableRef, + fromNs, + toNs, + widthPx, + actualWidthPx, + atLiveEdge, + }); + }); + } + + /** Unified resample router: dispatches to downsample and auto-bin paths. */ + performResample(range: [number, number] | null, width: number): void { + if (this.isDownsampled()) { + this.performDownsample(range, width).catch(err => { + log.warn('performDownsample failed', err); + }); + } + if (this.isAutoBinned()) { + this.performAutoBin(range, width); + } + } + + /** + * Send a user event (press / doublePress) back to Python via the widget + * channel. Fire-and-forget; the server produces no client response. + */ + sendEvent(handler: string, payload: unknown): void { + this.sendWidgetMessage({ type: 'EVENT', handler, payload }); + } + + /** Handler ids advertised by the figure (subset of press / doublePress). */ + getEnabledHandlers(): string[] { + return this.figureData?.enabledHandlers ?? []; + } + + private sendWidgetMessage(msg: Record): void { + try { + this.widget.sendMessage(JSON.stringify(msg), []); + } catch (e) { + log.error('Failed to send widget message', msg.type, e); + this.setPendingAutoBin(false); + } + } + + /** Handle an AUTOBIN_FIGURE message from the server. */ + private async handleAutoBinFigure( + msg: AutoBinFigureMessage, + exportedObjects: DhType.WidgetExportedObject[] + ): Promise { + try { + // Update meta first so the renderer reflects the new bin width. + this.autoBinMeta = msg.autoBinMeta; + if (this.figureData) { + this.figureData.autoBinMeta = msg.autoBinMeta; + } + this.revision = msg.revision; + + if (msg.noop === true || msg.new_references.length === 0) { + TradingViewChartModel.closeExportedObjects(exportedObjects); + this.setPendingAutoBin(false); + this.drainPendingAutoBin(); + return; + } + + // Fetch the swapped-in aggregated table for the affected ref. + const { tableRef } = msg; + if (tableRef >= exportedObjects.length) { + log.warn( + 'AUTOBIN_FIGURE tableRef out of range', + tableRef, + exportedObjects.length + ); + TradingViewChartModel.closeExportedObjects(exportedObjects); + this.setPendingAutoBin(false); + this.drainPendingAutoBin(); + return; + } + + // The server re-exports every table with each AUTOBIN_FIGURE, but only + // the swapped aggregation is fetched. Unfetched exports must be closed + // (per WidgetExportedObject docs) or each zoom leaks live exports + // server-side for the rest of the widget's life. + TradingViewChartModel.closeExportedObjects(exportedObjects, tableRef); + + const newTable = (await exportedObjects[ + tableRef + ].fetch()) as DhType.Table; + if (this.closed) { + try { + newTable.close(); + } catch { + // ignore + } + return; + } + + // Retire the previous aggregation: its subscription is released once + // its in-flight snapshot (if any) lands, and its export with it. A + // superseded aggregation has no other owner, so releasing it promptly + // keeps zoom churn from pinning dead aggregations server-side. + const oldTable = this.tables.get(tableRef); + this.retireSubscription( + tableRef, + oldTable !== newTable ? oldTable : undefined + ); + this.chartDataMap.delete(tableRef); + this.tableDataMap.delete(tableRef); + + this.tables.set(tableRef, newTable); + this.freshDownsampleTables.add(tableRef); + if (this.autoBinPendingIsReset) { + this.resetPendingForTable.add(tableRef); + } + // Server-side scoped: subscribe to the entire (small) agg table. + this.subscribeTable(tableRef, newTable); + } catch (err) { + log.error('Error handling AUTOBIN_FIGURE', err); + } finally { + this.setPendingAutoBin(false); + this.drainPendingAutoBin(); + } + } + + private drainPendingAutoBin(): void { + if (this.pendingAutoBinParams != null) { + const p = this.pendingAutoBinParams; + this.pendingAutoBinParams = null; + this.performAutoBin(p.range, p.width); + } + } + + // ---- Partition handling ---- + + /** + * Add a single partition key for a given template series: fetch its + * constituent table, subscribe, clone the template into a runtime + * series, and push it to the figure. + */ + private async addPartitionKey( + pt: DhType.PartitionedTable, + key: unknown, + template: TvlSeriesConfig + ): Promise { + const watcher = this.partitionWatchers.get(template.id); + if (!watcher) return; + const keyStr = String(key); + if (watcher.seenKeys.has(keyStr)) { + return; // Duplicate key — already added + } + watcher.seenKeys.add(keyStr); + + const table = await pt.getTable(key as object); + if (table == null) { + log.warn('getTable returned null for key:', key); + watcher.seenKeys.delete(keyStr); + return; + } + + const newTableId = this.nextTableId; + this.nextTableId += 1; + this.tables.set(newTableId, table as DhType.Table); + + // Clone the template into a runtime series. Preserve options, + // type, paneIndex, priceScaleOptions, columns; give it a unique id + // and a per-key title. + const newSeries: TvlSeriesConfig = { + id: `${template.id}_${keyStr}`, + type: template.type, + options: { ...(template.options ?? {}), title: keyStr }, + dataMapping: { + tableId: newTableId, + columns: { ...template.dataMapping.columns }, + }, + paneIndex: template.paneIndex, + priceScaleOptions: template.priceScaleOptions, + }; + + // Push the series config BEFORE subscribing so that + // getAllColumnsForTable() can find the column names for this tableId. + if (this.figureData) { + this.figureData.series.push(newSeries); + log.debug( + 'Added series for key:', + keyStr, + 'template:', + template.id, + 'total:', + this.figureData.series.length + ); + } + + if (this.shouldDownsamplePartitionTable(template, table as DhType.Table)) { + this.addPartitionDownsampleMeta( + newTableId, + template, + table as DhType.Table + ); + try { + await this.downsampleTable(newTableId); + } catch (err) { + log.warn( + 'Initial partition downsample failed for table', + newTableId, + err + ); + this.removePartitionDownsampleMeta(newTableId); + this.tables.set(newTableId, table as DhType.Table); + this.subscribeTable(newTableId, table as DhType.Table); + } + } else { + this.subscribeTable(newTableId, table as DhType.Table); + } + } + + private shouldDownsamplePartitionTable( + template: TvlSeriesConfig, + table: DhType.Table + ): boolean { + if (this.downsampleMeta[String(template.dataMapping.tableId)] == null) { + return false; + } + const size = + typeof table.size === 'number' + ? table.size + : this.downsampleMeta[String(template.dataMapping.tableId)].tableSize; + return size > DOWNSAMPLE_THRESHOLD; + } + + private addPartitionDownsampleMeta( + tableId: number, + template: TvlSeriesConfig, + table: DhType.Table + ): void { + const sourceMeta = + this.downsampleMeta[String(template.dataMapping.tableId)]; + if (sourceMeta == null) return; + const meta = { + ...sourceMeta, + tableSize: + typeof table.size === 'number' ? table.size : sourceMeta.tableSize, + }; + this.downsampleMeta[String(tableId)] = meta; + if (this.figureData != null) { + this.figureData.downsampleMeta = { + ...(this.figureData.downsampleMeta ?? {}), + [String(tableId)]: meta, + }; + } + this.originalTableMap.set(tableId, table); + this.jsDownsampledTableIds.add(tableId); + } + + private removePartitionDownsampleMeta(tableId: number): void { + this.jsDownsampledTableIds.delete(tableId); + this.originalTableMap.delete(tableId); + delete this.downsampleMeta[String(tableId)]; + if (this.figureData?.downsampleMeta) { + delete this.figureData.downsampleMeta[String(tableId)]; + } + } + + /** + * Fetch the PartitionedTable for a template series, discover all + * existing keys, subscribe to each, and listen for new keys. + * + * Per the DH JSAPI contract, the keyadded listener must be attached + * BEFORE :meth:`getKeys` is read so no keys are missed between snapshot + * and listener-attach. The per-template seenKeys dedup handles the + * overlap between the listener firing and the initial getKeys() sweep. + */ + private async setupPartitionWatcher( + template: TvlSeriesConfig, + exported: DhType.WidgetExportedObject + ): Promise { + try { + const pt = (await exported.fetch()) as DhType.PartitionedTable; + + // Resolve the keyadded event name from the DH namespace; fall back to + // the literal string if the constant isn't surfaced. + let eventName = 'keyadded'; + try { + const dhPT = this.dh.PartitionedTable; + if (dhPT?.EVENT_KEYADDED != null) { + eventName = dhPT.EVENT_KEYADDED; + } + } catch { + // PartitionedTable not on dh namespace, use string fallback + } + + // Pre-register the watcher entry so addPartitionKey can dedup. + this.partitionWatchers.set(template.id, { + partitionedTable: pt, + cleanup: () => { + /* no-op placeholder; replaced once the keyadded listener attaches */ + }, + seenKeys: new Set(), + }); + + // Attach the listener FIRST so any keys delivered between fetch() and + // our getKeys() sweep are still picked up. + const cleanup = pt.addEventListener( + eventName, + async (event: DhType.Event) => { + try { + await this.addPartitionKey(pt, event.detail, template); + if (this.figureData) { + this.emit({ + type: 'FIGURE_UPDATED', + figure: this.figureData, + tables: Array.from(this.tables.values()), + }); + } + } catch (err) { + log.error('Error handling new partition key', err); + } + } + ); + const entry = this.partitionWatchers.get(template.id); + if (entry) entry.cleanup = cleanup; + + // Discover existing keys. + const rawKeys: unknown = pt.getKeys(); + const existingKeys = + rawKeys != null && + typeof (rawKeys as Promise).then === 'function' + ? ((await rawKeys) as Set | null | undefined) + : (rawKeys as Set | null | undefined); + const initialCount = existingKeys?.size ?? 0; + log.debug( + 'Existing partition keys for template', + template.id, + ':', + initialCount + ); + if (existingKeys && initialCount > 0) { + const keyPromises: Promise[] = []; + existingKeys.forEach((key: unknown) => { + keyPromises.push(this.addPartitionKey(pt, key, template)); + }); + await Promise.all(keyPromises); + if (this.figureData) { + this.emit({ + type: 'FIGURE_UPDATED', + figure: this.figureData, + tables: Array.from(this.tables.values()), + }); + } + } + + log.debug( + 'Partition watcher set up for', + template.id, + 'with', + initialCount, + 'initial keys' + ); + } catch (err) { + log.error( + 'Failed to set up partition watcher for template', + template.id, + err + ); + this.emit({ + type: 'ERROR', + message: `Partition watcher failed: ${String(err)}`, + }); + } + } + + // ---- Subscription ---- + + /** + * Clean up subscriptions and event listeners for a specific table. + */ + /** + * Retire a table slot's subscription, and optionally a client-owned table + * export, without cancelling a Barrage snapshot that is still in flight. + * + * The server assembles and propagates an initial snapshot for every new + * subscription. If the client cancels the subscription (sub.close()) or + * releases the table's export (table.close()) before that snapshot has + * been delivered, the server errors the stream and the queued snapshot + * delivery throws "IllegalStateException: Stream was terminated by error" + * (BarrageMessageProducer.propagateSnapshotForSubscription). Under zoom + * churn, swap N+1 regularly tears down swap N's table while N's snapshot + * is in flight, so this happens with a live, well-behaved client. + * + * The subscription is detached from the model immediately (its listeners + * are removed, so the replacement slot owner takes over cleanly), but the + * actual release is deferred until the subscription's first update + * arrives — proof the snapshot has been delivered — or RETIRE_TIMEOUT_MS + * passes. + * + * @param tableId The table slot being replaced or torn down + * @param tableToClose A client-owned table to release along with the + * subscription: runChartDownsample results and superseded auto-bin + * aggregations (both are exports the widget close does not cover, or + * that would otherwise accumulate server-side for the widget's life). + * Leave undefined for tables that should outlive the subscription. + */ + private retireSubscription( + tableId: number, + tableToClose?: DhType.Table + ): void { + const cleanupSet = this.subscriptionCleanupMap.get(tableId); + if (cleanupSet) { + cleanupSet.forEach(cleanup => cleanup()); + this.subscriptionCleanupMap.delete(tableId); + } + const sub = this.tableSubscriptionMap.get(tableId); + this.tableSubscriptionMap.delete(tableId); + const delivered = this.deliveredTableIds.has(tableId); + this.deliveredTableIds.delete(tableId); + + const release = (): void => { + try { + sub?.close(); + } catch { + // ignore + } + try { + tableToClose?.close(); + } catch { + // ignore + } + }; + + if (sub == null || delivered) { + release(); + return; + } + + // Initial snapshot still in flight: release on first update or timeout. + let removeListener: (() => void) | null = null; + let timer: ReturnType | null = null; + const settle = (): void => { + if (!this.drainingRetirements.delete(settle)) return; + removeListener?.(); + if (timer != null) clearTimeout(timer); + release(); + this.maybeCloseWidget(); + // Quiescence may have just been reached with no DATA_UPDATED to + // follow (static tables); poke listeners so the view refreshes + // data-tvl-state and tests polling isQuiescent() see it. + this.emit({ type: 'RETIREMENT_DRAINED' }); + }; + this.drainingRetirements.add(settle); + removeListener = sub.addEventListener(this.dh.Table.EVENT_UPDATED, settle); + timer = setTimeout(settle, TradingViewChartModel.RETIRE_TIMEOUT_MS); + } + + /** + * Close widget-message exported objects that will not be fetched. Per the + * WidgetExportedObject contract, an export that is never fetched must be + * closed, or its server-side resources live until the widget closes. + * Closing an unfetched export is always safe — nothing is subscribed to it. + * + * @param exportedObjects The message's exported objects + * @param keepIndexes Indexes that will be fetched and must not be closed + */ + private static closeExportedObjects( + exportedObjects: DhType.WidgetExportedObject[], + ...keepIndexes: number[] + ): void { + exportedObjects.forEach((exported, i) => { + if (keepIndexes.includes(i)) return; + try { + exported.close(); + } catch { + // ignore + } + }); + } + + /** + * Release the widget if close() has run and no retirement is still + * draining. Closing the widget releases every export it owns in one + * server-side sweep, so it must wait for in-flight snapshots too. + */ + private maybeCloseWidget(): void { + if (!this.widgetCloseWhenDrained || this.drainingRetirements.size > 0) { + return; + } + this.widgetCloseWhenDrained = false; + try { + this.widget.close(); + } catch { + // ignore + } + } + + /** + * Subscribe to a table using full table.subscribe() with ChartData + * for delta updates. All tables (both original and downsampled) use + * this path — downsampled tables are small enough for full subscribe. + */ + private subscribeTable(tableId: number, table: DhType.Table): void { + if (!this.figureData) return; + + const columnNames = getAllColumnsForTable(this.figureData.series, tableId); + const columns = table.columns.filter((col: DhType.Column) => + columnNames.includes(col.name) + ); + if (columns.length === 0) return; + + let cleanupSet = this.subscriptionCleanupMap.get(tableId); + if (cleanupSet == null) { + cleanupSet = new Set(); + this.subscriptionCleanupMap.set(tableId, cleanupSet); + } + + // Full subscription with ChartData for delta updates + if (this.tableSubscriptionMap.has(tableId)) return; + + this.chartDataMap.set(tableId, new this.dh.plot.ChartData(table)); + this.tableDataMap.set(tableId, {}); + + const subscription = table.subscribe(columns); + this.tableSubscriptionMap.set(tableId, subscription); + + cleanupSet.add( + subscription.addEventListener( + this.dh.Table.EVENT_UPDATED, + e => { + this.handleTableUpdate(e, tableId); + } + ) + ); + + // Listen for table disconnect / reconnect + cleanupSet.add( + table.addEventListener(this.dh.Table.EVENT_DISCONNECT, () => { + log.warn('Table disconnected:', tableId); + this.emit({ type: 'DISCONNECTED', connected: false }); + }) + ); + cleanupSet.add( + table.addEventListener(this.dh.Table.EVENT_RECONNECT, () => { + log.info('Table reconnected:', tableId); + this.emit({ type: 'DISCONNECTED', connected: true }); + }) + ); + } + + // ---- Data update handler ---- + + /** + * Handle subscription update for a table. + * Uses ChartData for delta processing; emits incremental info. + */ + private handleTableUpdate( + event: DhType.Event, + tableId: number + ): void { + // First update == the subscription's initial snapshot has been + // delivered, so it is now safe to cancel/release (see retireSubscription). + this.deliveredTableIds.add(tableId); + this.dataUpdateSeq += 1; + + const chartData = this.chartDataMap.get(tableId); + const tableData = this.tableDataMap.get(tableId); + + if (chartData == null || tableData == null) { + log.warn('No chartData/tableData for table', tableId); + return; + } + + const { detail: updateEvent } = event; + + // Apply delta to ChartData + chartData.update(updateEvent); + + // Extract full column arrays via translators (stable refs for caching) + const timeCols = this.getTimeColumnsForTable(tableId); + updateEvent.columns.forEach((column: DhType.Column) => { + const translator = timeCols.has(column.name) + ? this.timeTranslator + : this.valueTranslator; + tableData[column.name] = chartData.getColumn( + column.name, + translator, + updateEvent + ); + }); + + const isFirstLoad = !this.initialLoadComplete; + if (isFirstLoad) { + this.initialLoadComplete = true; + } + + // Check if this is the first data from a fresh downsample + const isDownsampleSwap = this.freshDownsampleTables.has(tableId); + if (isDownsampleSwap) { + this.freshDownsampleTables.delete(tableId); + } + + // Check if this table has a pending reset (from double-click) + const isResetView = this.resetPendingForTable.has(tableId); + if (isResetView) { + this.resetPendingForTable.delete(tableId); + } + + const addedCount = updateEvent.added != null ? updateEvent.added.size : 0; + const removedCount = + updateEvent.removed != null ? updateEvent.removed.size : 0; + const modifiedCount = + updateEvent.modified != null ? updateEvent.modified.size : 0; + + this.emit({ + type: 'DATA_UPDATED', + tableId, + isInitialLoad: isFirstLoad, + addedCount, + removedCount, + modifiedCount, + isResetView, + isDownsampleSwap, + }); + } + + // ---- Widget & utility methods ---- + + private listenToWidget(): () => void { + const handler = ( + event: DhType.Event + ): void => { + try { + const data = event.detail; + const dataStr = data.getDataAsString(); + const msg = JSON.parse(dataStr); + this.dbg(`widget msg: type=${msg.type}`); + + if (msg.type === 'AUTOBIN_FIGURE') { + const exported = data.exportedObjects ?? []; + this.handleAutoBinFigure(msg as AutoBinFigureMessage, exported).catch( + err => log.error('handleAutoBinFigure failed', err) + ); + return; + } + + // Nothing is fetched from other message types, so release any + // exports they carry (unfetched exports must be closed, per the + // WidgetExportedObject contract). + TradingViewChartModel.closeExportedObjects(data.exportedObjects ?? []); + + if (msg.type === 'NEW_FIGURE' && msg.revision > this.revision) { + this.revision = msg.revision; + this.figureData = msg.figure; + + this.emit({ + type: 'FIGURE_UPDATED', + figure: msg.figure, + tables: Array.from(this.tables.values()), + }); + } + } catch (e) { + log.error('Error processing widget message', e); + } + }; + + this.widget.addEventListener(this.dh.Widget.EVENT_MESSAGE, handler); + + // Detect widget close (server disconnect / variable removed) + const closeHandler = (): void => { + log.warn('Widget closed'); + this.emit({ type: 'DISCONNECTED', connected: false }); + }; + this.widget.addEventListener(this.dh.Widget.EVENT_CLOSE, closeHandler); + + return () => { + this.widget.removeEventListener(this.dh.Widget.EVENT_MESSAGE, handler); + this.widget.removeEventListener(this.dh.Widget.EVENT_CLOSE, closeHandler); + }; + } + + /** + * Unwrap Deephaven wrapper types to plain JS values. + * DateWrapper -> epoch millis via asDate().getTime() + * LongWrapper -> number via asNumber() + */ + private static unwrapValue(val: unknown): unknown { + if (val == null) return val; + if (typeof val !== 'object') return val; + + const asDate = val as { asDate?: () => Date }; + if (typeof asDate.asDate === 'function') { + return asDate.asDate().getTime(); + } + + const asNum = val as { asNumber?: () => number }; + if (typeof asNum.asNumber === 'function') { + return asNum.asNumber(); + } + + return val; + } + + /** + * Whether the chart is "ready" — i.e. every series currently in + * ``figureData`` has at least one row of data in :attr:`tableDataMap`. + * + * For non-partitioned charts this becomes true on the first DATA_UPDATED + * after model.init. For ``by``-partitioned charts it additionally + * requires at least one runtime partition series to have been discovered. + * The image-snapshotter polls this signal to know when to take a stable + * screenshot without resorting to hard-coded waits. + */ + isReady(): boolean { + if (this.figureData == null) return false; + // A partitioned chart with no keys discovered yet should NOT be + // considered ready — there's nothing to render. + const renderableSeries = this.figureData.series.filter( + series => series.partition == null + ); + if (renderableSeries.length === 0) return false; + return renderableSeries.every(series => { + const { tableId } = series.dataMapping; + const tableData = this.tableDataMap.get(tableId); + if (!tableData) return false; + const { time: timeColName } = series.dataMapping.columns; + const timeCol = tableData[timeColName]; + if (timeCol != null && timeCol.length > 0) return true; + // No rows arrived yet. Distinguish "still waiting for first data" + // from "source table is intentionally empty" (e.g. a `where(...)` + // that filters everything out — used by the pane_preserve_empty + // docs example). An empty source reports size === 0 immediately + // after fetch, so treat that as ready instead of hanging forever. + const table = this.tables.get(tableId); + return table != null && table.size === 0; + }); + } + + getFigureData(): TvlFigureData | null { + return this.figureData; + } + + getColumnData(tableId: number): Map | undefined { + const tableData = this.tableDataMap.get(tableId); + if (!tableData) return undefined; + // Convert Record to Map for backward compat with TradingViewUtils + const map = new Map(); + Object.entries(tableData).forEach(([key, val]) => { + map.set(key, val); + }); + return map; + } + + getSeriesConfigs(): TvlSeriesConfig[] { + return this.figureData?.series ?? []; + } + + /** Get the set of table IDs that are JS-downsampled. */ + getDownsampledTableIds(): Set { + return this.jsDownsampledTableIds; + } + + /** Get a table by ID. */ + getTable(tableId: number): DhType.Table | undefined { + return this.tables.get(tableId); + } + + close(): void { + this.closed = true; + + // Retire every active subscription. Client-created downsample tables are + // released with their subscription; every other table is released by the + // widget close below. Retirement defers any release whose initial + // snapshot is still in flight (see retireSubscription), and the widget + // close waits for those retirements to drain. + Array.from(this.tableSubscriptionMap.keys()).forEach(tableId => { + this.retireSubscription(tableId, this.downsampledTableMap.get(tableId)); + }); + this.subscriptionCleanupMap.clear(); + this.tableSubscriptionMap.clear(); + this.deliveredTableIds.clear(); + + this.downsampledTableMap.clear(); + this.originalTableMap.clear(); + this.tables.clear(); + + // Clean up widget listener + if (this.widgetListenerCleanup) { + this.widgetListenerCleanup(); + this.widgetListenerCleanup = null; + } + + // Clean up all per-template partition watchers + this.partitionWatchers.forEach(({ partitionedTable, cleanup }) => { + try { + cleanup(); + } catch { + // ignore + } + if (partitionedTable?.close != null) { + try { + partitionedTable.close(); + } catch { + // ignore + } + } + }); + this.partitionWatchers.clear(); + + this.listeners.clear(); + this.chartDataMap.clear(); + this.tableDataMap.clear(); + this.resetPendingForTable.clear(); + this.freshDownsampleTables.clear(); + this.jsDownsampledTableIds.clear(); + this.autoBinnedTableIds.clear(); + + // Release the widget, which releases all of its exported tables. tvl never + // did this, so every reconnect (connectModel re-fetches) abandoned the + // previous widget and its exports. plotly-express closes the widget in + // close()/unsubscribe() and re-fetches on the next subscribe. Deferred + // until draining retirements settle so the mass export release can't + // cancel a snapshot that is still propagating. + this.widgetCloseWhenDrained = true; + this.maybeCloseWidget(); + } +} + +export default TradingViewChartModel; diff --git a/plugins/tradingview-lightweight/src/js/src/TradingViewChartPanel.tsx b/plugins/tradingview-lightweight/src/js/src/TradingViewChartPanel.tsx new file mode 100644 index 000000000..8cd3c50f8 --- /dev/null +++ b/plugins/tradingview-lightweight/src/js/src/TradingViewChartPanel.tsx @@ -0,0 +1,77 @@ +import React, { useCallback, useState } from 'react'; +import type { dh as DhType } from '@deephaven/jsapi-types'; +import { type WidgetPanelProps } from '@deephaven/plugin'; +import { WidgetPanel } from '@deephaven/dashboard-core-plugins'; +import TradingViewChart from './TradingViewChart'; + +/** + * Panel wrapper for TradingViewChart. + * + * Wraps the chart in WidgetPanel to get standard DH panel features: + * - Session-level disconnect/reconnect detection + * - LoadingOverlay (spinner + error + disconnect message) + * - Panel lifecycle (hide/show/focus events) + */ +export function TradingViewChartPanel( + props: WidgetPanelProps +): JSX.Element { + const { fetch, metadata, glContainer, glEventHub } = props; + + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const [isDisconnected, setIsDisconnected] = useState(false); + + const handleLoadingChange = useCallback((loading: boolean) => { + setIsLoading(loading); + }, []); + + const handleError = useCallback((err: string | null) => { + setError(err); + }, []); + + const handleSessionClose = useCallback(() => { + setIsDisconnected(true); + setError('Chart disconnected'); + }, []); + + const handleSessionOpen = useCallback(() => { + setIsDisconnected(false); + setError(null); + }, []); + + const name = metadata?.name ?? 'TradingView Chart'; + + // Cast to bridge golden-layout type version mismatches between + // @deephaven/plugin and @deephaven/dashboard-core-plugins. + // At runtime these are the same objects provided by the IDE. + const wp = WidgetPanel as unknown as React.ComponentType< + Record + >; + + return React.createElement( + wp, + { + glContainer, + glEventHub, + descriptor: { + name, + type: 'TvlChart', + }, + className: 'dh-tvl-panel', + isLoading, + isLoaded: !isLoading, + isDisconnected, + errorMessage: error ?? '', + onSessionClose: handleSessionClose, + onSessionOpen: handleSessionOpen, + }, + + ); +} + +export default TradingViewChartPanel; diff --git a/plugins/tradingview-lightweight/src/js/src/TradingViewChartRenderer.ts b/plugins/tradingview-lightweight/src/js/src/TradingViewChartRenderer.ts new file mode 100644 index 000000000..b978c963b --- /dev/null +++ b/plugins/tradingview-lightweight/src/js/src/TradingViewChartRenderer.ts @@ -0,0 +1,1248 @@ +import { + createChart, + createYieldCurveChart, + createOptionsChart, + createSeriesMarkers, + createTextWatermark, + ColorType, + CandlestickSeries, + BarSeries, + LineSeries, + AreaSeries, + BaselineSeries, + HistogramSeries, + TickMarkType, +} from 'lightweight-charts'; +import type { + IChartApi, + ISeriesApi, + IPriceLine, + SeriesType, + DeepPartial, + ChartOptions, + LogicalRange, + YieldCurveChartOptions, + PriceChartOptions, + SeriesMarker, + Time, + SeriesPartialOptionsMap, + ISeriesMarkersPluginApi, + ITextWatermarkPluginApi, + TextWatermarkOptions, + MouseEventParams, +} from 'lightweight-charts'; +import Log from '@deephaven/log'; +import type { + TvlChartType, + TvlSeriesConfig, + TvlMarkerData, + TvlTooltipOptions, +} from './TradingViewTypes'; +import { resolveColor, resolveColorsDeep } from './TradingViewColors'; +import { TradingViewTooltip } from './TradingViewTooltip'; + +const log = Log.module('TradingViewChartRenderer'); + +/** + * Registry of predefined price formatters. These are referenced by name + * from the Python API via `localization.priceFormatterName`. + */ +const PRICE_FORMATTERS: Record string> = { + currency_usd: (price: number) => + new Intl.NumberFormat('en-US', { + style: 'currency', + currency: 'USD', + }).format(price), + currency_eur: (price: number) => + new Intl.NumberFormat('de-DE', { + style: 'currency', + currency: 'EUR', + }).format(price), + currency_gbp: (price: number) => + new Intl.NumberFormat('en-GB', { + style: 'currency', + currency: 'GBP', + }).format(price), + currency_jpy: (price: number) => + new Intl.NumberFormat('ja-JP', { + style: 'currency', + currency: 'JPY', + }).format(price), + percent: (price: number) => `${price.toFixed(2)}%`, + compact: (price: number) => { + if (Math.abs(price) >= 1e9) return `${(price / 1e9).toFixed(1)}B`; + if (Math.abs(price) >= 1e6) return `${(price / 1e6).toFixed(1)}M`; + if (Math.abs(price) >= 1e3) return `${(price / 1e3).toFixed(1)}K`; + return price.toFixed(2); + }, + scientific: (price: number) => price.toExponential(2), +}; + +/** + * Resolve `localization.priceFormatterName` (a string) into a real + * `localization.priceFormatter` function, returning the remaining + * chart options with the substitution applied. + */ +function resolveLocalization( + opts: Record +): Record { + const { localization: locRaw, ...rest } = opts; + if (locRaw == null) return opts; + + const loc = locRaw as Record; + const { priceFormatterName, ...locRest } = loc; + if ( + typeof priceFormatterName === 'string' && + PRICE_FORMATTERS[priceFormatterName] != null + ) { + return { + ...rest, + localization: { + ...locRest, + priceFormatter: PRICE_FORMATTERS[priceFormatterName], + }, + }; + } + return opts; +} + +/** Default watermark font size when the user only provides text. */ +const DEFAULT_WATERMARK_FONT_SIZE = 66; + +/** Default watermark alpha applied to the chart's textColor. */ +const DEFAULT_WATERMARK_ALPHA = 0.2; + +/** + * Shape of watermark options as serialized by the Python API. + * The Python side sends a flat object; we convert it to the v5 + * `createTextWatermark` plugin format (which uses a `lines` array). + */ +interface WatermarkLineOptions { + text: string; + color?: string; + fontSize?: number; + fontStyle?: string; + lineHeight?: number; +} + +interface LegacyWatermarkOptions { + text?: string; + color?: string; + visible?: boolean; + fontSize?: number; + fontFamily?: string; + fontStyle?: string; + lineHeight?: number; + horzAlign?: string; + vertAlign?: string; + lines?: WatermarkLineOptions[]; +} + +/** + * Derive a semi-transparent watermark color from the chart's text color. + * Handles hex (#RGB, #RRGGBB), rgb(), and rgba() formats. + */ +function deriveWatermarkColor(textColor: string): string { + if (textColor.startsWith('#')) { + let hex = textColor; + // Expand shorthand #RGB to #RRGGBB + if (hex.length === 4) { + hex = `#${hex[1]}${hex[1]}${hex[2]}${hex[2]}${hex[3]}${hex[3]}`; + } + return hexToRgba(hex, DEFAULT_WATERMARK_ALPHA); + } + + // rgb(r, g, b) or rgba(r, g, b, a) — replace/add alpha + const match = textColor.match( + /rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:,\s*[\d.]+\s*)?\)/ + ); + if (match) { + return `rgba(${match[1]}, ${match[2]}, ${match[3]}, ${DEFAULT_WATERMARK_ALPHA})`; + } + + // Fallback: white at low opacity (works on dark and light backgrounds) + return `rgba(255, 255, 255, ${DEFAULT_WATERMARK_ALPHA})`; +} + +/** + * Format a UTC-seconds timestamp as a date/time string. + * @param utcSeconds UTC timestamp in seconds (as used by lightweight-charts Time) + * @param includeMs If true, append .SSS milliseconds + */ +function formatDateTime(utcSeconds: number, includeMs: boolean): string { + const d = new Date(utcSeconds * 1000); + const YYYY = d.getUTCFullYear(); + const MM = String(d.getUTCMonth() + 1).padStart(2, '0'); + const DD = String(d.getUTCDate()).padStart(2, '0'); + const hh = String(d.getUTCHours()).padStart(2, '0'); + const mm = String(d.getUTCMinutes()).padStart(2, '0'); + const ss = String(d.getUTCSeconds()).padStart(2, '0'); + const ms = String(d.getUTCMilliseconds()).padStart(3, '0'); + + const hasTime = hh !== '00' || mm !== '00' || ss !== '00' || ms !== '000'; + + if (!hasTime) { + return `${YYYY}-${MM}-${DD}`; + } + const base = `${YYYY}-${MM}-${DD} ${hh}:${mm}:${ss}`; + return includeMs ? `${base}.${ms}` : base; +} + +/** + * Custom tick mark formatter that uses uniform precision per level. + * Year/Month ticks show date only; DayOfMonth shows date; Time ticks + * show HH:MM:SS consistently (no mixing HH:MM and HH:MM:SS). + */ +function defaultTickMarkFormatter( + time: unknown, + tickMarkType: TickMarkType +): string | null { + const t = time as number; + const d = new Date(t * 1000); + + switch (tickMarkType) { + case TickMarkType.Year: + return String(d.getUTCFullYear()); + case TickMarkType.Month: + return d.toLocaleDateString('en-US', { + month: 'short', + year: 'numeric', + timeZone: 'UTC', + }); + case TickMarkType.DayOfMonth: { + return d.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric', + timeZone: 'UTC', + }); + } + case TickMarkType.Time: + case TickMarkType.TimeWithSeconds: + default: { + // Always show HH:MM:SS for uniform precision + const hh = String(d.getUTCHours()).padStart(2, '0'); + const mm = String(d.getUTCMinutes()).padStart(2, '0'); + const ss = String(d.getUTCSeconds()).padStart(2, '0'); + return `${hh}:${mm}:${ss}`; + } + } +} + +/** + * Tick mark formatter for the yield-curve chart. LWC's createYieldCurveChart + * treats horizontal-axis values as months and internally maps them to seconds + * from the unix epoch, so the default time-axis formatter would render + * Tenor=240 as "1990-01-01" (i.e. 240 months after 1970). We override the + * formatter so the axis reads as "Xm" / "Xy" instead. + */ +function yieldCurveTickMarkFormatter(time: unknown): string { + const months = Number(time) || 0; + if (months >= 12 && months % 12 === 0) return `${months / 12}y`; + if (months >= 12) return `${(months / 12).toFixed(1)}y`; + return `${months}m`; +} + +/** + * Crosshair time formatter for yield-curve charts: format the maturity + * value (months) as a duration instead of a date. + */ +function yieldCurveCrosshairFormatter(time: unknown): string { + return yieldCurveTickMarkFormatter(time); +} + +/** + * Tick mark formatter for the options / custom-numeric chart. LWC's + * createOptionsChart maps each x-value to seconds-from-epoch, so the + * default time-axis formatter renders X=5 as "1970-01-01 00:00:05". We + * override it so the axis reads back the raw numeric value (with light + * formatting so 100000 renders as "100,000"). + */ +function optionsTickMarkFormatter(time: unknown): string { + const n = Number(time); + if (!Number.isFinite(n)) return ''; + return n.toLocaleString('en-US', { maximumFractionDigits: 6 }); +} + +function optionsCrosshairFormatter(time: unknown): string { + return optionsTickMarkFormatter(time); +} + +/** + * Crosshair / tooltip time formatter — always shows full precision + * including milliseconds: "YYYY-MM-DD HH:MM:SS.mmm" + */ +function crosshairTimeFormatter(time: unknown): string { + return formatDateTime(time as number, true); +} + +// Map series type string to series definition constant +const SERIES_DEFINITIONS: Record = { + Candlestick: CandlestickSeries, + Bar: BarSeries, + Line: LineSeries, + Area: AreaSeries, + Baseline: BaselineSeries, + Histogram: HistogramSeries, +}; + +/** + * Series types whose color comes from OHLC up/down theme colors + * rather than the colorway palette. + */ +const OHLC_TYPES = new Set(['Candlestick', 'Bar']); + +/** + * Map of non-OHLC series type to the option key for its primary color. + */ +const PRIMARY_COLOR_KEY: Partial> = { + Line: 'color', + Area: 'lineColor', + Histogram: 'color', + Baseline: 'topLineColor', +}; + +/** + * Convert a hex color (#RRGGBB) to an rgba string with the given alpha. + */ +function hexToRgba(color: string, alpha: number): string { + // Handle #RGB shorthand + let hex = color; + if (hex.startsWith('#') && hex.length === 4) { + hex = `#${hex[1]}${hex[1]}${hex[2]}${hex[2]}${hex[3]}${hex[3]}`; + } + if (hex.startsWith('#') && hex.length >= 7) { + const r = parseInt(hex.slice(1, 3), 16); + const g = parseInt(hex.slice(3, 5), 16); + const b = parseInt(hex.slice(5, 7), 16); + return `rgba(${r}, ${g}, ${b}, ${alpha})`; + } + // Handle rgb(r, g, b) / rgba(r, g, b, a) — replace alpha + const match = color.match( + /rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*(?:,\s*[\d.]+\s*)?\)/ + ); + if (match) { + return `rgba(${match[1]}, ${match[2]}, ${match[3]}, ${alpha})`; + } + // Unrecognized format — return as-is rather than producing NaN + return color; +} + +/** + * Imperative wrapper around TradingView Lightweight Charts. + * Manages chart and series lifecycle, data updates, and theming. + */ +class TradingViewChartRenderer { + private chart: IChartApi; + + private chartType: TvlChartType; + + private seriesMap: Map> = new Map(); + + /** Resolved primary color per series id, used to tint the tracking tooltip. */ + private seriesColors: Map = new Map(); + + /** Active tracking tooltip, when enabled via chartOptions.tooltip.visible. */ + private tooltip: TradingViewTooltip | null = null; + + private markersMap: Map> = new Map(); + + /** Dynamic price lines that track a column's last-row value. */ + private dynamicPriceLines: Map< + string, + Array<{ priceLine: IPriceLine; column: string }> + > = new Map(); + + private watermarkPlugin: ITextWatermarkPluginApi