From 93725f602f85108fc92b2afa369e30a6c1f15feb Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Tue, 16 Jun 2026 15:25:42 -0400 Subject: [PATCH 01/16] feat(segmentation_user_layer): add user shader for segmentation layers + segment property ui control + segment properties in invlerp control --- docs/user-guide/index.rst | 11 +- docs/user-guide/segmentation_shaders.rst | 171 ++++ src/annotation/type_handler.browser_test.ts | 107 +++ src/annotation/type_handler.ts | 22 +- src/gpu_hash/shader.ts | 28 +- src/layer/annotation/index.ts | 2 +- src/layer/image/index.ts | 2 +- src/layer/index.ts | 8 + src/layer/segmentation/index.ts | 190 +++- src/layer/segmentation/json_keys.ts | 2 + src/layer/segmentation/rendering.md | 141 +++ .../segment_color_shader.browser_test.ts | 678 ++++++++++++++ src/layer/segmentation/style.css | 3 + src/layer/single_mesh/index.ts | 2 +- src/mesh/frontend.ts | 120 ++- src/segment_color.browser_test.ts | 2 +- src/segment_color.ts | 844 +++++++++++++++++- src/segmentation_display_state/frontend.ts | 131 +-- .../property_map.ts | 27 +- src/single_mesh/frontend.ts | 4 +- src/skeleton/frontend.ts | 158 +++- src/sliceview/single_texture_chunk_format.ts | 2 +- .../volume/segmentation_renderlayer.ts | 225 ++--- src/ui/segmentation_display_options_tab.ts | 60 +- src/volume_rendering/volume_render_layer.ts | 4 +- src/webgl/context.ts | 2 +- src/webgl/empirical_cdf.ts | 4 +- src/webgl/shader.ts | 14 +- src/webgl/shader_ui_controls.browser_test.ts | 345 ++++++- src/webgl/shader_ui_controls.ts | 391 +++++++- src/webgl/texture_access.browser_test.ts | 2 +- src/webgl/texture_access.ts | 34 + src/widget/invlerp.ts | 118 ++- src/widget/invlerp_range_finder.ts | 49 +- src/widget/layer_control_channel_invlerp.ts | 2 +- src/widget/layer_control_property.ts | 143 +++ src/widget/layer_control_property_invlerp.ts | 58 +- src/widget/shader_code_widget.ts | 13 +- src/widget/shader_controls.ts | 16 +- src/widget/transfer_function.ts | 2 +- 40 files changed, 3700 insertions(+), 437 deletions(-) create mode 100644 docs/user-guide/segmentation_shaders.rst create mode 100644 src/annotation/type_handler.browser_test.ts create mode 100644 src/layer/segmentation/rendering.md create mode 100644 src/layer/segmentation/segment_color_shader.browser_test.ts create mode 100644 src/widget/layer_control_property.ts diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst index 4230a8380c..4aac7f4788 100644 --- a/docs/user-guide/index.rst +++ b/docs/user-guide/index.rst @@ -12,10 +12,13 @@ Topics to segmentation layers. - :doc:`annotation_shaders` — writing custom GLSL shader code to drive the visual appearance of annotations from their properties. +- :doc:`segmentation_shaders` — writing custom GLSL shader code to color + segmentation layers from segment ids and segment properties. .. toctree:: - :hidden: + :hidden: - navigation - annotations - annotation_shaders + navigation + annotations + annotation_shaders + segmentation_shaders diff --git a/docs/user-guide/segmentation_shaders.rst b/docs/user-guide/segmentation_shaders.rst new file mode 100644 index 0000000000..422eb11e87 --- /dev/null +++ b/docs/user-guide/segmentation_shaders.rst @@ -0,0 +1,171 @@ +Segmentation Shaders +-------------------- + +Segmentation layers can use custom GLSL code to change how segments are colored. +The technical reference for the shader API is in +`the segmentation rendering guide `_. +This page is a gentler introduction to the same ideas. + +The Segment Color Function +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A segmentation shader defines a ``segmentColor`` function. The default shader +keeps Neuroglancer's existing segment color unchanged: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return color; + } + +The ``color`` argument is the color Neuroglancer would normally use for the +segment. It may come from the segment color hash, the layer's default segment +color, or an explicit color assigned to that segment. The ``isStated`` argument +is ``true`` when ``color`` came from an explicit segment color. The +``hasProperties`` argument is ``true`` when Neuroglancer found segment property +data for the current segment. + +To make every visible segment red, return a red ``vec3``: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return vec3(1.0, 0.0, 0.0); + } + +If you want to preserve explicit per-segment colors but recolor everything +else, use ``isStated``: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (isStated) { + return color; + } + return vec3(0.0, 0.6, 1.0); + } + +Opacity +~~~~~~~ + +Use a ``vec4`` return type to set opacity. The alpha channel is the fourth +component. Returning a negative alpha leaves the layer opacity unchanged. + +.. code-block:: text + + vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + if (isStated) { + return color; + } + return vec4(color.rgb, 0.35); + } + +Segment Properties +~~~~~~~~~~~~~~~~~~ + +Segment properties can drive color choices. If a segment property map is +available, a shader can read tags and properties directly by name: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + if (tag("axon")) { + return vec3(1.0, 0.4, 0.0); + } + if (prop("size") > 100u) { + return vec3(1.0, 1.0, 0.0); + } + return color; + } + +``tag("axon")`` returns ``true`` when the current segment has that tag. +``prop("size")`` reads a numerical property named ``size``. For string +properties, compare the value with a string literal: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + if (prop("class") == "interneuron") { + return vec3(0.0, 1.0, 0.6); + } + return color; + } + +Property Controls +~~~~~~~~~~~~~~~~~ + +A ``property`` UI control lets the user choose which segment property a shader +uses. The control can be filtered to tags, numerical properties, or string +properties. + +.. code-block:: glsl + + #uicontrol property selectedTag(type="tag") + #uicontrol property selectedSize(type="number") + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + if (selectedTag) { + return vec3(1.0, 0.0, 0.0); + } + if (selectedSize > 100u) { + return vec3(1.0, 1.0, 0.0); + } + return color; + } + +Use ``type="string"`` for a string property picker. ``type="number"`` and +``type="numerical"`` both select numerical properties. + +Data Mapping +~~~~~~~~~~~~ + +For continuous numerical properties, an ``invlerp`` control maps a property +range to ``0-1``. The user can adjust the selected property and range from the +shader UI. + +.. code-block:: glsl + + #uicontrol float intensity invlerp(property="size", range=[0, 1000]) + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + return intensity() * vec3(1.0, 0.2, 0.0); + } + +If ``property`` is omitted, Neuroglancer selects the first available numerical +segment property as the default. ``range`` controls the data values mapped to +``0`` and ``1``. ``window`` can be added to control the range shown by the UI +widget. + +Colormaps +~~~~~~~~~ + +The remapped value from an ``invlerp`` control can be passed to a colormap: + +.. code-block:: glsl + + #uicontrol float intensity invlerp(property="size", range=[0, 1000]) + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + if (isStated) { + return color; + } + return colormapJet(intensity()); + } + +This pattern is useful when a numerical segment property should control color +continuously while still respecting explicitly assigned segment colors. \ No newline at end of file diff --git a/src/annotation/type_handler.browser_test.ts b/src/annotation/type_handler.browser_test.ts new file mode 100644 index 0000000000..126e97b8e6 --- /dev/null +++ b/src/annotation/type_handler.browser_test.ts @@ -0,0 +1,107 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/annotation/bounding_box.js"; +import "#src/annotation/ellipsoid.js"; +import "#src/annotation/line.js"; +import "#src/annotation/point.js"; +import "#src/annotation/polyline.js"; + +import { describe, expect, it } from "vitest"; +import type { AnnotationPropertySpec } from "#src/annotation/index.js"; +import { AnnotationType } from "#src/annotation/index.js"; +import { + type AnnotationShaderGetter, + getAnnotationTypeRenderHandler, +} from "#src/annotation/type_handler.js"; +import { WatchableValue } from "#src/trackable_value.js"; +import { initializeWebGL } from "#src/webgl/context.js"; +import { + makeTrackableFragmentMain, + makeWatchableShaderError, +} from "#src/webgl/dynamic_shader.js"; +import { + getFallbackBuilderState, + parseShaderUiControls, + ShaderControlState, +} from "#src/webgl/shader_ui_controls.js"; + +describe("annotation property shaders", () => { + it("compiles a shader that colors by boolean property", () => { + const canvas = document.createElement("canvas"); + const gl = initializeWebGL(canvas); + const fragmentMain = makeTrackableFragmentMain(` +void main() { + if (prop_highlight() != 0u) { + setColor(vec3(1.0, 0.0, 0.0)); + } else { + setColor(vec3(0.0, 0.0, 1.0)); + } +} +`); + const shaderControlState = new ShaderControlState(fragmentMain); + const fallbackShaderParameters = new WatchableValue( + getFallbackBuilderState(parseShaderUiControls(fragmentMain.value)), + ); + const shaderError = makeWatchableShaderError(); + const properties: AnnotationPropertySpec[] = [ + { + identifier: "highlight", + description: undefined, + type: "bool", + default: 0, + }, + ]; + const renderHandler = getAnnotationTypeRenderHandler(AnnotationType.POINT); + const renderHelper = new renderHandler.perspectiveViewRenderHelper( + gl, + AnnotationType.POINT, + /*rank=*/ 3, + properties, + shaderControlState, + fallbackShaderParameters, + shaderError, + ); + renderHelper.targetIsSliceView = false; + renderHelper.pickIdsPerInstance = renderHandler.pickIdsPerInstance; + + const shaderGetter = ( + renderHelper as typeof renderHelper & { + shaderGetter3d: AnnotationShaderGetter; + } + ).shaderGetter3d; + const shaderResult = shaderGetter((builder) => { + builder.addOutputBuffer("vec4", "out_color", 0); + builder.addFragmentCode(` +void emit(vec4 color, highp uint pickId) { + out_color = color; +} +`); + }); + + expect(shaderError.value).toBeNull(); + expect(shaderResult.shader).not.toBeNull(); + expect(shaderResult.shader!.vertexSource).toContain( + "highp uint prop_highlight()", + ); + expect(shaderResult.shader!.vertexSource).toContain( + "if (prop_highlight() != 0u)", + ); + + renderHelper.dispose(); + shaderControlState.dispose(); + }); +}); diff --git a/src/annotation/type_handler.ts b/src/annotation/type_handler.ts index c8fae8d7c4..751c8282c2 100644 --- a/src/annotation/type_handler.ts +++ b/src/annotation/type_handler.ts @@ -42,6 +42,7 @@ import { parameterizedEmitterDependentShaderGetter, shaderCodeWithLineDirective, } from "#src/webgl/dynamic_shader.js"; +import { copyHistogramToCPU } from "#src/webgl/empirical_cdf.js"; import { defineInvlerpShaderFunction, enableLerpShaderFunction, @@ -444,7 +445,7 @@ float getMaxSubspaceClipCoefficient(float modelPointA[${this.rank}], float mode } `); - addControlsToBuilder(parameters, builder); + addControlsToBuilder(parameters, builder, /*fragment=*/ false); builder.addVertexCode(` const bool PROJECTION_VIEW = ${!this.targetIsSliceView}; bool ng_discardValue; @@ -776,21 +777,12 @@ gl_PointSize = 1.0; } gl.drawArrays(WebGL2RenderingContext.POINTS, 0, context.count); if (DEBUG_HISTOGRAMS) { - const tempBuffer = new Float32Array(256 * 4); - gl.readPixels( - 0, - 0, - 256, - 1, - WebGL2RenderingContext.RGBA, - WebGL2RenderingContext.FLOAT, - tempBuffer, + const histogram = copyHistogramToCPU(gl); + console.log( + "histogram property:", + propertyIdentifier, + histogram.join(" "), ); - const tempBuffer2 = new Float32Array(256); - for (let j = 0; j < 256; ++j) { - tempBuffer2[j] = tempBuffer[j * 4]; - } - console.log("histogram", tempBuffer2.join(" ")); } binder.disable(); break; diff --git a/src/gpu_hash/shader.ts b/src/gpu_hash/shader.ts index af44a0b97b..9140bdf899 100644 --- a/src/gpu_hash/shader.ts +++ b/src/gpu_hash/shader.ts @@ -119,7 +119,10 @@ export class HashSetShaderManager { this.readTable = prefix + "_readTable"; } - defineShader(builder: ShaderBuilder) { + defineShader(builder: ShaderBuilder, fragment = true) { + const addCode = fragment + ? builder.addFragmentCode.bind(builder) + : builder.addVertexCode.bind(builder); const { hashSeedsName, samplerName, numAlternatives, hashKeyMask } = this; builder.addUniform("highp uint", hashSeedsName, numAlternatives); builder.addUniform("highp uint", hashKeyMask); @@ -128,11 +131,11 @@ export class HashSetShaderManager { samplerName, this.textureUnitSymbol, ); - builder.addFragmentCode(glsl_hashCombine); - builder.addFragmentCode(glsl_uint64); - builder.addFragmentCode(glsl_equalUint64); + addCode(glsl_hashCombine); + addCode(glsl_uint64); + addCode(glsl_equalUint64); this.accessHelper.defineShader(builder); - builder.addFragmentCode( + addCode( this.accessHelper.getAccessor( this.readTable, this.samplerName, @@ -159,7 +162,7 @@ bool ${this.hasFunctionName}(uint64_t x) { return false; } `; - builder.addFragmentCode(s); + addCode(s); } get hasFunctionName() { @@ -172,7 +175,7 @@ bool ${this.hasFunctionName}(uint64_t x) { hashTable: GPUHashTable, ) { hashTable.copyToGPU(); - const textureUnit = shader.textureUnit(this.textureUnitSymbol); + const textureUnit = shader.textureUnit(this.textureUnitSymbol)!; gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + textureUnit); gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, hashTable.texture); gl.uniform1ui( @@ -186,15 +189,18 @@ bool ${this.hasFunctionName}(uint64_t x) { } disable(gl: GL, shader: ShaderProgram) { - const textureUnit = shader.textureUnit(this.textureUnitSymbol); + const textureUnit = shader.textureUnit(this.textureUnitSymbol)!; gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + textureUnit); gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, null); } } export class HashMapShaderManager extends HashSetShaderManager { - defineShader(builder: ShaderBuilder) { - super.defineShader(builder); + defineShader(builder: ShaderBuilder, fragment = true) { + const addCode = fragment + ? builder.addFragmentCode.bind(builder) + : builder.addVertexCode.bind(builder); + super.defineShader(builder, fragment); const { numAlternatives, hashSeedsName, hashKeyMask } = this; let s = ` bool ${this.getFunctionName}(uint64_t x, out uint64_t value) { @@ -215,7 +221,7 @@ bool ${this.getFunctionName}(uint64_t x, out uint64_t value) { return false; } `; - builder.addFragmentCode(s); + addCode(s); } get getFunctionName() { diff --git a/src/layer/annotation/index.ts b/src/layer/annotation/index.ts index 55c1a33463..ee119452f2 100644 --- a/src/layer/annotation/index.ts +++ b/src/layer/annotation/index.ts @@ -849,7 +849,7 @@ export class AnnotationUserLayer extends Base { function makeShaderCodeWidget(layer: AnnotationUserLayer) { return new ShaderCodeWidget({ shaderError: layer.annotationDisplayState.shaderError, - fragmentMain: layer.annotationDisplayState.shader, + fragment: layer.annotationDisplayState.shader, shaderControlState: layer.annotationDisplayState.shaderControls, }); } diff --git a/src/layer/image/index.ts b/src/layer/image/index.ts index b58b7cbaa0..802f43128a 100644 --- a/src/layer/image/index.ts +++ b/src/layer/image/index.ts @@ -471,7 +471,7 @@ export class ImageUserLayer extends Base { function makeShaderCodeWidget(layer: ImageUserLayer) { return new ShaderCodeWidget({ shaderError: layer.shaderError, - fragmentMain: layer.fragmentMain, + fragment: layer.fragmentMain, shaderControlState: layer.shaderControlState, }); } diff --git a/src/layer/index.ts b/src/layer/index.ts index fad0e1400e..0d3fb57d9d 100644 --- a/src/layer/index.ts +++ b/src/layer/index.ts @@ -364,6 +364,9 @@ export class UserLayer extends RefCounted { get isReady() { return this.loadingCounter === 0; } + isReadyWatchable: WatchableValueInterface = new WatchableValue( + false, + ); tabs = this.registerDisposer(new TabSpecification()); panels = new UserLayerSidePanelsState(this); @@ -379,6 +382,11 @@ export class UserLayer extends RefCounted { constructor(public managedLayer: Borrowed) { super(); + this.registerDisposer( + this.readyStateChanged.add(() => { + this.isReadyWatchable.value = this.isReady; + }), + ); this.toolBinder = this.registerDisposer( new LayerToolBinder(this, this.manager.root.toolBinder), ); diff --git a/src/layer/segmentation/index.ts b/src/layer/segmentation/index.ts index 8fc24b8fd4..f6032dab98 100644 --- a/src/layer/segmentation/index.ts +++ b/src/layer/segmentation/index.ts @@ -45,7 +45,12 @@ import { RenderScaleHistogram, trackableRenderScaleTarget, } from "#src/render_scale_statistics.js"; -import { getCssColor, SegmentColorHash } from "#src/segment_color.js"; +import { + encodeSegmentPropertyShaderDefinition, + getCssColor, + SegmentColorHash, + SegmentColorUserShaderManager, +} from "#src/segment_color.js"; import type { SegmentationColorGroupState, SegmentationDisplayState, @@ -92,6 +97,7 @@ import type { WatchableValueInterface, } from "#src/trackable_value.js"; import { + AggregateWatchableValue, IndirectTrackableValue, IndirectWatchableValue, makeCachedDerivedWatchableValue, @@ -109,6 +115,7 @@ import { DisplayOptionsTab } from "#src/ui/segmentation_display_options_tab.js"; import { Uint64Map } from "#src/uint64_map.js"; import { Uint64OrderedSet } from "#src/uint64_ordered_set.js"; import { Uint64Set } from "#src/uint64_set.js"; +import type { TypedNumberArray } from "#src/util/array.js"; import { gatherUpdate } from "#src/util/array.js"; import { packColor, @@ -119,7 +126,8 @@ import { } from "#src/util/color.js"; import type { Borrowed, Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; -import type { vec3, vec4 } from "#src/util/geom.js"; +import type { vec3 } from "#src/util/geom.js"; +import { vec4 } from "#src/util/geom.js"; import { parseArray, parseUint64, @@ -129,7 +137,14 @@ import { verifyString, } from "#src/util/json.js"; import { Signal } from "#src/util/signal.js"; -import { makeWatchableShaderError } from "#src/webgl/dynamic_shader.js"; +import { GLBuffer } from "#src/webgl/buffer.js"; +import { initializeWebGL } from "#src/webgl/context.js"; +import { + makeTrackableFragmentMain, + makeWatchableShaderError, + parameterizedEmitterDependentShaderGetter, +} from "#src/webgl/dynamic_shader.js"; +import { ShaderControlState } from "#src/webgl/shader_ui_controls.js"; import type { DependentViewContext } from "#src/widget/dependent_view_widget.js"; import { registerLayerShaderControlsTool } from "#src/widget/shader_controls.js"; @@ -410,7 +425,18 @@ class LinkedSegmentationGroupState< } } +export const DEFAULT_FRAGMENT_SEGMENT_COLOR = ` +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return color; +} +`; + class SegmentationUserLayerDisplayState implements SegmentationDisplayState { + private getSegmentColorShader; + + segmentColorShaderControlState: ShaderControlState; + segmentationColorUserShader: SegmentColorUserShaderManager; + constructor(public layer: SegmentationUserLayer) { // Even though `SegmentationUserLayer` assigns this to its `displayState` property, redundantly // assign it here first in order to allow it to be accessed by `segmentationGroupState`. @@ -520,8 +546,57 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { (group) => group.segmentPropertyMap, ), ); + + this.segmentColorShaderControlState = this.layer.registerDisposer( + new ShaderControlState( + this.fragmentSegmentColor, + makeCachedLazyDerivedWatchableValue( + (segmentPropertyMap, isReady) => { + const properties = new Map(); + const values = new Map>(); + if (segmentPropertyMap === undefined) { + // dont return a non null before layer.isReady to prevent losing shader control state during loading process + return isReady ? {} : null; + } + for (const property of segmentPropertyMap.numericalProperties) { + properties.set(property.id, property.dataType); + values.set(property.id, property.values); + } + const shaderName = (property: string) => { + const propertyIdx = + segmentPropertyMap.numericalProperties.findIndex( + (p) => p.id === property, + ); + return `numerical${propertyIdx}`; // TEMP extract this from the SegmentationColorUserShaderManager + }; + + return { properties, values, shaderName, segmentPropertyMap }; + }, + this.segmentationGroupState.value.segmentPropertyMap, + this.layer.isReadyWatchable, + ), + undefined, + {}, + DEFAULT_FRAGMENT_SEGMENT_COLOR, + ), + ); + + // 2d/3d share GL context + // the offscreen canvas is a separate context that doesn't share textures + this.segmentationColorUserShader = new SegmentColorUserShaderManager( + this, + this.layer.manager.chunkManager.chunkQueueManager.gl, + ); + + this.offscreenGL = initializeWebGL(new OffscreenCanvas(1, 1)); + this.offscreenSegmentationColorUserShader = + new SegmentColorUserShaderManager(this, this.offscreenGL); + this.getSegmentColorShader = this.makeSegmentColorShaderGetter(); } + offscreenGL; + offscreenSegmentationColorUserShader; + segmentSelectionState = new SegmentSelectionState(); selectedAlpha = trackableAlphaValue(0.5); saturation = trackableAlphaValue(1.0); @@ -536,6 +611,10 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { ignoreNullVisibleSet = new TrackableBoolean(true, true); skeletonRenderingOptions = new SkeletonRenderingOptions(); shaderError = makeWatchableShaderError(); + fragmentSegmentColor = makeTrackableFragmentMain( + DEFAULT_FRAGMENT_SEGMENT_COLOR, + ); + renderScaleHistogram = new RenderScaleHistogram(); renderScaleTarget = trackableRenderScaleTarget(1); selectSegment: (id: bigint, pin: boolean | "toggle") => void; @@ -551,6 +630,90 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { this.layer.moveToSegment(id); }; + makeSegmentColorShaderGetter = () => { + const parameters = this.layer.registerDisposer( + new AggregateWatchableValue(() => ({ + segmentColorParameters: + this.segmentationColorUserShader.shaderParameters, + segmentColorProperties: + this.offscreenSegmentationColorUserShader.usedProperties, + shaderBuilderState: this.segmentColorShaderControlState.builderState, + })), + ); + return parameterizedEmitterDependentShaderGetter( + this.layer, + this.offscreenGL, + { + memoizeKey: `segmentation/ColorShader`, + parameters, + encodeParameters: (p) => { + return `${p.shaderBuilderState.key}/${JSON.stringify(p.segmentColorParameters)}/${JSON.stringify(p.segmentColorProperties.map(encodeSegmentPropertyShaderDefinition))}`; + }, + shaderError: this.layer.displayState.shaderError, // TODO can I reuse this? + defineShader: (builder, { shaderBuilderState }) => { + this.offscreenSegmentationColorUserShader.defineShader( + builder, + /*fragment=*/ false, + shaderBuilderState, + ); + builder.addAttribute("highp vec4", "aVertexPosition"); + builder.addUniform("highp uvec2", "uID"); + builder.addVarying("highp vec4", "vColor"); + const vertexMain = ` +gl_Position = aVertexPosition; +vColor = segmentColorUserShader(uint64_t(uID)); +`; + builder.addVertexMain(vertexMain); + builder.addOutputBuffer("vec4", "out_fragColor", 0); + builder.setFragmentMain("out_fragColor = vColor;"); + }, + }, + ); + }; + + context = () => {}; // TEMP how to get rid of this? + + private tempColor = vec4.create(); + + getShaderSegmentColor = (id: bigint) => { + const { shader, parameters } = this.getSegmentColorShader(this.context); + if (shader === null) return; + shader.bind(); + const { gl } = shader; + const positionBuffer = GLBuffer.fromData( + gl, + new Float32Array([1, 1, -1, 1, 1, -1, -1, -1]), + ); + positionBuffer.bindToVertexAttrib(shader.attribute("aVertexPosition"), 2); + this.offscreenSegmentationColorUserShader.enable( + gl, + shader, + parameters.shaderBuilderState, + ); + gl.uniform2ui( + shader.uniform(`uID`), + Number(id & 0xffffffffn), + Number(id >> 32n), + ); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + const data = new Uint8Array(4); + // TODO can I read straight to float? + gl.readPixels( + 0, + 0, + 1, + 1, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + data, + ); + for (let i = 0; i < data.length; i++) { + this.tempColor[i] = data[i] / 255.0; + } + this.offscreenSegmentationColorUserShader.disable(gl, shader); + return this.tempColor; + }; + linkedSegmentationGroup: LinkedLayerGroup; linkedSegmentationColorGroup: LinkedLayerGroup; originalSegmentationGroupState: SegmentationUserLayerGroupState; @@ -689,6 +852,12 @@ export class SegmentationUserLayer extends Base { this.displayState.linkedSegmentationGroup.changed.add(() => this.updateDataSubsourceActivations(), ); + this.displayState.fragmentSegmentColor.changed.add( + this.specificationChanged.dispatch, + ); + this.displayState.segmentColorShaderControlState.changed.add( + this.specificationChanged.dispatch, + ); this.tabs.add("rendering", { label: "Render", order: -100, @@ -996,6 +1165,9 @@ export class SegmentationUserLayer extends Base { this.displayState.ignoreNullVisibleSet.restoreState( specification[json_keys.IGNORE_NULL_VISIBLE_SET_JSON_KEY], ); + this.displayState.fragmentSegmentColor.restoreState( + specification[json_keys.SEGMENT_COLOR_SHADER_JSON_KEY], + ); const { skeletonRenderingOptions } = this.displayState; skeletonRenderingOptions.restoreState( @@ -1040,6 +1212,9 @@ export class SegmentationUserLayer extends Base { this.displayState.segmentationColorGroupState.value.restoreState( specification, ); + this.displayState.segmentColorShaderControlState.restoreState( + specification[json_keys.SHADER_CONTROLS_JSON_KEY], + ); } toJSON() { @@ -1068,6 +1243,8 @@ export class SegmentationUserLayer extends Base { this.displayState.renderScaleTarget.toJSON(); x[json_keys.CROSS_SECTION_RENDER_SCALE_JSON_KEY] = this.sliceViewRenderScaleTarget.toJSON(); + x[json_keys.SEGMENT_COLOR_SHADER_JSON_KEY] = + this.displayState.fragmentSegmentColor.toJSON(); const { linkedSegmentationGroup, linkedSegmentationColorGroup } = this.displayState; @@ -1091,6 +1268,8 @@ export class SegmentationUserLayer extends Base { this.displayState.segmentationColorGroupState.value.toJSON(), ); } + x[json_keys.SHADER_CONTROLS_JSON_KEY] = + this.displayState.segmentColorShaderControlState.toJSON(); return x; } @@ -1367,6 +1546,7 @@ export class SegmentationUserLayer extends Base { const visibleSegments = [...visibleSegmentsSet]; const colors = visibleSegments.map((id) => { + // here we can do a batch get of colors using the segment color shader instead of one at a time const color = getCssColor(getBaseObjectColor(displayState, id)); return { color, id }; }); @@ -1403,6 +1583,10 @@ registerLayerTypeDetector((subsource) => { return undefined; }); +registerLayerShaderControlsTool(SegmentationUserLayer, (layer) => ({ + shaderControlState: layer.displayState.segmentColorShaderControlState, +})); + registerLayerShaderControlsTool( SegmentationUserLayer, (layer) => ({ diff --git a/src/layer/segmentation/json_keys.ts b/src/layer/segmentation/json_keys.ts index 47d603dab4..faf5c6397d 100644 --- a/src/layer/segmentation/json_keys.ts +++ b/src/layer/segmentation/json_keys.ts @@ -14,6 +14,7 @@ export const COLOR_SEED_JSON_KEY = "colorSeed"; export const SEGMENT_STATED_COLORS_JSON_KEY = "segmentColors"; export const MESH_RENDER_SCALE_JSON_KEY = "meshRenderScale"; export const CROSS_SECTION_RENDER_SCALE_JSON_KEY = "crossSectionRenderScale"; +export const SEGMENT_COLOR_SHADER_JSON_KEY = "segmentColorShader"; export const SKELETON_RENDERING_JSON_KEY = "skeletonRendering"; export const SKELETON_SHADER_JSON_KEY = "skeletonShader"; export const SKELETON_CODE_VISIBLE_KEY = "codeVisible"; @@ -26,3 +27,4 @@ export const SEGMENT_DEFAULT_COLOR_JSON_KEY = "segmentDefaultColor"; export const ANCHOR_SEGMENT_JSON_KEY = "anchorSegment"; export const SKELETON_RENDERING_SHADER_CONTROL_TOOL_ID = "skeletonShaderControl"; +export const SHADER_CONTROLS_JSON_KEY = "shaderControls"; diff --git a/src/layer/segmentation/rendering.md b/src/layer/segmentation/rendering.md new file mode 100644 index 0000000000..ced46e48a7 --- /dev/null +++ b/src/layer/segmentation/rendering.md @@ -0,0 +1,141 @@ +# Segmentation rendering + +Segmentation rendering can be customized using GLSL shader code. The shader is +applied after Neuroglancer has determined the base segment color, including +explicit per-segment colors and the default segment color. + +## Shader language + +The shader code must conform to the OpenGL ES Shading Language (GLSL) version 3.0, specified at . + +You may find the WebGL reference card helpful: . + +## UI Controls + +[UI control directives](../sliceview/image_layer_rendering.md#ui-controls) are +supported. Segmentation shaders also support segment property controls: + +```glsl +#uicontrol property selectedTag(type="tag") +#uicontrol property selectedNumber(type="number") +#uicontrol property selectedString(type="string") +``` + +The `type` parameter filters the property picker. Valid values are `tag`, +`number`, `numerical`, and `string`. In shader code, a tag property control is a +`bool`; numerical property controls have the GLSL type corresponding to the +selected property data type; string property controls are string values and may +be compared with string literals. + +Numerical segment properties can also be used with `invlerp` controls: + +```glsl +#uicontrol float intensity invlerp(property="size", range=[0, 100]) +``` + +If `property` is omitted, the control defaults to the first available numerical +segment property and the selected property may be changed from the UI. + +## API + +### Format of user shader + +The basic user shader is in the form of: + +```glsl +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return color; +} +``` + +- `color` is the color that would be displayed if you chose not to override it. + It includes the segment color hash, the default segment color, or a stated + per-segment color. + +- `hasProperties` indicates that segment property data was found for the target + segment. Check this before relying on property values if your shader needs a + fallback for segments without property rows. + +- `isStated` indicates that `color` came from an explicit stated segment color. + +A `vec4` version of `segmentColor` can also be used to override opacity. A +negative alpha means that the shader does not override the opacity determined by +the segmentation layer. + +```glsl +vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + return color; +} +``` + +### Accessing segmentation properties + +To retrieve a numerical or string property named `myProperty`, use the syntax +`prop("myProperty")`. + +To check if a tag named `myTag` is enabled for a segment, use the syntax + +```glsl +if (tag("myTag")) { + // do something +} +``` + +The helper functions require a segment property map. If the named tag or +property does not exist, shader compilation reports an error. + +### Examples + +```glsl +#uicontrol float property1 invlerp(property="myNumericalProperty", window=[1, 10]) +vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec4(0.5, 0.5, 0.5, 1.0); + } + if (isStated) { + return color; + } + vec4 newColor = vec4(0.0, 0.0, 0.0, 1.0); + newColor.rgb = colormapJet(property1()); + + if (tag("lot-of-axon")) { + return vec4(1.0, 1.0, 1.0, 1.0); + } + + newColor.a = 1.0; + + return newColor; +} +``` + +```glsl +#uicontrol property redTag(type="tag") +#uicontrol property size(type="number") +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5); + } + if (redTag) { + return vec3(1.0, 0.0, 0.0); + } + if (size > 50u) { + return vec3(1.0, 1.0, 0.0); + } + return color.rgb; +} +``` + +```glsl +vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec4(0.5, 0.5, 0.5, 1.0); + } + if (isStated) { + return color; + } + if (prop("NAis") > 10u) { + return vec4(1.0, 1.0, 1.0, 1.0); + } + return color; +} +``` diff --git a/src/layer/segmentation/segment_color_shader.browser_test.ts b/src/layer/segmentation/segment_color_shader.browser_test.ts new file mode 100644 index 0000000000..ffc2a56571 --- /dev/null +++ b/src/layer/segmentation/segment_color_shader.browser_test.ts @@ -0,0 +1,678 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect } from "vitest"; +import { DisplayContext } from "#src/display_context.js"; +import { makeLayer } from "#src/layer/index.js"; +import type { SegmentationUserLayer } from "#src/layer/segmentation/index.js"; +import "#layer/segmentation"; +import { + type InlineSegmentPropertyMap, + PreprocessedSegmentPropertyMap, + SegmentPropertyMap, +} from "#src/segmentation_display_state/property_map.js"; +import { packColor } from "#src/util/color.js"; +import { DataType } from "#src/util/data_type.js"; +import { vec3, vec4 } from "#src/util/geom.js"; +import { Viewer } from "#src/viewer.js"; +import { ShaderCompilationError } from "#src/webgl/shader.js"; +import type { SegmentPropertyReference } from "#src/webgl/shader_ui_controls.js"; + +const setupSegmentationLayer = () => { + const target = document.createElement("div"); + const display = new DisplayContext(target); + const viewer = new Viewer(display); + return makeLayer(viewer.layerSpecification, "test", { type: "segmentation" }) + .layer! as SegmentationUserLayer; +}; + +const setSegmentPropertyMap = ( + segmentationUserLayer: SegmentationUserLayer, + inlineProperties: InlineSegmentPropertyMap, +) => { + segmentationUserLayer.displayState.segmentPropertyMap.value = + new PreprocessedSegmentPropertyMap( + new SegmentPropertyMap({ inlineProperties }), + ); +}; + +const setSegmentPropertyControl = ( + segmentationUserLayer: SegmentationUserLayer, + controlName: string, + value: SegmentPropertyReference, +) => { + const controlState = + segmentationUserLayer.displayState.segmentColorShaderControlState.state.get( + controlName, + ); + expect(controlState).toBeDefined(); + controlState!.trackable.value = value; +}; + +const compareWithCPUHash = ( + segmentationUserLayer: SegmentationUserLayer, + objectId: bigint, +) => { + const outColor = + segmentationUserLayer.displayState.getShaderSegmentColor(objectId); + const colorGroupState = + segmentationUserLayer.displayState.segmentationColorGroupState.value; + const outColorCPU = vec4.create(); + colorGroupState.segmentColorHash.compute(outColorCPU, objectId); + expect(outColor).toBeDefined(); + expect(outColor!.length).toBe(4); + // console.log("comparing colors", outColor!.join(','), outColorCPU.join(',')); + for (let i = 0; i < 4; ++i) { + expect(outColor![i]).toBeCloseTo(outColorCPU[i]); + } +}; + +expect.extend({ + toBeCloseToFoo(received: number[] | Float32Array, expected: number[]) { + if (received.length !== expected.length) { + return { + pass: false, + message: () => + `Expected array length ${expected.length} but received ${received.length}`, + }; + } + for (let i = 0; i < received.length; ++i) { + if (Math.abs(received[i] - expected[i]) > 1e-6) { + return { + pass: false, + message: () => + `Expected element ${i} to be close to ${expected[i]} but received ${received[i]}`, + }; + } + } + return { + pass: true, + message: () => "Arrays are close", + }; + }, +}); + +const expectColor = ( + color: vec4, + expected: [number, number, number, number], +) => { + expect(color).toBeDefined(); + expect(color!.length).toBe(4); + expect([...color]).toEqual(expected.map((x) => expect.closeTo(x))); +}; + +describe("getShaderSegmentColor", () => { + it("default shader, return hash", () => { + const segmentationUserLayer = setupSegmentationLayer(); + const objectId = 1n; + compareWithCPUHash(segmentationUserLayer, objectId); + }); + it("default shader, random segment id", () => { + const segmentationUserLayer = setupSegmentationLayer(); + const objectId = BigInt(Math.floor(Math.random() * 100000)); + compareWithCPUHash(segmentationUserLayer, objectId); + }); + it("red shader", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return vec3(1.0, 0.0, 0.0); + }`; + const outColor = + segmentationUserLayer.displayState.getShaderSegmentColor(1n); + expectColor(outColor!, [1.0, 0.0, 0.0, 0.0]); + }); + + it("alpha shader", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` + vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + return vec4(0.0, 0.0, 0.0, 0.5); + }`; + const outColor = + segmentationUserLayer.displayState.getShaderSegmentColor(1n); + expectColor(outColor!, [0.0, 0.0, 0.0, 0.5]); + }); + + it("preserves alpha from mapped segment colors", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.segmentStatedColors.value.set( + 1n, + BigInt(packColor(vec4.fromValues(0.25, 0.5, 0.75, 0.5))), + ); + + expectColor(segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, [ + 0.25, + 0.5, + 0.75, + 128 / 255, + ]); + }); + + it("treats rgb mapped segment colors as having undefined alpha", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.segmentStatedColors.value.set( + 1n, + BigInt(packColor(vec3.fromValues(1.0, 0.0, 0.0))), + ); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` + vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + return vec4(color.rgb, color.a < 0.0 ? 0.75 : 0.25); + }`; + + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.75], + ); + }); + + it("returns undefined if segment properties have not been loaded and shader uses properties", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property redTag(type="tag") + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (redTag) { + return vec3(1.0, 0.0, 0.0); + } + return vec3(0.0, 0.0, 0.0); + }`; + + expect( + segmentationUserLayer.displayState.getShaderSegmentColor(1n), + ).toBeUndefined(); + }); + + it("colors by string properties", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "color", + type: "string", + values: ["red", "green"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property colorProperty(type="string") + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.0, 0.0, 1.0); + } + if (colorProperty == "red") { + return vec3(1.0, 0.0, 0.0); + } + if (colorProperty == "green") { + return vec3(0.0, 1.0, 0.0); + } + return vec3(0.5, 0.5, 0.5); + }`; + setSegmentPropertyControl(segmentationUserLayer, "colorProperty", { + type: "string", + id: "color", + }); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + [0.0, 1.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(3n)!, + [0.0, 0.0, 1.0, 0.0], + ); + }); + + it("defaults invalid string property control state", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "color", + type: "string", + values: ["red"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property colorProperty(type="string") + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (colorProperty == "red") { + return vec3(0.0, 1.0, 0.0); + } + return vec3(0.5, 0.5, 0.5); + }`; + const controlState = + segmentationUserLayer.displayState.segmentColorShaderControlState.state.get( + "colorProperty", + ); + expect(controlState).toBeDefined(); + controlState!.trackable.restoreState({ type: "string", id: "foo" }); + expect(controlState!.trackable.value).toEqual({ + type: "string", + id: "color", + }); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [0.0, 1.0, 0.0, 0.0], + ); + }); + + it("handles unmatched string property value", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "color", + type: "string", + values: ["foo"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property colorProperty(type="string") + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (colorProperty != "red") { + return vec3(0.0, 1.0, 0.0); + } + return vec3(0.5, 0.5, 0.5); + }`; + setSegmentPropertyControl(segmentationUserLayer, "colorProperty", { + type: "string", + id: "color", + }); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [0.0, 1.0, 0.0, 0.0], + ); + }); + + it("colors by tag", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "tag1", + type: "tags", + tags: ["red", "blue"], + tagDescriptions: ["red", "blue"], + values: ["\u0000", "\u0001"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property redTag(type="tag") +#uicontrol property blueTag(type="tag") +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.0, 1.0, 0.0); + } + if (redTag) { + return vec3(1.0, 0.0, 0.0); + } + if (blueTag) { + return vec3(0.0, 0.0, 1.0); + } + return vec3(0.3, 0.6, 0.9); +}`; + setSegmentPropertyControl(segmentationUserLayer, "redTag", { + type: "tag", + id: "red", + }); + setSegmentPropertyControl(segmentationUserLayer, "blueTag", { + type: "tag", + id: "blue", + }); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + [0.0, 0.0, 1.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(3n)!, + [0.0, 1.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(0n)!, + [0.0, 1.0, 0.0, 0.0], + ); + }); + + it("colors by tag helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "tag1", + type: "tags", + tags: ["red", "blue"], + tagDescriptions: ["red", "blue"], + values: ["\u0000", "\u0001"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.0, 1.0, 0.0); + } + if (tag("red")) { + return vec3(1.0, 0.0, 0.0); + } + if (tag("blue")) { + return vec3(0.0, 0.0, 1.0); + } + return vec3(0.3, 0.6, 0.9); +}`; + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + [0.0, 0.0, 1.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(3n)!, + [0.0, 1.0, 0.0, 0.0], + ); + }); + + it("reports shader error for missing tag helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "tag1", + type: "tags", + tags: ["red"], + tagDescriptions: ["red"], + values: ["\u0000"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (tag("blue")) { + return vec3(0.0, 0.0, 1.0); + } + float otherError = missingValue; + return vec3(0.0, 0.0, 0.0); +}`; + expect( + segmentationUserLayer.displayState.getShaderSegmentColor(1n), + ).toBeUndefined(); + const shaderError = segmentationUserLayer.displayState.shaderError.value; + expect(shaderError).toBeInstanceOf(ShaderCompilationError); + expect(shaderError!.message).toContain( + `'tag("blue")' : tag does not exist`, + ); + expect(shaderError!.message).not.toContain( + `'' : 'tag("blue")' : tag does not exist`, + ); + expect(shaderError!.message).not.toContain( + "no matching overloaded function found", + ); + expect(shaderError!.message).toContain("missingValue"); + expect( + (shaderError as ShaderCompilationError).errorMessages.find((x) => + x.message.includes(`'tag("blue")' : tag does not exist`), + )?.line, + ).toBe(2); + expect( + (shaderError as ShaderCompilationError).errorMessages.find((x) => + x.message.includes("missingValue"), + )?.line, + ).toBe(5); + }); + + it("colors by numerical property uint8", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.UINT8, + values: new Uint8Array([0, 50]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property prop1Property(type="number") +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (prop1Property == 50u) { + return vec3(1.0, 0.0, 0.0); + } + return vec3(0.0, 0.0, 0.0); +}`; + setSegmentPropertyControl(segmentationUserLayer, "prop1Property", { + type: "numerical", + id: "prop1", + }); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [0.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + [1.0, 0.0, 0.0, 0.0], + ); + }); + + it("colors by numerical property helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.UINT8, + values: new Uint8Array([0, 50]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (prop("prop1") == 50u) { + return vec3(1.0, 0.0, 0.0); + } + return vec3(0.0, 0.0, 0.0); +}`; + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [0.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + [1.0, 0.0, 0.0, 0.0], + ); + }); + + it("reports shader error for missing property helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.UINT8, + values: new Uint8Array([0]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (prop("prop2") == 50u) { + return vec3(1.0, 0.0, 0.0); + } + float otherError = missingValue; + return vec3(0.0, 0.0, 0.0); +}`; + expect( + segmentationUserLayer.displayState.getShaderSegmentColor(1n), + ).toBeUndefined(); + const shaderError = segmentationUserLayer.displayState.shaderError.value; + expect(shaderError).toBeInstanceOf(ShaderCompilationError); + expect(shaderError!.message).toContain( + `'prop("prop2")' : property does not exist`, + ); + expect(shaderError!.message).not.toContain( + `'' : 'prop("prop2")' : property does not exist`, + ); + expect(shaderError!.message).not.toContain( + "no matching overloaded function found", + ); + expect(shaderError!.message).toContain("missingValue"); + expect( + (shaderError as ShaderCompilationError).errorMessages.find((x) => + x.message.includes(`'prop("prop2")' : property does not exist`), + )?.line, + ).toBe(2); + expect( + (shaderError as ShaderCompilationError).errorMessages.find((x) => + x.message.includes("missingValue"), + )?.line, + ).toBe(5); + }); + + it("colors by string property helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "color", + type: "string", + values: ["red", "green"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.0, 0.0, 1.0); + } + if (prop("color") == "red") { + return vec3(1.0, 0.0, 0.0); + } + if (prop("color") == "green") { + return vec3(0.0, 1.0, 0.0); + } + return vec3(0.5, 0.5, 0.5); + }`; + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + [0.0, 1.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(3n)!, + [0.0, 0.0, 1.0, 0.0], + ); + }); + + it("colors by numerical property invlerp", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.UINT8, + values: new Uint8Array([0, 50]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol invlerp normalized(property="prop1", range=[0, 100]) +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return vec3(normalized(), 0.0, 0.0); +}`; + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [0.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + [0.5, 0.0, 0.0, 0.0], + ); + }); + + it("colors by numerical property float", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.FLOAT32, + values: new Float32Array([0, 0.75]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property prop1Property(type="number") +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return vec3(prop1Property, 0.0, 0.0); +}`; + setSegmentPropertyControl(segmentationUserLayer, "prop1Property", { + type: "numerical", + id: "prop1", + }); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + [0.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + [0.75, 0.0, 0.0, 0.0], + ); + }); +}); diff --git a/src/layer/segmentation/style.css b/src/layer/segmentation/style.css index 3d379ba8de..6ff4257c1c 100644 --- a/src/layer/segmentation/style.css +++ b/src/layer/segmentation/style.css @@ -22,6 +22,7 @@ .neuroglancer-segmentation-rendering-tab .neuroglancer-shader-code-widget { height: 6em; + resize: vertical; } .neuroglancer-segmentation-dropdown-skeleton-shader-header { @@ -30,6 +31,8 @@ align-items: center; } +.neuroglancer-segmentation-layer-segment-color-shader-overlay + .neuroglancer-shader-code-widget, .neuroglancer-segmentation-layer-skeleton-shader-overlay .neuroglancer-shader-code-widget { width: 80vw; diff --git a/src/layer/single_mesh/index.ts b/src/layer/single_mesh/index.ts index cbd65286a4..1279851b7f 100644 --- a/src/layer/single_mesh/index.ts +++ b/src/layer/single_mesh/index.ts @@ -143,7 +143,7 @@ export class SingleMeshUserLayer extends UserLayer { function makeShaderCodeWidget(layer: SingleMeshUserLayer) { return new ShaderCodeWidget({ - fragmentMain: layer.displayState.fragmentMain, + fragment: layer.displayState.fragmentMain, shaderError: layer.displayState.shaderError, shaderControlState: layer.displayState.shaderControlState, }); diff --git a/src/mesh/frontend.ts b/src/mesh/frontend.ts index bd6ded8562..3f6ff9f758 100644 --- a/src/mesh/frontend.ts +++ b/src/mesh/frontend.ts @@ -43,6 +43,10 @@ import type { import { PerspectiveViewRenderLayer } from "#src/perspective_view/render_layer.js"; import type { ThreeDimensionalRenderLayerAttachmentState } from "#src/renderlayer.js"; import { update3dRenderLayerAttachment } from "#src/renderlayer.js"; +import { + encodeSegmentPropertyShaderDefinition, + SegmentColorShaderManager, +} from "#src/segment_color.js"; import { forEachVisibleSegment, getObjectKey, @@ -54,9 +58,11 @@ import { SegmentationLayerSharedObject, } from "#src/segmentation_display_state/frontend.js"; import type { WatchableValueInterface } from "#src/trackable_value.js"; -import { makeCachedDerivedWatchableValue } from "#src/trackable_value.js"; +import { + AggregateWatchableValue, + makeCachedDerivedWatchableValue, +} from "#src/trackable_value.js"; import type { Borrowed, RefCounted } from "#src/util/disposable.js"; -import type { vec4 } from "#src/util/geom.js"; import { getFrustumPlanes, mat3, @@ -68,8 +74,10 @@ import { import * as matrix from "#src/util/matrix.js"; import { GLBuffer } from "#src/webgl/buffer.js"; import type { GL } from "#src/webgl/context.js"; +import type { WatchableShaderError } from "#src/webgl/dynamic_shader.js"; import { parameterizedEmitterDependentShaderGetter } from "#src/webgl/dynamic_shader.js"; import type { ShaderBuilder, ShaderProgram } from "#src/webgl/shader.js"; +import { glsl_uint64 } from "#src/webgl/shader_lib.js"; import type { RPC } from "#src/worker_rpc.js"; import { registerSharedObjectOwner } from "#src/worker_rpc.js"; @@ -179,7 +187,7 @@ function getFloatPositionHandler( defineShader: (builder: ShaderBuilder) => { builder.addAttribute("highp vec3", "aVertexPosition"); builder.addVertexCode( - "highp vec3 getVertexPosition() { return aVertexPosition; }", + "highp vec3 getVertexPosition() { return aVertexPosition; }\n", ); }, bind( @@ -265,14 +273,18 @@ export class MeshShaderManager { } } - setColor(gl: GL, shader: ShaderProgram, color: vec4) { - gl.uniform4fv(shader.uniform("uColor"), color); - } - setPickID(gl: GL, shader: ShaderProgram, pickID: number) { gl.uniform1ui(shader.uniform("uPickID"), pickID); } + setID(gl: GL, shader: ShaderProgram, id: bigint) { + gl.uniform2ui( + shader.uniform(`uID`), + Number(id & 0xffffffffn), + Number(id >> 32n), + ); + } + beginModel( gl: GL, shader: ShaderProgram, @@ -343,30 +355,56 @@ export class MeshShaderManager { this.drawFragmentHelper(gl, shader, fragmentChunk, indexBegin, indexEnd); } - endLayer(gl: GL, shader: ShaderProgram) { + endLayer(gl: GL, shader: ShaderProgram, displayState: MeshDisplayState) { + displayState.segmentationColorUserShader.disable(gl, shader); this.vertexPositionHandler.endLayer(gl, shader); gl.disableVertexAttribArray(shader.attribute("aVertexNormal")); } makeGetter(layer: RefCounted & { gl: GL; displayState: MeshDisplayState }) { - const silhouetteRenderingEnabled = layer.registerDisposer( - makeCachedDerivedWatchableValue( - (x) => x > 0, - [layer.displayState.silhouetteRendering], - ), + const parameters = layer.registerDisposer( + new AggregateWatchableValue((refCounted) => ({ + segmentColorParameters: + layer.displayState.segmentationColorUserShader.shaderParameters, + segmentColorProperties: + layer.displayState.segmentationColorUserShader.usedProperties, + shaderBuilderState: + layer.displayState.segmentColorShaderControlState.builderState, + silhouetteRenderingEnabled: refCounted.registerDisposer( + makeCachedDerivedWatchableValue( + (x) => x > 0, + [layer.displayState.silhouetteRendering], + ), + ), + })), ); + return parameterizedEmitterDependentShaderGetter(layer, layer.gl, { memoizeKey: `mesh/MeshShaderManager/${this.fragmentRelativeVertices}/${this.vertexPositionFormat}`, - parameters: silhouetteRenderingEnabled, - defineShader: (builder, silhouetteRenderingEnabled) => { + parameters, + encodeParameters: (p) => { + return `${p.shaderBuilderState.key}/${JSON.stringify(p.segmentColorParameters)}/${JSON.stringify(p.segmentColorProperties.map(encodeSegmentPropertyShaderDefinition))}/${p.silhouetteRenderingEnabled}`; + }, + shaderError: layer.displayState.shaderError, + defineShader: ( + builder, + { shaderBuilderState, silhouetteRenderingEnabled }, + ) => { this.vertexPositionHandler.defineShader(builder); + layer.displayState.segmentationColorUserShader.defineShader( + builder, + /*fragment=*/ false, + shaderBuilderState, + ); builder.addAttribute("highp vec2", "aVertexNormal"); builder.addVarying("highp vec4", "vColor"); builder.addUniform("highp vec4", "uLightDirection"); - builder.addUniform("highp vec4", "uColor"); builder.addUniform("highp mat3", "uNormalMatrix"); builder.addUniform("highp mat4", "uModelViewProjection"); builder.addUniform("highp uint", "uPickID"); + builder.addUniform("highp uvec2", "uID"); + builder.addUniform("highp float", "uAlpha"); + if (silhouetteRenderingEnabled) { builder.addUniform("highp float", "uSilhouettePower"); } @@ -375,6 +413,7 @@ export class MeshShaderManager { builder.addUniform("highp vec3", "uFragmentShape"); } builder.addVertexCode(glsl_decodeNormalOctahedronSnorm8); + builder.addVertexCode(glsl_uint64); let vertexMain = ""; if (this.fragmentRelativeVertices) { vertexMain += ` @@ -393,7 +432,12 @@ vec3 origNormal = decodeNormalOctahedronSnorm8(aVertexNormal); vec3 normal = normalize(uNormalMatrix * (normalMultiplier * origNormal)); float absCosAngle = abs(dot(normal, uLightDirection.xyz)); float lightingFactor = absCosAngle + uLightDirection.w; -vColor = vec4(lightingFactor * uColor.rgb, uColor.a); +vColor = segmentColorUserShader(uint64_t(uID)); +float alpha = uAlpha; +if (vColor.a >= 0.0) { + alpha *= vColor.a; +} +vColor = vec4(lightingFactor * vColor.rgb, alpha); `; if (silhouetteRenderingEnabled) { vertexMain += ` @@ -401,6 +445,8 @@ vColor *= pow(1.0 - absCosAngle, uSilhouettePower); `; } builder.setVertexMain(vertexMain); + const shaderManager = new SegmentColorShaderManager("getColor"); + shaderManager.defineShader(builder); builder.setFragmentMain("emit(vColor, uPickID);"); }, }); @@ -409,6 +455,7 @@ vColor *= pow(1.0 - absCosAngle, uSilhouettePower); export interface MeshDisplayState extends SegmentationDisplayState3D { silhouetteRendering: WatchableValueInterface; + shaderError: WatchableShaderError; } export class MeshLayer extends PerspectiveViewRenderLayer { @@ -490,11 +537,16 @@ export class MeshLayer extends PerspectiveViewRenderLayer { + (objectId, _color, pickIndex) => { const key = getObjectKey(objectId); const manifestChunk = manifestChunks.get(key); ++totalChunks; if (manifestChunk === undefined) return; ++presentChunks; if (renderContext.emitColor) { - meshShaderManager.setColor(gl, shader, color!); + gl.uniform1f( + shader.uniform("uAlpha"), + displayState.objectAlpha.value, + ); } if (renderContext.emitPickID) { meshShaderManager.setPickID(gl, shader, pickIndex!); } + meshShaderManager.setID(gl, shader, objectId); totalChunks += manifestChunk.fragmentIds.length; for (const fragmentId of manifestChunk.fragmentIds) { @@ -550,7 +606,7 @@ export class MeshLayer extends PerspectiveViewRenderLayer { + (objectId, _color, pickIndex) => { const key = getObjectKey(objectId); const manifestChunk = chunks.get(key); ++totalManifestChunks; @@ -895,11 +956,16 @@ export class MultiscaleMeshLayer extends PerspectiveViewRenderLayer> 24; value.r = float((uintValue & 0x0000ffu)) / 255.0; value.g = float((uintValue & 0x00ff00u) >> 8) / 255.0; value.b = float((uintValue & 0xff0000u) >> 16) / 255.0; - value.a = float((uintValue & 0xff000000u) >> 24) / 255.0; + value.a = alpha == 0u ? -1.0 : float(alpha) / 255.0; return true; } return false; } `; - builder.addFragmentCode(s); + addCode(s); } get getFunctionName() { @@ -179,3 +221,781 @@ bool ${this.getFunctionName}(uint64_t x, out vec4 value) { this.hashMapShaderManager.disable(gl, shader); } } + +interface SegmentPropertyShaderData { + texture: WebGLTexture; + dataType: DataType; + sourceValues: unknown; + stringLiteralIds: ShaderStringLiteralIdMap | undefined; +} + +export interface SegmentPropertyShaderDefinition { + identifier: string; + dataType: DataType; +} + +export function encodeSegmentPropertyShaderDefinition( + definition: SegmentPropertyShaderDefinition, +) { + return `${definition.identifier}:${definition.dataType}`; +} + +interface SegmentPropertyReferencesResult { + segmentProperties: SegmentPropertyReference[]; + allPropertiesFound: boolean; + errors: SegmentPropertyReferenceError[]; +} + +interface SegmentPropertyReferenceError { + line: number; + message: string; +} + +const segmentPropertyHelperCallPattern = + /\b(tag|prop)\s*\(\s*string_t\(\s*(\d+)u\s*\)\s*\)/g; + +function getLineNumberAtIndex(code: string, index: number) { + return Math.max(0, code.substring(0, index).split("\n").length - 1); +} + +export interface SegmentationColorUserShaderManagerParameters { + hasSegmentDefaultColor: boolean; + hasSegmentStatedColors: boolean; +} + +export interface SegmentationColorUserShaderManagerInputs { + segmentDefaultColor: vec3 | vec4 | undefined; + segmentStatedColors: Uint64Map; +} + +export class SegmentColorUserShaderManager extends RefCounted { + protected segmentColorShaderManager = new SegmentColorShaderManager( + "segmentColorHash", + ); + protected hashMapManager = new HashMapShaderManager("SegmentToPropertyIndex"); + protected segmentStatedColorShaderManager = + new SegmentStatedColorShaderManager("segmentStatedColor"); + + private gpuSegmentStatedColorHashTable: + | GPUHashTable + | undefined; + + public shaderParameters: AggregateWatchableValue; + public usedProperties: WatchableValueInterface< + readonly SegmentPropertyShaderDefinition[] + >; + + private segmentPropertyIndexMap = new HashMapUint64(); + private segmentPropertyIndexMapSource: + | PreprocessedSegmentPropertyMap + | undefined; + private segmentPropertyShaderData = new Map< + string, + SegmentPropertyShaderData + >(); + + constructor( + private displayState: SegmentationDisplayState, + private gl: GL, + ) { + super(); + this.shaderParameters = this.registerDisposer( + new AggregateWatchableValue((refCounted) => ({ + hasSegmentDefaultColor: refCounted.registerDisposer( + makeCachedDerivedWatchableValue( + (segmentDefaultColor) => { + return segmentDefaultColor !== undefined; + }, + [displayState.segmentDefaultColor], + ), + ), + hasSegmentStatedColors: refCounted.registerDisposer( + makeCachedDerivedWatchableValue( + (segmentStatedColors: Uint64Map) => { + return segmentStatedColors.size !== 0; + }, + [displayState.segmentStatedColors], + ), + ), + })), + ); + + this.usedProperties = this.registerDisposer( + makeCachedLazyDerivedWatchableValue( + (builderState, segmentPropertyMap) => { + const { segmentProperties } = this.getSegmentPropertyReferences( + builderState, + segmentPropertyMap, + ); + const { definitions } = this.getSegmentPropertyShaderDefinitions( + segmentProperties, + segmentPropertyMap, + ); + return definitions; + }, + this.displayState.segmentColorShaderControlState.builderState, + this.displayState.segmentationGroupState.value.segmentPropertyMap, + ), + ); + } + + private getSegmentPropertyMap() { + return this.displayState.segmentationGroupState.value.segmentPropertyMap + .value; + } + + private getDefaultInputs(): SegmentationColorUserShaderManagerInputs { + return { + segmentDefaultColor: this.displayState.segmentDefaultColor.value, + segmentStatedColors: this.displayState.segmentStatedColors.value, + }; + } + + private getSegmentPropertyShaderDefinition( + prop: SegmentPropertyReference, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + ): SegmentPropertyShaderDefinition | undefined { + if (prop.type === "tag") { + const { tags } = segmentPropertyMap; + if (!tags) return undefined; + const index = tags.tags.indexOf(prop.id); + return index === -1 + ? undefined + : { identifier: `tag${index}`, dataType: DataType.UINT8 }; + } + if (prop.type === "numerical") { + const index = segmentPropertyMap.numericalProperties.findIndex( + (p) => p.id === prop.id, + ); + if (index === -1) return undefined; + return { + identifier: `numerical${index}`, + dataType: segmentPropertyMap.numericalProperties[index].dataType, + }; + } + const index = segmentPropertyMap.strings.findIndex((p) => p.id === prop.id); + return index === -1 + ? undefined + : { identifier: `string${index}`, dataType: DataType.UINT8 }; + } + + private getSegmentPropertyReferenceForHelperCall( + helperName: string, + id: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + ): SegmentPropertyReference | undefined { + if (helperName === "tag") { + return segmentPropertyMap.tags?.tags.includes(id) + ? { type: "tag", id } + : undefined; + } + if (segmentPropertyMap.numericalProperties.some((p) => p.id === id)) { + return { type: "numerical", id }; + } + if (segmentPropertyMap.strings.some((p) => p.id === id)) { + return { type: "string", id }; + } + return undefined; + } + + private getSegmentPropertyReferences( + builderState: ShaderControlsBuilderState, + segmentPropertyMap: PreprocessedSegmentPropertyMap | undefined, + ): SegmentPropertyReferencesResult { + const segmentProperties = [...builderState.segmentProperties]; + const errors: SegmentPropertyReferenceError[] = []; + let allPropertiesFound = true; + const stringLiteralValues = new Map( + [...builderState.parseResult.preprocessing.stringLiteralIds].map( + ([value, id]) => [id, value], + ), + ); + const addReference = (reference: SegmentPropertyReference) => { + if ( + !segmentProperties.some( + (x) => x.type === reference.type && x.id === reference.id, + ) + ) { + segmentProperties.push(reference); + } + }; + for (const match of builderState.parseResult.code.matchAll( + segmentPropertyHelperCallPattern, + )) { + const helperName = match[1]; + const line = getLineNumberAtIndex( + builderState.parseResult.code, + match.index ?? 0, + ); + const propertyId = stringLiteralValues.get(Number(match[2])); + if (propertyId === undefined) { + errors.push({ + line, + message: `'${helperName}(unknown)' : property string literal does not exist`, + }); + continue; + } + if (segmentPropertyMap === undefined) { + allPropertiesFound = false; + continue; + } + const reference = this.getSegmentPropertyReferenceForHelperCall( + helperName, + propertyId, + segmentPropertyMap, + ); + if (reference === undefined) { + errors.push({ + line, + message: + helperName === "tag" + ? `'tag(${JSON.stringify(propertyId)})' : tag does not exist` + : `'prop(${JSON.stringify(propertyId)})' : property does not exist`, + }); + continue; + } + addReference(reference); + } + return { segmentProperties, allPropertiesFound, errors }; + } + + private rewriteSegmentPropertyHelperCalls( + code: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + stringLiteralIds: ShaderStringLiteralIdMap, + ) { + const stringLiteralValues = new Map( + [...stringLiteralIds].map(([value, id]) => [id, value]), + ); + return code.replace( + segmentPropertyHelperCallPattern, + (match, helperName: string, literalId: string) => { + const propertyId = stringLiteralValues.get(Number(literalId)); + if (propertyId === undefined) return match; + const reference = this.getSegmentPropertyReferenceForHelperCall( + helperName, + propertyId, + segmentPropertyMap, + ); + if (reference === undefined) return match; + const definition = this.getSegmentPropertyShaderDefinition( + reference, + segmentPropertyMap, + ); + if (definition === undefined) return match; + switch (reference.type) { + case "tag": + return `(${definition.identifier} == 1u)`; + case "string": + return `string_t(${definition.identifier})`; + case "numerical": + return definition.identifier; + } + }, + ); + } + + private getSegmentPropertyShaderDefinitions( + segmentProperties: SegmentPropertyReference[], + segmentPropertyMap: PreprocessedSegmentPropertyMap | undefined, + ) { + const definitions = new Map(); + let allPropertiesFound = true; + if (segmentProperties.length !== 0 && segmentPropertyMap === undefined) { + allPropertiesFound = false; + } + if (segmentPropertyMap !== undefined) { + for (const prop of segmentProperties) { + const definition = this.getSegmentPropertyShaderDefinition( + prop, + segmentPropertyMap, + ); + if (definition === undefined) { + allPropertiesFound = false; + continue; + } + definitions.set(definition.identifier, definition); + } + } + return { definitions: [...definitions.values()], allPropertiesFound }; + } + + private updateSegmentPropertyIndexMap( + segmentPropertyMap: PreprocessedSegmentPropertyMap | undefined, + ) { + if (this.segmentPropertyIndexMapSource === segmentPropertyMap) return; + this.segmentPropertyIndexMap.clear(); + this.segmentPropertyIndexMapSource = segmentPropertyMap; + const inlineProperties = + segmentPropertyMap?.segmentPropertyMap.inlineProperties; + if (inlineProperties === undefined) return; + for (let i = 0; i < inlineProperties.ids.length; i++) { + this.segmentPropertyIndexMap.set(inlineProperties.ids[i], BigInt(i)); + } + } + + private deleteSegmentPropertyTexture(identifier: string) { + const data = this.segmentPropertyShaderData.get(identifier); + if (data === undefined) return; + this.gl.deleteTexture(data.texture); + this.segmentPropertyShaderData.delete(identifier); + } + + private updateShaderData( + identifier: string, + values: TypedNumberArray, + dataType: DataType, + sourceValues: unknown = values, + stringLiteralIds?: ShaderStringLiteralIdMap, + ) { + const existing = this.segmentPropertyShaderData.get(identifier); + if ( + existing !== undefined && + existing.dataType === dataType && + existing.sourceValues === sourceValues && + existing.stringLiteralIds === stringLiteralIds + ) { + return; + } + this.deleteSegmentPropertyTexture(identifier); + this.segmentPropertyShaderData.set(identifier, { + ...createSegmentPropertyTextureData(values, this.gl, dataType), + sourceValues, + stringLiteralIds, + }); + } + + private tagPropertyToShaderData( + id: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + ) { + const { tags } = segmentPropertyMap; + if (!tags) return; + const index = tags.tags.indexOf(id); + if (index === -1) return; + const { values } = tags; + const propertyShaderIdentifier = `tag${index}`; + const codeUnit = String.fromCharCode(index); + const valuesForTag = values.map((x) => (x.includes(codeUnit) ? 1 : 0)); + this.updateShaderData( + propertyShaderIdentifier, + new Uint8Array(valuesForTag), + DataType.UINT8, + values, + ); + return propertyShaderIdentifier; + } + + private numericPropertyToShaderData( + id: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + ) { + const { numericalProperties } = segmentPropertyMap; + const index = numericalProperties.findIndex((p) => p.id === id); + if (index === -1) return; + const property = numericalProperties[index]; + const propertyShaderIdentifier = `numerical${index}`; + this.updateShaderData( + propertyShaderIdentifier, + property.values, + property.dataType, + ); + return propertyShaderIdentifier; + } + + private stringPropertyToShaderData( + id: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + stringLiteralIds: ShaderStringLiteralIdMap, + ) { + const { strings } = segmentPropertyMap; + const index = strings.findIndex((p) => p.id === id); + if (index === -1) return; + const property = strings[index]; + const propertyShaderIdentifier = `string${index}`; + this.updateShaderData( + propertyShaderIdentifier, + new Uint8Array(property.values.map((x) => stringLiteralIds.get(x) ?? 0)), + DataType.UINT8, + property.values, + stringLiteralIds, + ); + return propertyShaderIdentifier; + } + + private updateSegmentPropertyTexture( + prop: SegmentPropertyReference, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + stringLiteralIds: ShaderStringLiteralIdMap, + ) { + switch (prop.type) { + case "tag": + return this.tagPropertyToShaderData(prop.id, segmentPropertyMap); + case "numerical": + return this.numericPropertyToShaderData(prop.id, segmentPropertyMap); + case "string": + return this.stringPropertyToShaderData( + prop.id, + segmentPropertyMap, + stringLiteralIds, + ); + } + } + + private updateSegmentPropertyTextures( + builderState: ShaderControlsBuilderState, + ) { + const activeIdentifiers = new Set(); + const { parseResult } = builderState; + const segmentPropertyMap = this.getSegmentPropertyMap(); + const { segmentProperties } = this.getSegmentPropertyReferences( + builderState, + segmentPropertyMap, + ); + this.updateSegmentPropertyIndexMap(segmentPropertyMap); + if (segmentPropertyMap !== undefined) { + const { + preprocessing: { stringLiteralIds }, + } = parseResult; + for (const prop of segmentProperties) { + const identifier = this.updateSegmentPropertyTexture( + prop, + segmentPropertyMap, + stringLiteralIds, + ); + if (typeof identifier === "string") { + activeIdentifiers.add(identifier); + } + } + } + for (const identifier of this.segmentPropertyShaderData.keys()) { + if (!activeIdentifiers.has(identifier)) { + this.deleteSegmentPropertyTexture(identifier); + } + } + return activeIdentifiers; + } + + private getMappedIdColor( + builder: ShaderBuilder, + fragment: boolean, + shaderParameters: SegmentationColorUserShaderManagerParameters, + ) { + const { hasSegmentStatedColors, hasSegmentDefaultColor } = shaderParameters; + let getMappedIdColor = `vec4 getMappedIdColor(uint64_t value, out bool isStated) { +`; + if (hasSegmentStatedColors) { + this.segmentStatedColorShaderManager.defineShader(builder, fragment); + getMappedIdColor += ` + vec4 rgba; + if (${this.segmentStatedColorShaderManager.getFunctionName}(value, rgba)) { + isStated = true; + return rgba; + } +`; + } + if (hasSegmentDefaultColor) { + builder.addUniform("highp vec4", "uSegmentDefaultColor"); + getMappedIdColor += ` return uSegmentDefaultColor; +`; + } else { + this.segmentColorShaderManager.defineShader(builder, fragment); + getMappedIdColor += ` return vec4(segmentColorHash(value), -1.0); +`; + } + getMappedIdColor += ` +} +`; + return getMappedIdColor; + } + + defineShader( + builder: ShaderBuilder, + fragment: boolean, + builderState: ShaderControlsBuilderState = this.displayState + .segmentColorShaderControlState.builderState.value, + shaderParameters: SegmentationColorUserShaderManagerParameters = this + .shaderParameters.value, + ) { + builder.addUniform("highp float", "uSaturation"); + builder.addUniform("bool", "uHasSelectedSegment"); + builder.addUniform("highp uvec2", "uSelectedSegment"); + const addCode = fragment + ? builder.addFragmentCode.bind(builder) + : builder.addVertexCode.bind(builder); + const { hashMapManager } = this; + hashMapManager.defineShader(builder, fragment); + addCode(this.getMappedIdColor(builder, fragment, shaderParameters)); + const { parseResult } = builderState; + const segmentPropertyMap = this.getSegmentPropertyMap(); + const { + segmentProperties, + allPropertiesFound: allReferencedPropertiesFound, + errors, + } = this.getSegmentPropertyReferences(builderState, segmentPropertyMap); + const { definitions, allPropertiesFound } = + this.getSegmentPropertyShaderDefinitions( + segmentProperties, + segmentPropertyMap, + ); + let userCode = + allReferencedPropertiesFound && allPropertiesFound + ? parseResult.code + : ""; + if (errors.length !== 0 && segmentPropertyMap !== undefined) { + userCode = `${errors + .map(({ line, message }) => `#line ${line} 1\n#error ${message}\n`) + .join("")} +bool tag(string_t value) { + return false; +} +highp uint prop(string_t value) { + return 0u; +} +#line 0 1 +${parseResult.code}`; + } + if (userCode && segmentPropertyMap !== undefined) { + userCode = this.rewriteSegmentPropertyHelperCalls( + userCode, + segmentPropertyMap, + parseResult.preprocessing.stringLiteralIds, + ); + } + if (userCode) { + addCode(glsl_COLORMAPS); + if (parseResult.preprocessing.stringLiteralIds.size !== 0) { + addCode(glsl_string); + } + for (const { identifier, dataType } of definitions) { + const accessHelper = new OneDimensionalTextureAccessHelper( + `segmentproperty_${identifier}`, + ); + builder.addTextureSampler( + `${getSamplerPrefixForDataType(dataType)}sampler2D`, + `${identifier}_sampler`, + Symbol.for(identifier), + ); + accessHelper.defineShader(builder); + addCode( + accessHelper.getAccessor( + `${identifier}_read`, + `${identifier}_sampler`, + dataType, + ), + ); + addCode(`highp ${getShaderOutputType(dataType)} ${identifier};\n`); + } + addControlsToBuilder(builderState, builder, fragment); + const loadSegmentPropertiesCode = ` +bool loadSegmentProperties(uint64_t id) { + uint64_t propertyIndex_64; + if (!${hashMapManager.getFunctionName}(id, propertyIndex_64)) { + return false; + } + uint propertyIndex = propertyIndex_64.value[0]; + ${definitions + .map(({ identifier, dataType }) => { + return ` + ${identifier} = ${identifier}_read(propertyIndex)${dataType === DataType.FLOAT32 ? "" : ".value"}; +`; + }) + .join("\n")} + return true; +}`; + addCode(loadSegmentPropertiesCode); + addCode(shaderCodeWithLineDirective(userCode)); + if (userCode.includes("vec3 segmentColor(")) { + addCode(` +vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + return vec4(segmentColor(color.rgb, hasProperties, isStated), color.a); +}`); + } + } + addCode(` +vec4 segmentColorUserShader(uint64_t segmentId, float adjustment) { + float alpha = -1.0; // negative = undefined + bool isStated = false; + vec4 color = getMappedIdColor(segmentId, isStated); + alpha = color.a; + float saturation = uSaturation; + if (uHasSelectedSegment && uSelectedSegment == segmentId.value) { + if (saturation > adjustment) { + saturation -= adjustment; + } else { + saturation += adjustment; + } + } +${ + userCode + ? ` + bool hasProperties = loadSegmentProperties(segmentId); + color = segmentColor(color, hasProperties, isStated); +` + : "" +} + return vec4(mix(vec3(1.0,1.0,1.0), vec3(color), saturation), color.a); +} +vec4 segmentColorUserShader(uint64_t segmentId) { + return segmentColorUserShader(segmentId, 0.5); +} +`); + } + + enable( + gl: GL, + shader: ShaderProgram, + builderState: ShaderControlsBuilderState, + shaderParameters: SegmentationColorUserShaderManagerParameters = this + .shaderParameters.value, + inputs: SegmentationColorUserShaderManagerInputs = this.getDefaultInputs(), + ) { + const { parseResult } = builderState; + const { displayState } = this; + let selectedSegmentLow = 0; + let selectedSegmentHigh = 0; + const { segmentSelectionState } = this.displayState; + const hasSelectedSegment = + segmentSelectionState.hasSelectedSegment && + displayState.hoverHighlight.value; + gl.uniform1ui( + shader.uniform("uHasSelectedSegment"), + hasSelectedSegment ? 1 : 0, + ); + if (hasSelectedSegment) { + const seg = displayState.baseSegmentHighlighting.value + ? segmentSelectionState.baseSelectedSegment + : segmentSelectionState.selectedSegment; + selectedSegmentLow = Number(seg & 0xffffffffn); + selectedSegmentHigh = Number(seg >> 32n); + // only update when we have a selected segment since we ignore when uHasSelectedSegment is false + gl.uniform2ui( + shader.uniform("uSelectedSegment"), + selectedSegmentLow, + selectedSegmentHigh, + ); + } + gl.uniform1f(shader.uniform("uSaturation"), displayState.saturation.value); + const { hasSegmentDefaultColor, hasSegmentStatedColors } = shaderParameters; + if (hasSegmentDefaultColor) { + const { segmentDefaultColor } = inputs; + if (segmentDefaultColor) { + const [r, g, b, a] = segmentDefaultColor; + gl.uniform4f( + shader.uniform("uSegmentDefaultColor"), + r, + g, + b, + a === undefined ? -1.0 : a, + ); + } + } else { + const { + segmentColorHash: { value: segmentColorHash }, + } = displayState; + this.segmentColorShaderManager.enable(gl, shader, segmentColorHash); + } + setControlsInShader( + gl, + shader, + this.displayState.segmentColorShaderControlState, + parseResult, + ); + const activeSegmentPropertyIdentifiers = + this.updateSegmentPropertyTextures(builderState); + this.hashMapManager.enable( + gl, + shader, + GPUHashTable.get(this.gl, this.segmentPropertyIndexMap), + ); + for (const identifier of activeSegmentPropertyIdentifiers) { + const { texture } = this.segmentPropertyShaderData.get(identifier)!; + const textureUnit = shader.textureUnit(Symbol.for(identifier)); + if (textureUnit !== undefined) { + gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + textureUnit); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, texture); + } + } + if (hasSegmentStatedColors) { + const { segmentStatedColors } = inputs; + let { gpuSegmentStatedColorHashTable } = this; + if ( + gpuSegmentStatedColorHashTable === undefined || + gpuSegmentStatedColorHashTable.hashTable !== + segmentStatedColors.hashTable + ) { + gpuSegmentStatedColorHashTable?.dispose(); + this.gpuSegmentStatedColorHashTable = gpuSegmentStatedColorHashTable = + GPUHashTable.get(gl, segmentStatedColors.hashTable); + } + this.segmentStatedColorShaderManager.enable( + gl, + shader, + gpuSegmentStatedColorHashTable, + ); + } + } + + disable( + gl: GL, + shader: ShaderProgram, + shaderParameters: SegmentationColorUserShaderManagerParameters = this + .shaderParameters.value, + ) { + this.hashMapManager.disable(gl, shader); + const { hasSegmentStatedColors } = shaderParameters; + if (hasSegmentStatedColors) { + this.segmentStatedColorShaderManager.disable(gl, shader); + } + } + + disposed() { + for (const { texture } of this.segmentPropertyShaderData.values()) { + this.gl.deleteTexture(texture); + } + this.segmentPropertyShaderData.clear(); + this.gpuSegmentStatedColorHashTable?.dispose(); + } +} + +function createSegmentPropertyTextureData( + values: TypedNumberArray, + gl: GL, + dataType: DataType, +) { + const texture = gl.createTexture(); + // for now, immediately load the data into the texture + { + const textureFormat = computeTextureFormat( + new TextureFormat(), + dataType, + 1, + ); + gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + gl.tempTextureUnit); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, texture); + setOneDimensionalTextureData(gl, textureFormat, values); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, null); + } + + return { + texture, + dataType, + }; +} + +function getShaderOutputType(ioType: DataType): string { + switch (ioType) { + case DataType.UINT8: + case DataType.UINT16: + case DataType.UINT32: + return "uint"; + case DataType.INT8: + case DataType.INT16: + case DataType.INT32: + return "int"; + case DataType.FLOAT32: + return "float"; + case DataType.UINT64: + return "uint64_t"; + } +} diff --git a/src/segmentation_display_state/frontend.ts b/src/segmentation_display_state/frontend.ts index 8ffdebbad5..e0f08408c9 100644 --- a/src/segmentation_display_state/frontend.ts +++ b/src/segmentation_display_state/frontend.ts @@ -23,7 +23,10 @@ import type { PickIDManager } from "#src/object_picking.js"; import type { WatchableRenderLayerTransform } from "#src/render_coordinate_transform.js"; import type { RenderScaleHistogram } from "#src/render_scale_statistics.js"; import type { RenderLayer } from "#src/renderlayer.js"; -import type { SegmentColorHash } from "#src/segment_color.js"; +import type { + SegmentColorHash, + SegmentColorUserShaderManager, +} from "#src/segment_color.js"; import { getCssColor } from "#src/segment_color.js"; import type { VisibleSegmentsState } from "#src/segmentation_display_state/base.js"; import { @@ -61,6 +64,7 @@ import { kOneVec, vec4 } from "#src/util/geom.js"; import { parseUint64 } from "#src/util/json.js"; import { NullarySignal } from "#src/util/signal.js"; import { withSharedVisibility } from "#src/visibility_priority/frontend.js"; +import type { ShaderControlState } from "#src/webgl/shader_ui_controls.js"; import { makeCopyButton } from "#src/widget/copy_button.js"; import { makeEyeButton } from "#src/widget/eye_button.js"; import { makeFilterButton } from "#src/widget/filter_button.js"; @@ -193,6 +197,7 @@ export interface SegmentationColorGroupState { } export interface SegmentationDisplayState { + layer: SegmentationUserLayer; segmentSelectionState: SegmentSelectionState; saturation: TrackableAlphaValue; hoverHighlight: WatchableValueInterface; @@ -200,10 +205,14 @@ export interface SegmentationDisplayState { baseSegmentHighlighting: WatchableValueInterface; segmentationGroupState: WatchableValueInterface; segmentationColorGroupState: WatchableValueInterface; + segmentColorShaderControlState: ShaderControlState; + segmentationColorUserShader: SegmentColorUserShaderManager; + fragmentSegmentColor: WatchableValueInterface; selectSegment: (id: bigint, pin: boolean | "toggle" | "force-unpin") => void; filterBySegmentLabel: (id: bigint) => void; moveToSegment: (id: bigint) => void; + getShaderSegmentColor: (id: bigint) => Float32Array | undefined; // Indirect properties hideSegmentZero: WatchableValueInterface; @@ -960,6 +969,12 @@ export function registerCallbackWhenSegmentationDisplayStateChanged( c.registerDisposer( colorGroupState.segmentDefaultColor.changed.add(callback), ); + c.registerDisposer( + colorGroupState.tempSegmentDefaultColor2d.changed.add(callback), + ); + c.registerDisposer( + colorGroupState.tempSegmentStatedColors2d.changed.add(callback), + ); }, displayState.segmentationColorGroupState), ); context.registerDisposer(displayState.saturation.changed.add(callback)); @@ -973,6 +988,15 @@ export function registerCallbackWhenSegmentationDisplayStateChanged( context.registerDisposer( displayState.segmentStatedColors.changed.add(callback), ); + context.registerDisposer( + displayState.useTempSegmentStatedColors2d.changed.add(callback), + ); + context.registerDisposer( + displayState.layer.displayState.fragmentSegmentColor.changed.add(callback), + ); + context.registerDisposer( + displayState.segmentColorShaderControlState.changed.add(callback), + ); } export function registerRedrawWhenSegmentationDisplayStateChanged( @@ -1039,29 +1063,31 @@ export function getBaseObjectColor( color.fill(1); return color; } - const colorGroupState = displayState.segmentationColorGroupState.value; - const { segmentStatedColors } = colorGroupState; - let statedColor: bigint | undefined; - if ( - segmentStatedColors.size !== 0 && - (statedColor = colorGroupState.segmentStatedColors.get(objectId)) !== - undefined - ) { - // If displayState maps the ID to a color, use it - color[0] = Number(statedColor & 0x0000ffn) / 255.0; - color[1] = (Number(statedColor & 0x00ff00n) >>> 8) / 255.0; - color[2] = (Number(statedColor & 0xff0000n) >>> 16) / 255.0; - return color; - } - const segmentDefaultColor = colorGroupState.segmentDefaultColor.value; - if (segmentDefaultColor !== undefined) { - color[0] = segmentDefaultColor[0]; - color[1] = segmentDefaultColor[1]; - color[2] = segmentDefaultColor[2]; - return color; - } - colorGroupState.segmentColorHash.compute(color, objectId); - return color; + const { getShaderSegmentColor } = displayState; + return getShaderSegmentColor(objectId) ?? color; + // const colorGroupState = displayState.segmentationColorGroupState.value; + // const { segmentStatedColors } = colorGroupState; + // let statedColor: bigint | undefined; + // if ( + // segmentStatedColors.size !== 0 && + // (statedColor = colorGroupState.segmentStatedColors.get(objectId)) !== + // undefined + // ) { + // // If displayState maps the ID to a color, use it + // color[0] = Number(statedColor & 0x0000ffn) / 255.0; + // color[1] = (Number(statedColor & 0x00ff00n) >>> 8) / 255.0; + // color[2] = (Number(statedColor & 0xff0000n) >>> 16) / 255.0; + // return getShaderSegmentColor(objectId, color); + // } + // const segmentDefaultColor = colorGroupState.segmentDefaultColor.value; + // if (segmentDefaultColor !== undefined) { + // color[0] = segmentDefaultColor[0]; + // color[1] = segmentDefaultColor[1]; + // color[2] = segmentDefaultColor[2]; + // return getShaderSegmentColor(objectId, color); + // } + // colorGroupState.segmentColorHash.compute(color, objectId); // no need to do hash if we do it in the shader + // return getShaderSegmentColor(objectId, color); } /** @@ -1075,24 +1101,25 @@ export function getObjectColor( const color = tempColor; color[3] = alpha; getBaseObjectColor(displayState, objectId, color); - let saturation = displayState.saturation.value; - if ( - displayState.hoverHighlight.value && - displayState.segmentSelectionState.isSelected(objectId) - ) { - if (saturation > 0.5) { - saturation = saturation -= 0.5; - } else { - saturation += 0.5; - } - } - for (let i = 0; i < 3; ++i) { - color[i] = color[i] * saturation + (1 - saturation); - } - - color[0] *= alpha; - color[1] *= alpha; - color[2] *= alpha; + // TODO move saturation to color shader + // let saturation = displayState.saturation.value; + // if ( + // displayState.hoverHighlight.value && + // displayState.segmentSelectionState.isSelected(objectId) + // ) { + // if (saturation > 0.5) { + // saturation = saturation -= 0.5; + // } else { + // saturation += 0.5; + // } + // } + // for (let i = 0; i < 3; ++i) { + // color[i] = color[i] * saturation + (1 - saturation); + // } + + // color[0] *= alpha; + // color[1] *= alpha; + // color[2] *= alpha; return color; } @@ -1156,19 +1183,21 @@ export function forEachVisibleSegmentToDraw( rootObjectId: bigint, ) => void, ) { - const alpha = Math.min(1, displayState.objectAlpha.value); - const baseSegmentColoring = displayState.baseSegmentColoring.value; + emitColor; + // const alpha = Math.min(1, displayState.objectAlpha.value); + // const baseSegmentColoring = displayState.baseSegmentColoring.value; forEachVisibleSegment( displayState.segmentationGroupState.value, (objectId, rootObjectId) => { const pickIndex = pickIDs?.registerUint64(renderLayer, objectId); - const color = emitColor - ? getObjectColor( - displayState, - baseSegmentColoring ? objectId : rootObjectId, - alpha, - ) - : undefined; + const color = undefined; + // const color = emitColor + // ? getObjectColor( + // displayState, + // baseSegmentColoring ? objectId : rootObjectId, + // alpha, + // ) + // : undefined; callback(objectId, color, pickIndex, rootObjectId); }, ); diff --git a/src/segmentation_display_state/property_map.ts b/src/segmentation_display_state/property_map.ts index ce50b0e90d..7e046ba097 100644 --- a/src/segmentation_display_state/property_map.ts +++ b/src/segmentation_display_state/property_map.ts @@ -27,7 +27,7 @@ import type { import { mergeSequences } from "#src/util/array.js"; import { bigintCompare } from "#src/util/bigint.js"; import { DataType } from "#src/util/data_type.js"; -import type { Borrowed } from "#src/util/disposable.js"; +import { type Borrowed } from "#src/util/disposable.js"; import { murmurHash3_x86_32Hash64Bits_Bigint } from "#src/util/hash.js"; import { parseUint64 } from "#src/util/json.js"; import type { DataTypeInterval } from "#src/util/lerp.js"; @@ -181,6 +181,7 @@ export class PreprocessedSegmentPropertyMap { tags: InlineSegmentTagsProperty | undefined; labels: InlineSegmentStringProperty | undefined; numericalProperties: InlineSegmentNumericalProperty[]; + strings: InlineSegmentStringProperty[]; getSegmentInlineIndex(id: bigint): number { const { inlineIdToIndex } = this; @@ -196,6 +197,23 @@ export class PreprocessedSegmentPropertyMap { const { inlineProperties } = segmentPropertyMap; if (inlineProperties !== undefined) { this.inlineIdToIndex = makeUint64PermutationHashMap(inlineProperties.ids); + + const hasStringProperty = inlineProperties.properties.some( + (p) => p.type === "string", + ); + if (!hasStringProperty) { + const numIds = inlineProperties.ids.length; + const categories = ["neuron", "glia", "synapse", "unknown"]; + inlineProperties.properties.push({ + id: "debug_category", + type: "string", + description: "Debug injected string property", + values: Array.from( + { length: numIds }, + (_, i) => categories[i % categories.length], + ), + }); + } } this.tags = inlineProperties?.properties.find((p) => p.type === "tags") as | InlineSegmentTagsProperty @@ -206,6 +224,9 @@ export class PreprocessedSegmentPropertyMap { this.numericalProperties = (inlineProperties?.properties.filter( (p) => p.type === "number", ) ?? []) as InlineSegmentNumericalProperty[]; + this.strings = (inlineProperties?.properties.filter( + (p) => p.type === "string", + ) ?? []) as InlineSegmentStringProperty[]; } getSegmentLabel(id: bigint): string | undefined { @@ -296,7 +317,7 @@ function remapNumericalProperty( numMerged: number, toMerged: Uint32Array, ): InlineSegmentNumericalProperty { - const values = new Float32Array(numMerged); + const values = property.values.slice(0, numMerged); values.fill(Number.NaN); remapArray(property.values, values, toMerged); return { ...property, values }; @@ -1079,7 +1100,7 @@ function updatePropertyHistogram( const { values } = property; const [min, max] = bounds as [number, number]; const multiplier = max <= min ? 0 : numBins / (max - min); - const histogram = new Uint32Array(numBins + 2); + const histogram = new Uint32Array(numBins + 2); // TODO here we have 258 entries, are the texture ones 256? const { numericalConstraints } = queryResult!.query as FilterQuery; const constraintIndex = numericalConstraints.findIndex( (c) => c.fieldId === property.id, diff --git a/src/single_mesh/frontend.ts b/src/single_mesh/frontend.ts index 69b640201f..530dd36d83 100644 --- a/src/single_mesh/frontend.ts +++ b/src/single_mesh/frontend.ts @@ -312,7 +312,7 @@ vLightingFactor = abs(dot(normal, uLightDirection.xyz)) + uLightDirection.w; const bindTexture = (texture: WebGLTexture | null) => { const textureUnit = WebGL2RenderingContext.TEXTURE0 + - shader.textureUnit(vertexAttributeSamplerSymbols[index]); + shader.textureUnit(vertexAttributeSamplerSymbols[index])!; gl.activeTexture(textureUnit); gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, texture); ++index; @@ -338,7 +338,7 @@ vLightingFactor = abs(dot(normal, uLightDirection.xyz)) + uLightDirection.w; } for (let i = 0; i < numTextures; ++i) { const curTextureUnit = - shader.textureUnit(vertexAttributeSamplerSymbols[i]) + + shader.textureUnit(vertexAttributeSamplerSymbols[i])! + WebGL2RenderingContext.TEXTURE0; gl.activeTexture(curTextureUnit); gl.bindTexture(gl.TEXTURE_2D, null); diff --git a/src/skeleton/frontend.ts b/src/skeleton/frontend.ts index f001663dd6..fd09bff904 100644 --- a/src/skeleton/frontend.ts +++ b/src/skeleton/frontend.ts @@ -26,6 +26,7 @@ import type { ThreeDimensionalRenderLayerAttachmentState, } from "#src/renderlayer.js"; import { update3dRenderLayerAttachment } from "#src/renderlayer.js"; +import { encodeSegmentPropertyShaderDefinition } from "#src/segment_color.js"; import { forEachVisibleSegment, getObjectKey, @@ -42,10 +43,13 @@ import type { SliceViewPanel } from "#src/sliceview/panel.js"; import type { SliceViewPanelRenderContext } from "#src/sliceview/renderlayer.js"; import { SliceViewPanelRenderLayer } from "#src/sliceview/renderlayer.js"; import { TrackableBoolean } from "#src/trackable_boolean.js"; -import { TrackableValue, WatchableValue } from "#src/trackable_value.js"; +import { + AggregateWatchableValue, + TrackableValue, + WatchableValue, +} from "#src/trackable_value.js"; import { DataType } from "#src/util/data_type.js"; import { RefCounted } from "#src/util/disposable.js"; -import type { vec3 } from "#src/util/geom.js"; import { mat4 } from "#src/util/geom.js"; import { verifyFinitePositiveFloat } from "#src/util/json.js"; import { NullarySignal } from "#src/util/signal.js"; @@ -60,9 +64,9 @@ import { } from "#src/webgl/circles.js"; import { glsl_COLORMAPS } from "#src/webgl/colormaps.js"; import type { GL } from "#src/webgl/context.js"; -import type { WatchableShaderError } from "#src/webgl/dynamic_shader.js"; import { makeTrackableFragmentMain, + makeWatchableShaderError, parameterizedEmitterDependentShaderGetter, shaderCodeWithLineDirective, } from "#src/webgl/dynamic_shader.js"; @@ -77,7 +81,6 @@ import type { ShaderSamplerType, } from "#src/webgl/shader.js"; import { glsl_string } from "#src/webgl/shader_lib.js"; -import type { ShaderControlsBuilderState } from "#src/webgl/shader_ui_controls.js"; import { addControlsToBuilder, getFallbackBuilderState, @@ -126,9 +129,9 @@ class RenderHelper extends RefCounted { defineCommonShader(builder: ShaderBuilder) { defineVertexId(builder); - builder.addUniform("highp vec4", "uColor"); builder.addUniform("highp mat4", "uProjection"); builder.addUniform("highp uint", "uPickID"); + builder.addUniform("highp uvec2", "uID"); } edgeShaderGetter; @@ -143,6 +146,23 @@ class RenderHelper extends RefCounted { public targetIsSliceView: boolean, ) { super(); + + const layer = base; + + const parameters = layer.registerDisposer( + new AggregateWatchableValue(() => ({ + segmentColorParameters: + layer.displayState.segmentationColorUserShader.shaderParameters, + segmentColorProperties: + layer.displayState.segmentationColorUserShader.usedProperties, + segmentColorShaderBuilderState: + layer.displayState.segmentColorShaderControlState.builderState, + skeletonShaderBuilderState: + this.base.displayState.skeletonRenderingOptions.shaderControlState + .builderState, + })), + ); + this.vertexIdHelper = this.registerDisposer(VertexIdHelper.get(this.gl)); this.edgeShaderGetter = parameterizedEmitterDependentShaderGetter( this, @@ -152,18 +172,25 @@ class RenderHelper extends RefCounted { type: "skeleton/SkeletonShaderManager/edge", vertexAttributes: this.vertexAttributes, }, - fallbackParameters: this.base.fallbackShaderParameters, - parameters: - this.base.displayState.skeletonRenderingOptions.shaderControlState - .builderState, - shaderError: this.base.displayState.shaderError, + // fallbackParameters: this.base.fallbackShaderParameters, + parameters, + encodeParameters: (p) => { + return `${p.skeletonShaderBuilderState.key}/${p.segmentColorShaderBuilderState.key}/${JSON.stringify(p.segmentColorParameters)}/${JSON.stringify(p.segmentColorProperties.map(encodeSegmentPropertyShaderDefinition))}}`; + }, + shaderError: + this.base.displayState.skeletonRenderingOptions.shaderError, defineShader: ( builder: ShaderBuilder, - shaderBuilderState: ShaderControlsBuilderState, + { segmentColorShaderBuilderState, skeletonShaderBuilderState }, ) => { - if (shaderBuilderState.parseResult.errors.length !== 0) { + if (skeletonShaderBuilderState.parseResult.errors.length !== 0) { throw new Error("Invalid UI control specification"); } + layer.displayState.segmentationColorUserShader.defineShader( + builder, + /*fragment=*/ true, + segmentColorShaderBuilderState, + ); this.defineCommonShader(builder); this.defineAttributeAccess(builder); defineLineShader(builder); @@ -179,13 +206,17 @@ highp uint vertexIndex = aVertexIndex.x * (1u - lineEndpointIndex) + aVertexInde builder.addFragmentCode(` vec4 segmentColor() { - return uColor; + vec4 res = segmentColorUserShader(uint64_t(uID)); + res.a = 1.0; + return res; } void emitRGB(vec3 color) { - emit(vec4(color * uColor.a, uColor.a * getLineAlpha() * ${this.getCrossSectionFadeFactor()}), uPickID); + float alpha = segmentColor().a; + emit(vec4(color * alpha, alpha * getLineAlpha() * ${this.getCrossSectionFadeFactor()}), uPickID); } void emitDefault() { - emit(vec4(uColor.rgb, uColor.a * getLineAlpha() * ${this.getCrossSectionFadeFactor()}), uPickID); + vec4 color = segmentColor(); + emit(vec4(color.rgb, color.a * getLineAlpha() * ${this.getCrossSectionFadeFactor()}), uPickID); } `); builder.addFragmentCode(glsl_COLORMAPS); @@ -201,10 +232,12 @@ void emitDefault() { ); } builder.setVertexMain(vertexMain); - addControlsToBuilder(shaderBuilderState, builder); builder.addFragmentCode(glsl_string); + addControlsToBuilder(skeletonShaderBuilderState, builder); builder.setFragmentMainFunction( - shaderCodeWithLineDirective(shaderBuilderState.parseResult.code), + shaderCodeWithLineDirective( + skeletonShaderBuilderState.parseResult.code, + ), ); }, }, @@ -218,18 +251,25 @@ void emitDefault() { type: "skeleton/SkeletonShaderManager/node", vertexAttributes: this.vertexAttributes, }, - fallbackParameters: this.base.fallbackShaderParameters, - parameters: - this.base.displayState.skeletonRenderingOptions.shaderControlState - .builderState, - shaderError: this.base.displayState.shaderError, + // fallbackParameters: this.base.fallbackShaderParameters, + parameters, + encodeParameters: (p) => { + return `${p.skeletonShaderBuilderState.key}/${p.segmentColorShaderBuilderState.key}/${JSON.stringify(p.segmentColorParameters)}/${JSON.stringify(p.segmentColorProperties.map(encodeSegmentPropertyShaderDefinition))}}`; + }, + shaderError: + this.base.displayState.skeletonRenderingOptions.shaderError, defineShader: ( builder: ShaderBuilder, - shaderBuilderState: ShaderControlsBuilderState, + { segmentColorShaderBuilderState, skeletonShaderBuilderState }, ) => { - if (shaderBuilderState.parseResult.errors.length !== 0) { + if (skeletonShaderBuilderState.parseResult.errors.length !== 0) { throw new Error("Invalid UI control specification"); } + layer.displayState.segmentationColorUserShader.defineShader( + builder, + /*fragment=*/ true, + segmentColorShaderBuilderState, + ); this.defineCommonShader(builder); this.defineAttributeAccess(builder); defineCircleShader( @@ -245,7 +285,9 @@ emitCircle(uProjection * vec4(vertexPosition, 1.0), uNodeDiameter, 0.0); builder.addFragmentCode(` vec4 segmentColor() { - return uColor; + vec4 res = segmentColorUserShader(uint64_t(uID)); + res.a = 1.0; + return res; } void emitRGBA(vec4 color) { vec4 borderColor = color; @@ -255,7 +297,7 @@ void emitRGB(vec3 color) { emitRGBA(vec4(color, 1.0)); } void emitDefault() { - emitRGBA(uColor); + emitRGBA(segmentColor()); } `); builder.addFragmentCode(glsl_COLORMAPS); @@ -271,10 +313,12 @@ void emitDefault() { ); } builder.setVertexMain(vertexMain); - addControlsToBuilder(shaderBuilderState, builder); builder.addFragmentCode(glsl_string); + addControlsToBuilder(skeletonShaderBuilderState, builder); builder.setFragmentMainFunction( - shaderCodeWithLineDirective(shaderBuilderState.parseResult.code), + shaderCodeWithLineDirective( + skeletonShaderBuilderState.parseResult.code, + ), ); }, }, @@ -328,14 +372,18 @@ void emitDefault() { this.vertexIdHelper.enable(); } - setColor(gl: GL, shader: ShaderProgram, color: vec3) { - gl.uniform4fv(shader.uniform("uColor"), color); - } - setPickID(gl: GL, shader: ShaderProgram, pickID: number) { gl.uniform1ui(shader.uniform("uPickID"), pickID); } + setID(gl: GL, shader: ShaderProgram, id: bigint) { + gl.uniform2ui( + shader.uniform(`uID`), + Number(id & 0xffffffffn), + Number(id >> 32n), + ); + } + drawSkeleton( gl: GL, edgeShader: ShaderProgram, @@ -349,7 +397,7 @@ void emitDefault() { for (let i = 0; i < numAttributes; ++i) { const textureUnit = WebGL2RenderingContext.TEXTURE0 + - edgeShader.textureUnit(vertexAttributeSamplerSymbols[i]); + edgeShader.textureUnit(vertexAttributeSamplerSymbols[i])!; gl.activeTexture(textureUnit); gl.bindTexture( WebGL2RenderingContext.TEXTURE_2D, @@ -389,17 +437,22 @@ void emitDefault() { } } - endLayer(gl: GL, shader: ShaderProgram) { + endLayer( + gl: GL, + shader: ShaderProgram, + displayState: SkeletonLayerDisplayState, + ) { const { vertexAttributes } = this; const numAttributes = vertexAttributes.length; for (let i = 0; i < numAttributes; ++i) { const curTextureUnit = - shader.textureUnit(vertexAttributeSamplerSymbols[i]) + + shader.textureUnit(vertexAttributeSamplerSymbols[i])! + WebGL2RenderingContext.TEXTURE0; gl.activeTexture(curTextureUnit); gl.bindTexture(gl.TEXTURE_2D, null); } this.vertexIdHelper.disable(); + displayState.segmentationColorUserShader.disable(gl, shader); } } @@ -435,6 +488,7 @@ export class SkeletonRenderingOptions implements Trackable { } shader = makeTrackableFragmentMain(DEFAULT_FRAGMENT_MAIN); + shaderError = makeWatchableShaderError(); shaderControlState = new ShaderControlState(this.shader); hideInactiveShaderControls = new TrackableBoolean(false); params2d: ViewSpecificSkeletonRenderingOptions = { @@ -476,7 +530,6 @@ export class SkeletonRenderingOptions implements Trackable { } export interface SkeletonLayerDisplayState extends SegmentationDisplayState3D { - shaderError: WatchableShaderError; skeletonRenderingOptions: SkeletonRenderingOptions; } @@ -501,11 +554,12 @@ export class SkeletonLayer extends RefCounted { super(); registerRedrawWhenSegmentationDisplayState3DChanged(displayState, this); - this.displayState.shaderError.value = undefined; + const { shaderError } = this.displayState.skeletonRenderingOptions; + shaderError.value = undefined; const { skeletonRenderingOptions: renderingOptions } = displayState; this.registerDisposer( renderingOptions.shader.changed.add(() => { - this.displayState.shaderError.value = undefined; + shaderError.value = undefined; this.redrawNeeded.dispatch(); }), ); @@ -593,7 +647,12 @@ export class SkeletonLayer extends RefCounted { gl, edgeShader, shaderControlState, - edgeShaderParameters.parseResult, + edgeShaderParameters.skeletonShaderBuilderState.parseResult, + ); + displayState.segmentationColorUserShader.enable( + gl, + edgeShader, + edgeShaderParameters.segmentColorShaderBuilderState, ); gl.uniform1f(edgeShader.uniform("uLineWidth"), lineWidth!); @@ -604,7 +663,12 @@ export class SkeletonLayer extends RefCounted { gl, nodeShader, shaderControlState, - nodeShaderParameters.parseResult, + nodeShaderParameters.skeletonShaderBuilderState.parseResult, + ); + displayState.segmentationColorUserShader.enable( + gl, + nodeShader, + nodeShaderParameters.segmentColorShaderBuilderState, ); const skeletons = source.chunks; @@ -614,7 +678,7 @@ export class SkeletonLayer extends RefCounted { layer, renderContext.emitColor, renderContext.emitPickID ? renderContext.pickIDs : undefined, - (objectId, color, pickIndex) => { + (objectId, _color, pickIndex) => { const key = getObjectKey(objectId); const skeleton = skeletons.get(key); if ( @@ -623,18 +687,16 @@ export class SkeletonLayer extends RefCounted { ) { return; } - if (color !== undefined) { - edgeShader.bind(); - renderHelper.setColor(gl, edgeShader, (color)); - nodeShader.bind(); - renderHelper.setColor(gl, nodeShader, (color)); - } if (pickIndex !== undefined) { edgeShader.bind(); renderHelper.setPickID(gl, edgeShader, pickIndex); nodeShader.bind(); renderHelper.setPickID(gl, nodeShader, pickIndex); } + edgeShader.bind(); + renderHelper.setID(gl, edgeShader, objectId); + nodeShader.bind(); + renderHelper.setID(gl, nodeShader, objectId); renderHelper.drawSkeleton( gl, edgeShader, @@ -644,7 +706,7 @@ export class SkeletonLayer extends RefCounted { ); }, ); - renderHelper.endLayer(gl, edgeShader); + renderHelper.endLayer(gl, edgeShader, displayState); } isReady() { diff --git a/src/sliceview/single_texture_chunk_format.ts b/src/sliceview/single_texture_chunk_format.ts index 53468aedfd..8509b4ca6e 100644 --- a/src/sliceview/single_texture_chunk_format.ts +++ b/src/sliceview/single_texture_chunk_format.ts @@ -57,7 +57,7 @@ export abstract class SingleTextureChunkFormat abstract get shaderSamplerType(): ShaderSamplerType; beginDrawing(gl: GL, shader: ShaderProgram) { - const textureUnit = shader.textureUnit(textureUnitSymbol); + const textureUnit = shader.textureUnit(textureUnitSymbol)!; gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + textureUnit); (shader)[textureLayoutSymbol] = null; } diff --git a/src/sliceview/volume/segmentation_renderlayer.ts b/src/sliceview/volume/segmentation_renderlayer.ts index c4a6c44b16..f49d323c4c 100644 --- a/src/sliceview/volume/segmentation_renderlayer.ts +++ b/src/sliceview/volume/segmentation_renderlayer.ts @@ -21,8 +21,9 @@ import { HashSetShaderManager, } from "#src/gpu_hash/shader.js"; import { - SegmentColorShaderManager, - SegmentStatedColorShaderManager, + encodeSegmentPropertyShaderDefinition, + type SegmentationColorUserShaderManagerParameters, + type SegmentPropertyShaderDefinition, } from "#src/segment_color.js"; import { getVisibleSegments } from "#src/segmentation_display_state/base.js"; import type { @@ -49,6 +50,7 @@ import { import type { Uint64Map } from "#src/uint64_map.js"; import type { DisjointUint64Sets } from "#src/util/disjoint_sets.js"; import type { ShaderBuilder, ShaderProgram } from "#src/webgl/shader.js"; +import { type ShaderControlsBuilderState } from "#src/webgl/shader_ui_controls.js"; export class EquivalencesHashMap { generation = Number.NaN; @@ -78,25 +80,18 @@ export interface SliceViewSegmentationDisplayState } interface ShaderParameters { + segmentColorParameters: SegmentationColorUserShaderManagerParameters; + segmentColorProperties: readonly SegmentPropertyShaderDefinition[]; hasEquivalences: boolean; baseSegmentColoring: boolean; baseSegmentHighlighting: boolean; - hasSegmentStatedColors: boolean; hideSegmentZero: boolean; - hasSegmentDefaultColor: boolean; hasHighlightColor: boolean; + shaderBuilderState: ShaderControlsBuilderState; } -const HAS_SELECTED_SEGMENT_FLAG = 1; -const SHOW_ALL_SEGMENTS_FLAG = 2; - export class SegmentationRenderLayer extends SliceViewVolumeRenderLayer { public readonly segmentationGroupState: SegmentationGroupState; - protected segmentColorShaderManager = new SegmentColorShaderManager( - "segmentColorHash", - ); - protected segmentStatedColorShaderManager = - new SegmentStatedColorShaderManager("segmentStatedColor"); private gpuSegmentStatedColorHashTable: | GPUHashTable | undefined; @@ -114,47 +109,58 @@ export class SegmentationRenderLayer extends SliceViewVolumeRenderLayer ({ - hasEquivalences: refCounted.registerDisposer( - makeCachedDerivedWatchableValue( - (x) => x.size !== 0, - [displayState.segmentationGroupState.value.segmentEquivalences], + encodeShaderParameters: (p) => { + const { shaderBuilderState, segmentColorProperties, ...rest } = p; + return { + shaderBuilderStateKey: shaderBuilderState.key, + segmentColorProperties: segmentColorProperties.map( + encodeSegmentPropertyShaderDefinition, ), - ), - hasSegmentStatedColors: refCounted.registerDisposer( + ...rest, + }; + }, + shaderError: displayState.shaderError, + shaderParameters: new AggregateWatchableValue((refCounted) => ({ + segmentColorParameters: refCounted.registerDisposer( makeCachedDerivedWatchableValue( ( + segmentDefaultColor, + tempSegmentDefaultColor2d, segmentStatedColors: Uint64Map, tempSegmentStatedColors2d: Uint64Map, useTempSegmentStatedColors2d: boolean, ) => { - const releventMap = useTempSegmentStatedColors2d + const activeSegmentDefaultColor = + tempSegmentDefaultColor2d ?? segmentDefaultColor; + const activeSegmentStatedColors = useTempSegmentStatedColors2d ? tempSegmentStatedColors2d : segmentStatedColors; - return releventMap.size !== 0; + return { + hasSegmentDefaultColor: activeSegmentDefaultColor !== undefined, + hasSegmentStatedColors: activeSegmentStatedColors.size !== 0, + }; }, [ + displayState.segmentDefaultColor, + displayState.tempSegmentDefaultColor2d, displayState.segmentStatedColors, displayState.tempSegmentStatedColors2d, displayState.useTempSegmentStatedColors2d, ], ), ), - hasSegmentDefaultColor: refCounted.registerDisposer( + segmentColorProperties: + displayState.segmentationColorUserShader.usedProperties, + shaderBuilderState: + displayState.segmentColorShaderControlState.builderState, + hasEquivalences: refCounted.registerDisposer( makeCachedDerivedWatchableValue( - (segmentDefaultColor, tempSegmentDefaultColor2d) => { - return ( - segmentDefaultColor !== undefined || - tempSegmentDefaultColor2d !== undefined - ); - }, - [ - displayState.segmentDefaultColor, - displayState.tempSegmentDefaultColor2d, - ], + (x) => x.size !== 0, + [displayState.segmentationGroupState.value.segmentEquivalences], ), ), hasHighlightColor: refCounted.registerDisposer( + // TODO move to segment color user shader manager? it might only apply to 2d makeCachedDerivedWatchableValue( (x) => x !== undefined, [displayState.highlightColor], @@ -222,7 +228,14 @@ export class SegmentationRenderLayer extends SliceViewVolumeRenderLayer adjustment) { - saturation -= adjustment; - } else { - saturation += adjustment; - } + bool has = uShowAllSegments || ${this.hashTableManager.hasFunctionName}(value); + if (uHasSelectedSegment && uSelectedSegment == valueForHighlight.value) { `; if (parameters.hasHighlightColor) { builder.addUniform("highp vec4", "uHighlightColor"); fragmentMain += ` - emit(uHighlightColor); - return; -`; + emit(uHighlightColor); + return; + `; } fragmentMain += ` } else if (!has) { alpha = uNotSelectedAlpha; } + //alpha = 1.0; `; - let getMappedIdColor = `vec4 getMappedIdColor(uint64_t value) { -`; - // If the value has a mapped color, use it; otherwise, compute the color. - if (parameters.hasSegmentStatedColors) { - this.segmentStatedColorShaderManager.defineShader(builder); - getMappedIdColor += ` - vec4 rgba; - if (${this.segmentStatedColorShaderManager.getFunctionName}(value, rgba)) { - return rgba; - } -`; - } - if (parameters.hasSegmentDefaultColor) { - builder.addUniform("highp vec4", "uSegmentDefaultColor"); - getMappedIdColor += ` return uSegmentDefaultColor; -`; - } else { - this.segmentColorShaderManager.defineShader(builder); - getMappedIdColor += ` return vec4(segmentColorHash(value), 0.0); -`; - } - getMappedIdColor += ` -} -`; - builder.addFragmentCode(getMappedIdColor); - fragmentMain += ` - vec4 rgba = getMappedIdColor(valueForColor); - if (rgba.a > 0.0) { + vec4 rgba = segmentColorUserShader(valueForColor, has ? 0.5 : 0.75); + if (rgba.a >= 0.0) { alpha = rgba.a; } - emit(vec4(mix(vec3(1.0,1.0,1.0), vec3(rgba), saturation), alpha)); + emit(vec4(rgba.rgb, alpha)); `; builder.setFragmentMain(fragmentMain); } @@ -340,47 +319,32 @@ uint64_t getMappedObjectId(uint64_t value) { ) { const { gl } = this; const { displayState, segmentationGroupState } = this; - const { segmentSelectionState } = this.displayState; const { segmentDefaultColor: { value: segmentDefaultColor }, - segmentColorHash: { value: segmentColorHash }, + segmentStatedColors: { value: segmentStatedColors }, highlightColor: { value: highlightColor }, tempSegmentDefaultColor2d: { value: tempSegmentDefaultColor2d }, + tempSegmentStatedColors2d: { value: tempSegmentStatedColors2d }, + useTempSegmentStatedColors2d: { value: useTempSegmentStatedColors2d }, } = this.displayState; + const activeSegmentDefaultColor = + tempSegmentDefaultColor2d ?? segmentDefaultColor; + const activeSegmentStatedColors = useTempSegmentStatedColors2d + ? tempSegmentStatedColors2d + : segmentStatedColors; const visibleSegments = getVisibleSegments(segmentationGroupState); const ignoreNullSegmentSet = this.displayState.ignoreNullVisibleSet.value; - let selectedSegmentLow = 0; - let selectedSegmentHigh = 0; - let flags = 0; - if ( - segmentSelectionState.hasSelectedSegment && - displayState.hoverHighlight.value - ) { - const seg = displayState.baseSegmentHighlighting.value - ? segmentSelectionState.baseSelectedSegment - : segmentSelectionState.selectedSegment; - selectedSegmentLow = Number(seg & 0xffffffffn); - selectedSegmentHigh = Number(seg >> 32n); - flags |= HAS_SELECTED_SEGMENT_FLAG; - } gl.uniform1f( shader.uniform("uSelectedAlpha"), displayState.selectedAlpha.value, ); - gl.uniform1f(shader.uniform("uSaturation"), displayState.saturation.value); gl.uniform1f( shader.uniform("uNotSelectedAlpha"), displayState.notSelectedAlpha.value, ); - gl.uniform2ui( - shader.uniform("uSelectedSegment"), - selectedSegmentLow, - selectedSegmentHigh, - ); - if (visibleSegments.hashTable.size === 0 && ignoreNullSegmentSet) { - flags |= SHOW_ALL_SEGMENTS_FLAG; - } - gl.uniform1ui(shader.uniform("uFlags"), flags); + const showAllSegments = + visibleSegments.hashTable.size === 0 && ignoreNullSegmentSet; + gl.uniform1ui(shader.uniform("uShowAllSegments"), showAllSegments ? 1 : 0); this.hashTableManager.enable( gl, shader, @@ -403,58 +367,35 @@ uint64_t getMappedObjectId(uint64_t value) { : this.gpuEquivalencesHashTable, ); } - const activeSegmentDefaultColor = - tempSegmentDefaultColor2d || segmentDefaultColor; - if (activeSegmentDefaultColor) { - const [r, g, b, a] = activeSegmentDefaultColor; - gl.uniform4f( - shader.uniform("uSegmentDefaultColor"), - r, - g, - b, - a === undefined ? 0 : a, - ); - } else { - this.segmentColorShaderManager.enable(gl, shader, segmentColorHash); - } - if (parameters.hasSegmentStatedColors) { - const segmentStatedColors = displayState.useTempSegmentStatedColors2d - .value - ? displayState.tempSegmentStatedColors2d.value - : displayState.segmentStatedColors.value; - let { gpuSegmentStatedColorHashTable } = this; - if ( - gpuSegmentStatedColorHashTable === undefined || - gpuSegmentStatedColorHashTable.hashTable !== - segmentStatedColors.hashTable - ) { - gpuSegmentStatedColorHashTable?.dispose(); - this.gpuSegmentStatedColorHashTable = gpuSegmentStatedColorHashTable = - GPUHashTable.get(gl, segmentStatedColors.hashTable); - } - this.segmentStatedColorShaderManager.enable( - gl, - shader, - gpuSegmentStatedColorHashTable, - ); - } if (highlightColor !== undefined) { gl.uniform4fv(shader.uniform("uHighlightColor"), highlightColor); } + displayState.segmentationColorUserShader.enable( + gl, + shader, + parameters.shaderBuilderState, + parameters.segmentColorParameters, + { + segmentDefaultColor: activeSegmentDefaultColor, + segmentStatedColors: activeSegmentStatedColors, + }, + ); } endSlice( sliceView: SliceView, shader: ShaderProgram, parameters: ShaderParameters, ) { - const { gl } = this; + const { gl, displayState } = this; + displayState.segmentationColorUserShader.disable( + gl, + shader, + parameters.segmentColorParameters, + ); this.hashTableManager.disable(gl, shader); if (parameters.hasEquivalences) { this.equivalencesShaderManager.disable(gl, shader); } - if (parameters.hasSegmentStatedColors) { - this.segmentStatedColorShaderManager.disable(gl, shader); - } super.endSlice(sliceView, shader, parameters); } } diff --git a/src/ui/segmentation_display_options_tab.ts b/src/ui/segmentation_display_options_tab.ts index 9063b3f266..8ca21e0d22 100644 --- a/src/ui/segmentation_display_options_tab.ts +++ b/src/ui/segmentation_display_options_tab.ts @@ -29,10 +29,18 @@ import { import { ShaderControls } from "#src/widget/shader_controls.js"; import { Tab } from "#src/widget/tab_view.js"; -function makeSkeletonShaderCodeWidget(layer: SegmentationUserLayer) { +function makeSegmentColorShaderCodeWidget(layer: SegmentationUserLayer) { return new ShaderCodeWidget({ - fragmentMain: layer.displayState.skeletonRenderingOptions.shader, + fragment: layer.displayState.fragmentSegmentColor, shaderError: layer.displayState.shaderError, + shaderControlState: layer.displayState.segmentColorShaderControlState, + }); +} + +function makeSkeletonShaderCodeWidget(layer: SegmentationUserLayer) { + return new ShaderCodeWidget({ + fragment: layer.displayState.skeletonRenderingOptions.shader, + shaderError: layer.displayState.skeletonRenderingOptions.shaderError, shaderControlState: layer.displayState.skeletonRenderingOptions.shaderControlState, }); @@ -70,6 +78,34 @@ export class DisplayOptionsTab extends Tab { ); } + const segmentColorCodeWidget = this.registerDisposer( + makeSegmentColorShaderCodeWidget(layer), + ); + element.appendChild( + makeShaderCodeWidgetTopRow( + this.layer, + segmentColorCodeWidget, + SegmentColorShaderCodeOverlay, + { + title: "Documentation on segmentation layer rendering", + href: "https://github.com/google/neuroglancer/blob/master/src/layer/segmentation/rendering.md", + }, + "neuroglancer-segmentation-dropdown-segment-color-shader-header", + "Segment Color Shader", + ), + ); + element.appendChild(segmentColorCodeWidget.element); + element.appendChild( + this.registerDisposer( + new ShaderControls( + this.layer.displayState.segmentColorShaderControlState, + this.layer.manager.root.display, + this.layer, + { visibility: this.visibility }, + ), + ).element, + ); + const skeletonControls = this.registerDisposer( new DependentViewWidget( layer.hasSkeletonsLayer, @@ -94,12 +130,13 @@ export class DisplayOptionsTab extends Tab { makeShaderCodeWidgetTopRow( this.layer, codeWidget, - ShaderCodeOverlay, + SkeletonShaderCodeOverlay, { title: "Documentation on image layer rendering", href: "https://github.com/google/neuroglancer/blob/master/src/sliceview/image_layer_rendering.md", }, "neuroglancer-segmentation-dropdown-skeleton-shader-header", + "Skeleton Shader", ), ); parent.appendChild(codeWidget.element); @@ -128,7 +165,22 @@ export class DisplayOptionsTab extends Tab { } } -class ShaderCodeOverlay extends Overlay { +class SegmentColorShaderCodeOverlay extends Overlay { + codeWidget: ShaderCodeWidget; + constructor(public layer: SegmentationUserLayer) { + super(); + this.codeWidget = this.registerDisposer( + makeSegmentColorShaderCodeWidget(layer), + ); + this.content.classList.add( + "neuroglancer-segmentation-layer-segment-color-shader-overlay", + ); + this.content.appendChild(this.codeWidget.element); + this.codeWidget.textEditor.refresh(); + } +} + +class SkeletonShaderCodeOverlay extends Overlay { codeWidget: ShaderCodeWidget; constructor(public layer: SegmentationUserLayer) { super(); diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index d28eb1d5e5..fc4e6dda6e 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -812,7 +812,7 @@ gl_Position = uModelViewProjectionMatrix * vec4(position, 1.0); if (prevChunkFormat !== null) { prevChunkFormat!.endDrawing(gl, shader); } - const depthTextureUnit = shader.textureUnit(depthSamplerTextureUnit); + const depthTextureUnit = shader.textureUnit(depthSamplerTextureUnit)!; gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + depthTextureUnit); gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, null); if (presentCount !== 0 || notPresentCount !== 0) { @@ -1315,7 +1315,7 @@ gl_Position = uModelViewProjectionMatrix * vec4(position, 1.0); renderContext.depthBufferTexture !== undefined && renderContext.depthBufferTexture !== null ) { - const depthTextureUnit = shader.textureUnit(depthSamplerTextureUnit); + const depthTextureUnit = shader.textureUnit(depthSamplerTextureUnit)!; gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + depthTextureUnit); gl.bindTexture( WebGL2RenderingContext.TEXTURE_2D, diff --git a/src/webgl/context.ts b/src/webgl/context.ts index 3da15d46e2..cb8700ff54 100644 --- a/src/webgl/context.ts +++ b/src/webgl/context.ts @@ -27,7 +27,7 @@ export interface GL extends WebGL2RenderingContext { export const DEBUG_SHADERS = false; -export function initializeWebGL(canvas: HTMLCanvasElement) { +export function initializeWebGL(canvas: HTMLCanvasElement | OffscreenCanvas) { const options: any = { antialias: false, stencil: true, diff --git a/src/webgl/empirical_cdf.ts b/src/webgl/empirical_cdf.ts index ebd5ffac08..40288d5df1 100644 --- a/src/webgl/empirical_cdf.ts +++ b/src/webgl/empirical_cdf.ts @@ -185,8 +185,8 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); WebGL2RenderingContext.UNSIGNED_BYTE, /*normalized=*/ true, ); - const dataUnit = shader.textureUnit(histogramDataSamplerTextureUnit); - const depthUnit = shader.textureUnit(histogramDepthTextureUnit); + const dataUnit = shader.textureUnit(histogramDataSamplerTextureUnit)!; + const depthUnit = shader.textureUnit(histogramDepthTextureUnit)!; gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + depthUnit); gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, depthTexture); setRawTextureParameters(gl); diff --git a/src/webgl/shader.ts b/src/webgl/shader.ts index 5804cb7b4d..3060beea93 100644 --- a/src/webgl/shader.ts +++ b/src/webgl/shader.ts @@ -35,17 +35,21 @@ export interface ShaderErrorMessage { message: string; } +function getCleanShaderInfoLog(log: string) { + return log.replace("\0", "").replace(/(:\d+:\s*)''\s*:\s*/g, "$1"); +} + /** * Parses the output of getShaderInfoLog into a list of messages. */ export function parseShaderErrors(log: string) { - log = log.replace("\0", ""); const result: ShaderErrorMessage[] = []; for (let line of log.split("\n")) { let m = line.match(/^ERROR:\s*(\d+):(\d+)\s*(.+)$/); if (m !== null) { + const message = m[3].trim(); result.push({ - message: m[3].trim(), + message, file: parseInt(m[1], 10), line: parseInt(m[2], 10), }); @@ -113,7 +117,7 @@ export function getShader( gl.compileShader(shader); if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { - const log = gl.getShaderInfoLog(shader) || ""; + const log = getCleanShaderInfoLog(gl.getShaderInfoLog(shader) || ""); if (DEBUG_SHADER) { const lines = source @@ -251,8 +255,8 @@ export class ShaderProgram extends RefCounted { return this.attributes.get(name)!; } - textureUnit(symbol: symbol | string): number { - return this.textureUnits.get(symbol)!; + textureUnit(symbol: symbol | string): number | undefined { + return this.textureUnits.get(symbol); } bind() { diff --git a/src/webgl/shader_ui_controls.browser_test.ts b/src/webgl/shader_ui_controls.browser_test.ts index 4b53206226..9b1fe53b87 100644 --- a/src/webgl/shader_ui_controls.browser_test.ts +++ b/src/webgl/shader_ui_controls.browser_test.ts @@ -15,7 +15,14 @@ */ import { expect, describe, it } from "vitest"; -import { constantWatchableValue } from "#src/trackable_value.js"; +import { + PreprocessedSegmentPropertyMap, + SegmentPropertyMap, +} from "#src/segmentation_display_state/property_map.js"; +import { + constantWatchableValue, + WatchableValue, +} from "#src/trackable_value.js"; import { DataType } from "#src/util/data_type.js"; import { vec3, vec4 } from "#src/util/geom.js"; import { defaultDataTypeRange } from "#src/util/lerp.js"; @@ -27,6 +34,7 @@ import { getFallbackBuilderState, setControlsInShader, ShaderControlState, + type ShaderDataContext, TrackableTransferFunctionParameters, parseShaderUiControls, parseTransferFunctionParameters, @@ -1108,6 +1116,341 @@ void main() { }); }); +describe("ShaderControlState property controls", () => { + const makeSegmentPropertyMap = () => + new PreprocessedSegmentPropertyMap( + new SegmentPropertyMap({ + inlineProperties: { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "tags", + type: "tags", + tags: ["red", "blue"], + tagDescriptions: ["red", "blue"], + values: ["\u0000"], + }, + { + id: "score", + type: "number", + dataType: DataType.UINT8, + values: new Uint8Array([1]), + description: "score", + bounds: [0, 1], + }, + { + id: "name", + type: "string", + values: ["alpha"], + }, + ], + }, + }), + ); + + const getSelectedPropertyValue = (shaderControlState: ShaderControlState) => + shaderControlState.state.get("selected")!.trackable.value; + + it("uses fallback data context while real data context is loading", () => { + const code = ` +void main() { +} +`; + const fragmentMain = new WatchableValue(code); + const dataContext = new WatchableValue(null); + const shaderControlState = new ShaderControlState( + fragmentMain, + dataContext, + undefined, + {}, + ); + try { + expect(shaderControlState.parseResult.value).toEqual({ + source: code, + code, + controls: new Map(), + preprocessing: emptyParsePreprocessing, + errors: [], + }); + expect(shaderControlState.builderState.value.parseResult.source).toBe( + code, + ); + } finally { + shaderControlState.dispose(); + } + }); + + it("does not report fallback errors for metadata-dependent controls", () => { + const code = ` +#uicontrol property selected +void main() { +} +`; + const fragmentMain = new WatchableValue(code); + const dataContext = new WatchableValue(null); + const shaderControlState = new ShaderControlState( + fragmentMain, + dataContext, + undefined, + {}, + ); + try { + expect(shaderControlState.parseErrors.value).toEqual([]); + expect(shaderControlState.parseResult.value.source).toBe(""); + expect(shaderControlState.state.size).toBe(0); + + dataContext.value = { segmentPropertyMap: makeSegmentPropertyMap() }; + expect(shaderControlState.parseErrors.value).toEqual([]); + expect(shaderControlState.parseResult.value.source).toBe(code); + expect(getSelectedPropertyValue(shaderControlState)).toEqual({ + type: "tag", + id: "red", + }); + } finally { + shaderControlState.dispose(); + } + }); + + it("can use a fallback shader while metadata-dependent controls are loading", () => { + const code = ` +#uicontrol property selected +void main() { +} +`; + const fallbackCode = ` +void main() { +} +`; + const fragmentMain = new WatchableValue(code); + const dataContext = new WatchableValue(null); + const shaderControlState = new ShaderControlState( + fragmentMain, + dataContext, + undefined, + {}, + fallbackCode, + ); + try { + expect(shaderControlState.parseErrors.value).toEqual([]); + expect(shaderControlState.parseResult.value.source).toBe(fallbackCode); + expect(shaderControlState.builderState.value.builderValues).toEqual({}); + expect(shaderControlState.builderState.value.segmentProperties).toEqual( + [], + ); + expect(shaderControlState.state.size).toBe(0); + + dataContext.value = { segmentPropertyMap: makeSegmentPropertyMap() }; + expect(shaderControlState.parseErrors.value).toEqual([]); + expect(shaderControlState.parseResult.value.source).toBe(code); + expect(getSelectedPropertyValue(shaderControlState)).toEqual({ + type: "tag", + id: "red", + }); + } finally { + shaderControlState.dispose(); + } + }); + + it("tracks annotation property invlerp controls as referenced properties", () => { + const code = ` +#uicontrol invlerp normalized(property="score") +void main() { +} +`; + const fragmentMain = new WatchableValue(code); + const dataContext = new WatchableValue({ + properties: new Map([["score", DataType.UINT8]]), + }); + const shaderControlState = new ShaderControlState( + fragmentMain, + dataContext, + ); + try { + expect( + shaderControlState.builderState.value.referencedProperties, + ).toEqual(["score"]); + expect(shaderControlState.builderState.value.segmentProperties).toEqual( + [], + ); + } finally { + shaderControlState.dispose(); + } + }); + + it("tracks segment property invlerp controls as segment properties", () => { + const code = ` +#uicontrol invlerp normalized(property="score") +void main() { +} +`; + const fragmentMain = new WatchableValue(code); + const segmentPropertyMap = makeSegmentPropertyMap(); + const dataContext = new WatchableValue({ + properties: new Map([["score", DataType.UINT8]]), + values: new Map([["score", new Uint8Array([1])]]), + shaderName: () => "numerical0", + segmentPropertyMap, + }); + const shaderControlState = new ShaderControlState( + fragmentMain, + dataContext, + ); + try { + expect( + shaderControlState.builderState.value.referencedProperties, + ).toEqual([]); + expect(shaderControlState.builderState.value.segmentProperties).toEqual([ + { type: "numerical", id: "score" }, + ]); + } finally { + shaderControlState.dispose(); + } + }); + + it("defaults to first property when created", () => { + const unfilteredCode = ` +#uicontrol property selected +void main() { +} +`; + const fragmentMain = new WatchableValue(unfilteredCode); + const dataContext = new WatchableValue({ + segmentPropertyMap: makeSegmentPropertyMap(), + }); + const shaderControlState = new ShaderControlState( + fragmentMain, + dataContext, + ); + try { + expect(getSelectedPropertyValue(shaderControlState)).toEqual({ + type: "tag", + id: "red", + }); + expect(shaderControlState.builderState.value.segmentProperties).toEqual([ + { type: "tag", id: "red" }, + ]); + } finally { + shaderControlState.dispose(); + } + }); + + it("defaults to first property allowed by type filter when created", () => { + const numericalFilteredCode = ` +#uicontrol property selected(type="number") +void main() { +} +`; + const fragmentMain = new WatchableValue(numericalFilteredCode); + const dataContext = new WatchableValue({ + segmentPropertyMap: makeSegmentPropertyMap(), + }); + const shaderControlState = new ShaderControlState( + fragmentMain, + dataContext, + ); + try { + expect(getSelectedPropertyValue(shaderControlState)).toEqual({ + type: "numerical", + id: "score", + }); + expect(shaderControlState.builderState.value.segmentProperties).toEqual([ + { type: "numerical", id: "score" }, + ]); + } finally { + shaderControlState.dispose(); + } + }); + + it("preserves selected tag when adding and removing property type filter", () => { + const unfilteredCode = ` +#uicontrol property selected +void main() { +} +`; + const tagFilteredCode = ` +#uicontrol property selected(type="tag") +void main() { +} +`; + const fragmentMain = new WatchableValue(unfilteredCode); + const dataContext = new WatchableValue({ + segmentPropertyMap: makeSegmentPropertyMap(), + }); + const shaderControlState = new ShaderControlState( + fragmentMain, + dataContext, + ); + try { + shaderControlState.state.get("selected")!.trackable.value = { + type: "tag", + id: "red", + }; + expect(getSelectedPropertyValue(shaderControlState)).toEqual({ + type: "tag", + id: "red", + }); + + fragmentMain.value = tagFilteredCode; + expect(getSelectedPropertyValue(shaderControlState)).toEqual({ + type: "tag", + id: "red", + }); + expect(shaderControlState.builderState.value.segmentProperties).toEqual([ + { type: "tag", id: "red" }, + ]); + + fragmentMain.value = unfilteredCode; + expect(getSelectedPropertyValue(shaderControlState)).toEqual({ + type: "tag", + id: "red", + }); + expect(shaderControlState.builderState.value.segmentProperties).toEqual([ + { type: "tag", id: "red" }, + ]); + } finally { + shaderControlState.dispose(); + } + }); + + it("defaults to first valid property when preserved value is filtered out", () => { + const unfilteredCode = ` +#uicontrol property selected +void main() { +} +`; + const tagFilteredCode = ` +#uicontrol property selected(type="tag") +void main() { +} +`; + const fragmentMain = new WatchableValue(unfilteredCode); + const dataContext = new WatchableValue({ + segmentPropertyMap: makeSegmentPropertyMap(), + }); + const shaderControlState = new ShaderControlState( + fragmentMain, + dataContext, + ); + try { + shaderControlState.state.get("selected")!.trackable.value = { + type: "numerical", + id: "score", + }; + + fragmentMain.value = tagFilteredCode; + expect(getSelectedPropertyValue(shaderControlState)).toEqual({ + type: "tag", + id: "red", + }); + expect(shaderControlState.builderState.value.segmentProperties).toEqual([ + { type: "tag", id: "red" }, + ]); + } finally { + shaderControlState.dispose(); + } + }); +}); + describe("parseShaderUiControls select", () => { it("handles basic select control", () => { const code = ` diff --git a/src/webgl/shader_ui_controls.ts b/src/webgl/shader_ui_controls.ts index 753d0cdc4f..0da03ece72 100644 --- a/src/webgl/shader_ui_controls.ts +++ b/src/webgl/shader_ui_controls.ts @@ -15,6 +15,7 @@ */ import type { CoordinateSpaceCombiner } from "#src/coordinate_transform.js"; +import type { PreprocessedSegmentPropertyMap } from "#src/segmentation_display_state/property_map.js"; import { TrackableBoolean } from "#src/trackable_boolean.js"; import type { TrackableValueInterface, @@ -27,6 +28,7 @@ import { TrackableValue, WatchableValue, } from "#src/trackable_value.js"; +import type { TypedNumberArray } from "#src/util/array.js"; import { arraysEqual, arraysEqualWithPredicate } from "#src/util/array.js"; import { parseRGBColorSpecification, @@ -71,6 +73,7 @@ import { activeControlsEqual, computeActiveControls, } from "#src/webgl/shader_control_reachability.js"; +import { glsl_string } from "#src/webgl/shader_lib.js"; import { preprocessStrings, type ShaderStringLiteralIdMap, @@ -112,6 +115,8 @@ export interface ShaderPropertyInvlerpControl { type: "propertyInvlerp"; clamp: boolean; properties: PropertiesSpecification; + values?: Map>; + shaderName?: (arg0: string) => string; default: PropertyInvlerpParameters; } @@ -143,6 +148,18 @@ export interface ShaderTransferFunctionControl { default: TransferFunctionParameters; } +export interface AvailableSegmentProperties { + tags: string[]; + numericalProperties: Map; + stringProperties: string[]; +} + +export interface ShaderPropertyControl { + type: "property"; + segmentProperties: AvailableSegmentProperties; + default?: SegmentPropertyReference; +} + export type ShaderUiControl = | ShaderSliderControl | ShaderColorControl @@ -150,7 +167,8 @@ export type ShaderUiControl = | ShaderPropertyInvlerpControl | ShaderCheckboxControl | ShaderSelectControl - | ShaderTransferFunctionControl; + | ShaderTransferFunctionControl + | ShaderPropertyControl; export interface ShaderControlParseError { line: number; @@ -172,6 +190,7 @@ export interface ShaderControlsBuilderState { parseResult: ShaderControlsParseResult; builderValues: ShaderBuilderValues; referencedProperties: string[]; + segmentProperties: SegmentPropertyReference[]; } // Strips comments from GLSL code. Also handles string literals since they are used in ui control @@ -619,18 +638,80 @@ function parseInvlerpDirective( parameters: DirectiveParameters, dataContext: ShaderDataContext, ): DirectiveParseResult { - const { imageData, properties } = dataContext; + const { imageData, properties, values, shaderName } = dataContext; if (imageData !== undefined) { return parseImageInvlerpDirective(valueType, parameters, imageData); } if (properties !== undefined) { - return parsePropertyInvlerpDirective(valueType, parameters, properties); + return parsePropertyInvlerpDirective( + valueType, + parameters, + properties, + values, + shaderName, + ); } const errors = []; errors.push("invlerp control not supported"); return { errors }; } +function parsePropertyDirective( + valueType: string, + parameters: DirectiveParameters, + dataContext: ShaderDataContext, +): DirectiveParseResult { + const { segmentPropertyMap } = dataContext; + const errors: string[] = []; + if (valueType !== "property") { + errors.push("type must be property"); + } + + let typeFilter: SegmentPropertyReference["type"] | undefined; + for (const [key, value] of parameters) { + if (key !== "type") { + errors.push(`Invalid parameter: ${key}`); + continue; + } + if (value === "number") { + typeFilter = "numerical"; + } else if (value === "tag" || value === "numerical" || value === "string") { + typeFilter = value; + } else { + errors.push(`Invalid type value: ${JSON.stringify(value)}`); + } + } + if (errors.length > 0) { + return { errors }; + } + + const includeType = (type: SegmentPropertyReference["type"]) => { + return typeFilter === undefined || typeFilter === type; + }; + + const segmentProperties = { + tags: segmentPropertyMap?.tags?.tags.filter(() => includeType("tag")) || [], + numericalProperties: new Map( + (segmentPropertyMap?.numericalProperties ?? []) + .filter(() => includeType("numerical")) + .map((x) => [x.id, x.dataType]), + ), + stringProperties: + (segmentPropertyMap?.strings ?? []) + .filter(() => includeType("string")) + .map((x) => x.id) || [], + }; + + return { + control: { + type: "property", + segmentProperties, + default: getFirstSegmentPropertyReference(segmentProperties), + } satisfies ShaderPropertyControl, + errors: undefined, + }; +} + function parseImageInvlerpDirective( valueType: string, parameters: DirectiveParameters, @@ -674,8 +755,9 @@ function parseImageInvlerpDirective( errors.push(`Invalid parameter: ${key}`); break; } - } catch (e) { - errors.push(`Invalid ${key} value: ${e.message}`); + } catch (error) { + const message = error instanceof Error ? error.message : `${error}`; + errors.push(`Invalid ${key} value: ${message}`); } } if (errors.length > 0) { @@ -700,6 +782,8 @@ function parsePropertyInvlerpDirective( valueType: string, parameters: DirectiveParameters, properties: Map, + values?: Map>, + shaderName?: (arg0: string) => string, ) { const errors = []; if (valueType !== "invlerp") { @@ -742,8 +826,9 @@ function parsePropertyInvlerpDirective( errors.push(`Invalid parameter: ${key}`); break; } - } catch (e) { - errors.push(`Invalid ${key} value: ${e.message}`); + } catch (error) { + const message = error instanceof Error ? error.message : `${error}`; + errors.push(`Invalid ${key} value: ${message}`); } } if (errors.length > 0) { @@ -767,8 +852,10 @@ function parsePropertyInvlerpDirective( type: "propertyInvlerp", clamp, properties, - default: { range, window, property, dataType }, - } as ShaderPropertyInvlerpControl, + values, + shaderName, + default: { range, window, property: property!, dataType }, + } satisfies ShaderPropertyInvlerpControl, errors: undefined, }; } @@ -827,8 +914,9 @@ function parseTransferFunctionDirective( errors.push(`Invalid parameter: ${key}`); break; } - } catch (e) { - errors.push(`Invalid ${key} value: ${e.message}`); + } catch (error) { + const message = error instanceof Error ? error.message : `${error}`; + errors.push(`Invalid ${key} value: ${message}`); } } @@ -861,6 +949,9 @@ export interface ImageDataSpecification { export interface ShaderDataContext { imageData?: ImageDataSpecification; properties?: Map; + values?: Map>; + shaderName?: (arg0: string) => string; + segmentPropertyMap?: PreprocessedSegmentPropertyMap; } const controlParsers = new Map< @@ -877,6 +968,7 @@ const controlParsers = new Map< ["checkbox", parseCheckboxDirective], ["select", parseSelectDirective], ["transferFunction", parseTransferFunctionDirective], + ["property", parsePropertyDirective], ]); export function parseShaderUiControls( @@ -961,7 +1053,11 @@ function uniformName(controlName: string) { export function addControlsToBuilder( builderState: ShaderControlsBuilderState, builder: ShaderBuilder, + fragment = true, ) { + const addCode = fragment + ? builder.addFragmentCode.bind(builder) + : builder.addVertexCode.bind(builder); const { builderValues } = builderState; for (const [name, control] of builderState.parseResult.controls) { const uName = uniformName(name); @@ -981,33 +1077,50 @@ float ${uName}() { } `, ]; - builder.addFragmentCode(code); - builder.addFragmentCode(`#define ${name} ${uName}\n`); + addCode(code); + addCode(`#define ${name} ${uName}\n`); break; } case "propertyInvlerp": { const property = builderValue.property; const dataType = control.properties.get(property)!; + const propertyShaderName = control.shaderName + ? control.shaderName(property) + : `prop_${property}()`; const code = [ defineInvlerpShaderFunction(builder, uName, dataType, control.clamp), ` float ${uName}() { - return ${uName}(prop_${property}()); + return ${uName}(${propertyShaderName}); } `, ]; - builder.addVertexCode(code); - builder.addVertexCode(`#define ${name} ${uName}\n`); + addCode(code); + addCode(`#define ${name} ${uName}\n`); + break; + } + case "property": { + if (builderValue) { + const { type, id } = builderValue; + let code = `#define ${name} ${type}${id}\n`; + if (type === "tag") { + code = `#define ${name} (${type}${id} == 1u)\n`; + } + if (type === "string") { + addCode(glsl_string); + code = `#define ${name} string_t(${type}${id})\n`; + } + addCode(code); + } break; } case "checkbox": { const code = `#define ${name} ${builderValue.value}\n`; - builder.addFragmentCode(code); - builder.addVertexCode(code); + addCode(code); break; } case "transferFunction": { - builder.addFragmentCode(`#define ${name} ${uName}\n`); + addCode(`#define ${name} ${uName}\n`); builder.addFragmentCode( defineTransferFunctionShader( builder, @@ -1033,8 +1146,7 @@ float ${uName}() { } default: { builder.addUniform(`highp ${control.valueType}`, uName); - builder.addVertexCode(`#define ${name} ${uName}\n`); - builder.addFragmentCode(`#define ${name} ${uName}\n`); + addCode(`#define ${name} ${uName}\n`); break; } } @@ -1411,6 +1523,68 @@ export class TrackableTransferFunctionParameters extends TrackableValue; getBuilderValue: (value: any) => any; @@ -1463,6 +1637,15 @@ function getControlTrackable(control: ShaderUiControl): { dataType: value.dataType, }), }; + case "property": + return { + trackable: new TrackableValue( + control.default, + (x) => parseSegmentPropertyReference(control.segmentProperties, x), + ), + getBuilderValue: (x: SegmentPropertyReference | undefined) => + getSegmentPropertyBuilderValue(control, x), + }; case "checkbox": return { trackable: new TrackableBoolean(control.default), @@ -1516,31 +1699,99 @@ export type ShaderBuilderValues = { function encodeBuilderStateKey( builderValues: ShaderBuilderValues, parseResult: ShaderControlsParseResult, + referencedProperties: string[], + segmentProperties: SegmentPropertyReference[], +) { + return ( + JSON.stringify(builderValues) + + "\0" + + JSON.stringify(referencedProperties) + + "\0" + + JSON.stringify(segmentProperties) + + "\0" + + parseResult.source + ); +} + +function addReferencedPropertiesFromControl( + control: ShaderUiControl, + builderValue: any, + trackableValue: any, + referencedProperties: string[], + segmentProperties: SegmentPropertyReference[], ) { - return JSON.stringify(builderValues) + "\0" + parseResult.source; + if (control.type === "propertyInvlerp") { + if (control.shaderName === undefined) { + referencedProperties.push(builderValue.property); + } else { + segmentProperties.push({ + type: "numerical", + id: builderValue.property, + }); + } + } + if (control.type === "property" && builderValue !== undefined) { + const { type, id } = trackableValue as SegmentPropertyReference; + segmentProperties.push({ type, id }); + } } export function getFallbackBuilderState( parseResult: ShaderControlsParseResult, ): ShaderControlsBuilderState { const builderValues: ShaderBuilderValues = {}; - const referencedProperties = []; + const referencedProperties: string[] = []; + const segmentProperties: SegmentPropertyReference[] = []; for (const [key, control] of parseResult.controls) { const { trackable, getBuilderValue } = getControlTrackable(control); const builderValue = getBuilderValue(trackable.value); builderValues[key] = builderValue; - if (control.type === "propertyInvlerp") { - referencedProperties.push(builderValue.property); - } + addReferencedPropertiesFromControl( + control, + builderValue, + trackable.value, + referencedProperties, + segmentProperties, + ); } return { builderValues, parseResult, - key: encodeBuilderStateKey(builderValues, parseResult), + key: encodeBuilderStateKey( + builderValues, + parseResult, + referencedProperties, + segmentProperties, + ), referencedProperties, + segmentProperties, }; } +function getLoadingParseResult(): ShaderControlsParseResult { + return { + source: "", + code: "", + controls: new Map(), + preprocessing: { stringLiteralIds: new Map() }, + errors: [{ line: 0, message: "Loading" }], + }; +} + +function canUseFallbackParseResult(result: ShaderControlsParseResult) { + if (result.errors.length !== 0) return false; + for (const control of result.controls.values()) { + switch (control.type) { + case "imageInvlerp": + case "propertyInvlerp": + case "property": + case "transferFunction": + return false; + } + } + return true; +} + export class ShaderControlState extends RefCounted implements Trackable, WatchableValueInterface @@ -1561,7 +1812,7 @@ export class ShaderControlState private dataContextGeneration = -1; private parseErrors_: ShaderControlParseError[] = []; private processedFragmentMain_ = ""; - private parseResult_: ShaderControlsParseResult; + private parseResult_!: ShaderControlsParseResult; private controlsGeneration = -1; private parseResultChanged = new NullarySignal(); private lastReportedProgram: WebGLProgram | undefined = undefined; @@ -1572,6 +1823,8 @@ export class ShaderControlState {}, ), public channelCoordinateSpaceCombiner?: CoordinateSpaceCombiner | undefined, + private fallbackDataContext?: ShaderDataContext, + private fallbackFragmentMain?: string, ) { super(); this.registerDisposer( @@ -1608,19 +1861,31 @@ export class ShaderControlState this.builderState = makeCachedDerivedWatchableValue( (parseResult: ShaderControlsParseResult, state: ShaderControlMap) => { const builderValues: ShaderBuilderValues = {}; - const referencedProperties = []; + const referencedProperties: string[] = []; + const segmentProperties: SegmentPropertyReference[] = []; for (const [key, { control, trackable, getBuilderValue }] of state) { + if (!parseResult.controls.has(key)) continue; const builderValue = getBuilderValue(trackable.value); builderValues[key] = builderValue; - if (control.type === "propertyInvlerp") { - referencedProperties.push(builderValue.property); - } + addReferencedPropertiesFromControl( + control, + builderValue, + trackable.value, + referencedProperties, + segmentProperties, + ); } return { - key: encodeBuilderStateKey(builderValues, parseResult), + key: encodeBuilderStateKey( + builderValues, + parseResult, + referencedProperties, + segmentProperties, + ), parseResult, builderValues, referencedProperties, + segmentProperties, }; }, [this.parseResult, this], @@ -1693,27 +1958,38 @@ export class ShaderControlState } this.fragmentMainGeneration = generation; this.dataContextGeneration = dataContextGeneration; - const dataContext = this.dataContext.value; + const realDataContext = this.dataContext.value; + const dataContext = realDataContext ?? this.fallbackDataContext ?? null; if (dataContext === null) { - this.parseResult_ = { - source: "", - code: "", - controls: new Map(), - preprocessing: { stringLiteralIds: new Map() }, - errors: [{ line: 0, message: "Loading" }], - }; + this.parseResult_ = getLoadingParseResult(); this.parseErrors_ = []; this.processedFragmentMain_ = ""; this.controls.value = undefined; } else { - const result = (this.parseResult_ = parseShaderUiControls( + const result = parseShaderUiControls( this.fragmentMain.value, dataContext, - )); - this.parseErrors_ = result.errors; - this.processedFragmentMain_ = result.code; - if (result.errors.length === 0) { - this.controls.value = result.controls; + ); + if ( + realDataContext === null && + this.fallbackDataContext !== undefined && + !canUseFallbackParseResult(result) + ) { + const fallbackFragmentMain = this.fallbackFragmentMain; + this.parseResult_ = + fallbackFragmentMain === undefined + ? getLoadingParseResult() + : parseShaderUiControls(fallbackFragmentMain, dataContext); + this.parseErrors_ = []; + this.processedFragmentMain_ = this.parseResult_.code; + this.controls.value = undefined; + } else { + this.parseResult_ = result; + this.parseErrors_ = result.errors; + this.processedFragmentMain_ = result.code; + if (result.errors.length === 0) { + this.controls.value = result.controls; + } } } // The active-controls set was derived from the previous shader. Forget the @@ -1760,15 +2036,30 @@ export class ShaderControlState } for (const [name, control] of controls) { let controlState = state_.get(name); + let preservedValue: unknown; if ( controlState !== undefined && JSON.stringify(controlState.control) !== JSON.stringify(control) ) { + if ( + controlState.control.type === "property" && + control.type === "property" + ) { + preservedValue = controlState.trackable.value; + } controlState.trackable.changed.remove(this.changed.dispatch); controlState = undefined; } if (controlState === undefined) { const { trackable, getBuilderValue } = getControlTrackable(control); + if (preservedValue !== undefined) { + trackable.restoreState(preservedValue); + if (control.type === "property" && trackable.value === undefined) { + trackable.restoreState( + getFirstSegmentPropertyReference(control.segmentProperties), + ); + } + } controlState = { control, trackable, getBuilderValue }; controlState.trackable.changed.add(this.changed.dispatch); state_.set(name, controlState); @@ -1928,6 +2219,11 @@ function setControlInShader( control.dataType, value.sortedControlPoints, ); + break; + case "property": + // Property values are compile-time defines backed by textures managed by + // SegmentColorUserShaderManager. + break; } } @@ -1947,6 +2243,7 @@ export function setControlsInShader( preprocessing: { stringLiteralIds }, } = parseResult; const { state } = shaderControlState; + if (shaderControlState.controls.value === controls) { // Case when shader doesn't have any errors. for (const [name, controlState] of state) { diff --git a/src/webgl/texture_access.browser_test.ts b/src/webgl/texture_access.browser_test.ts index 0cb1030eb8..b09b7c4a17 100644 --- a/src/webgl/texture_access.browser_test.ts +++ b/src/webgl/texture_access.browser_test.ts @@ -62,7 +62,7 @@ outputValue = readValue(uOffset); const { shader } = tester; shader.bind(); - const textureUnit = shader.textureUnit(textureUnitSymbol); + const textureUnit = shader.textureUnit(textureUnitSymbol)!; const texture = gl.createTexture(); tester.registerDisposer(() => { gl.deleteTexture(texture); diff --git a/src/webgl/texture_access.ts b/src/webgl/texture_access.ts index d3a58b5e5d..cf6dcfd331 100644 --- a/src/webgl/texture_access.ts +++ b/src/webgl/texture_access.ts @@ -612,3 +612,37 @@ void ${this.readTextureValue}(highp ${samplerPrefix}sampler${this.textureDims}D ]; } } + +export function create1DTexture(gl: GL, data: Float32Array) { + const texture = gl.createTexture(); + // for now, immediately load the data into the texture + { + const format = computeTextureFormat( + new TextureFormat(), + DataType.FLOAT32, + 1, + ); + gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + gl.tempTextureUnit); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, texture); + { + const { textureInternalFormat, textureFormat, texelType } = format; + + // const padded = maybePadArray(data, requiredSize); + gl.pixelStorei(WebGL2RenderingContext.UNPACK_ALIGNMENT, 1); + setRawTextureParameters(gl); + gl.texImage2D( + WebGL2RenderingContext.TEXTURE_2D, + /*level=*/ 0, + textureInternalFormat, + /*width=*/ data.length, + /*height=*/ 1, + /*border=*/ 0, + textureFormat, + texelType, + data, + ); + } + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, null); + } + return texture; +} diff --git a/src/widget/invlerp.ts b/src/widget/invlerp.ts index ab292d8831..726bd0002e 100644 --- a/src/widget/invlerp.ts +++ b/src/widget/invlerp.ts @@ -20,10 +20,15 @@ import svg_arrowLeft from "ikonate/icons/arrow-left.svg?raw"; import svg_arrowRight from "ikonate/icons/arrow-right.svg?raw"; import type { DisplayContext } from "#src/display_context.js"; import { IndirectRenderedPanel } from "#src/display_context.js"; -import type { WatchableValueInterface } from "#src/trackable_value.js"; +import { + makeCachedDerivedWatchableValue, + makeCachedLazyDerivedWatchableValue, + type WatchableValueInterface, +} from "#src/trackable_value.js"; import type { ToolActivation } from "#src/ui/tool.js"; import { animationFrameDebounce } from "#src/util/animation_frame_debounce.js"; -import type { DataType } from "#src/util/data_type.js"; +import type { TypedNumberArray } from "#src/util/array.js"; +import { DataType } from "#src/util/data_type.js"; import type { Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; import { removeChildren, updateInputFieldWidth } from "#src/util/dom.js"; @@ -67,6 +72,7 @@ import { getShaderType } from "#src/webgl/shader_lib.js"; import type { InvlerpParameters } from "#src/webgl/shader_ui_controls.js"; import { getSquareCornersBuffer } from "#src/webgl/square_corners_buffer.js"; import { setRawTextureParameters } from "#src/webgl/texture.js"; +import { create1DTexture } from "#src/webgl/texture_access.js"; import { makeIcon } from "#src/widget/icon.js"; import { AutoRangeFinder } from "#src/widget/invlerp_range_finder.js"; import type { LayerControlTool } from "#src/widget/layer_control.js"; @@ -442,7 +448,10 @@ out_color = uColor; gl.drawArrays(WebGL2RenderingContext.TRIANGLE_FAN, 0, 4); gl.disableVertexAttribArray(aVertexPosition); } - if (this.parent.histogramSpecifications.producerVisibility.visible) { + if ( + this.parent.histogramSpecifications.producerVisibility.visible || + this.parent.values + ) { const { renderViewport } = this; lineShader.bind(); initializeLineShader( @@ -455,13 +464,14 @@ out_color = uColor; ); const histogramTextureUnit = lineShader.textureUnit( histogramSamplerTextureUnit, - ); + )!; gl.uniform1f( lineShader.uniform("uBoundsFraction"), getIntervalBoundsEffectiveFraction(dataType, bounds.window), ); + const { texture } = this.parent; gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + histogramTextureUnit); - gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, this.parent.texture); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, texture); setRawTextureParameters(gl); const aDataValue = lineShader.attribute("aDataValue"); this.dataValuesBuffer.bindToVertexAttribI( @@ -728,12 +738,74 @@ export function adjustInvlerpBrightnessContrast( }; } +// The first and last bin are for values below the lower bound/above the upper +// To simulate output from the GLSL shader function on CPU +function countDataInBins( + inputData: TypedNumberArray, + dataType: DataType, + min: number | bigint, + max: number | bigint, + numDataBins: number = 254, +): Float32Array { + // Total number of bins is numDataBins + 2, one for values below the lower + // bound and one for values above the upper bound. + const counts = new Float32Array(numDataBins + 2).fill(0); + let binSize: number; + let binIndex: number; + if (dataType === DataType.UINT64) { + binSize = Number((max as bigint) - (min as bigint)) / numDataBins; + } else { + binSize = ((max as number) - (min as number)) / numDataBins; + } + for (let i = 0; i < inputData.length; i++) { + const value = inputData[i]; + if (dataTypeCompare(value, min) < 0) { + counts[0]++; + } else if (dataTypeCompare(value, max) > 0) { + counts[numDataBins + 1]++; + } else { + // TODO segment properties doesn't support uint64 yet + // if (dataType === DataType.UINT64) { + // binIndex = Math.floor( + // Number((value as bigint) - (min as bigint)) / binSize, + // ); + // } else { + binIndex = Math.floor(((value as number) - (min as number)) / binSize); + // } + counts[binIndex + 1]++; + } + } + return counts; +} + +const createHistogramTextureFromValues = ( + values: TypedNumberArray, + window: DataTypeInterval, + dataType: DataType, + gl: GL, +) => { + const histogram = countDataInBins( + values, + dataType, + window[0], + window[1], + NUM_HISTOGRAM_BINS_IN_RANGE, + ); + return create1DTexture(gl, histogram); +}; + export class InvlerpWidget extends Tab { + // TODO should this implement ParentInverpWidget? cdfPanel; boundElements; invertArrows: HTMLElement[]; autoRangeFinder: AutoRangeFinder; + textureFromValues?: WatchableValueInterface; + get texture() { + if (this.textureFromValues?.value) { + return this.textureFromValues.value; + } return this.histogramSpecifications.getFramebuffers(this.display.gl)[ this.histogramIndex ].colorBuffers[0].texture; @@ -749,6 +821,9 @@ export class InvlerpWidget extends Tab { public histogramSpecifications: HistogramSpecifications, public histogramIndex: number, public legendShaderOptions: LegendShaderOptions | undefined, + public values?: WatchableValueInterface< + TypedNumberArray | undefined + >, ) { super(visibility); this.cdfPanel = this.registerDisposer(new CdfPanel(this)); @@ -795,6 +870,35 @@ export class InvlerpWidget extends Tab { this.autoRangeFinder.maybeAutoComputeRange(); }), ); + + const derivedWindowWatchable = makeCachedDerivedWatchableValue( + (p) => p.window, + [this.trackable], + ); + + if (values) { + let previousTextureFromValues: WebGLTexture | null = null; + this.textureFromValues = this.registerDisposer( + makeCachedLazyDerivedWatchableValue( + (values, window) => { + const { gl } = this.display; + gl.deleteTexture(previousTextureFromValues); + if (!values || !window) return; + const { dataType } = this; + const texture = createHistogramTextureFromValues( + values, + window, + dataType, + gl, + ); + previousTextureFromValues = texture; + return texture; + }, + values, + derivedWindowWatchable, + ), + ); + } } updateView() { @@ -847,6 +951,9 @@ export class VariableDataTypeInvlerpWidget extends Tab { public histogramSpecifications: HistogramSpecifications, public histogramIndex: number, public legendShaderOptions: LegendShaderOptions | undefined, + public values?: WatchableValueInterface< + TypedNumberArray | undefined + >, ) { super(visibility); this.invlerpWidget = this.makeInvlerpWidget(); @@ -878,6 +985,7 @@ export class VariableDataTypeInvlerpWidget extends Tab { this.histogramSpecifications, this.histogramIndex, this.legendShaderOptions, + this.values, ); this.element.appendChild(widget.element); return widget; diff --git a/src/widget/invlerp_range_finder.ts b/src/widget/invlerp_range_finder.ts index 034de1b9e4..f523d6b969 100644 --- a/src/widget/invlerp_range_finder.ts +++ b/src/widget/invlerp_range_finder.ts @@ -15,6 +15,8 @@ */ import type { DisplayContext } from "#src/display_context.js"; +import type { WatchableValueInterface } from "#src/trackable_value.js"; +import type { TypedNumberArray } from "#src/util/array.js"; import { DataType } from "#src/util/data_type.js"; import { RefCounted } from "#src/util/disposable.js"; import { computePercentilesFromEmpiricalHistogram } from "#src/util/empirical_cdf.js"; @@ -53,6 +55,7 @@ interface ParentInvlerpWidget { element: HTMLDivElement; histogramSpecifications: HistogramSpecifications; histogramIndex: number; + values?: WatchableValueInterface | undefined>; } export class AutoRangeFinder extends RefCounted { @@ -86,24 +89,38 @@ export class AutoRangeFinder extends RefCounted { } autoComputeRange(minPercentile: number, maxPercentile: number) { - if (!this.autoRangeData.autoComputeInProgress) { - const { autoRangeData } = this; - const { dataType, display } = this.parent; + const { values } = this.parent; + if (values?.value) { + const valuesWithoutNaN = values.value.filter((x) => !Number.isNaN(x)); + const valuesSorted = valuesWithoutNaN.slice().sort((a, b) => a - b); + const n = valuesSorted.length; + const minIndex = Math.floor(minPercentile * (n - 1)); + const maxIndex = Math.ceil(maxPercentile * (n - 1)); + const min = valuesSorted[minIndex]; + const max = valuesSorted[maxIndex]; + const range: DataTypeInterval = [min, max]; + this.setTrackableValue(range, range); + this.parent.display.scheduleRedraw(); + } else { + if (!this.autoRangeData.autoComputeInProgress) { + const { autoRangeData } = this; + const { dataType, display } = this.parent; - // Reset the auto-compute state - autoRangeData.inputPercentileBounds = [minPercentile, maxPercentile]; - this.resetAutoRangeData(autoRangeData); - autoRangeData.invertedInitialRange = this.wasInputInverted(); - display.force3DHistogramForAutoRange = true; + // Reset the auto-compute state + autoRangeData.inputPercentileBounds = [minPercentile, maxPercentile]; + this.resetAutoRangeData(autoRangeData); + autoRangeData.invertedInitialRange = this.wasInputInverted(); + display.force3DHistogramForAutoRange = true; - // Create a large range to search over - // It's easier to contract the range than to expand it - const maxRange = - dataType === DataType.FLOAT32 - ? ([-65536, 65536] as [number, number]) - : defaultDataTypeRange[dataType]; - this.setTrackableValue(maxRange, maxRange); - display.scheduleRedraw(); + // Create a large range to search over + // It's easier to contract the range than to expand it + const maxRange = + dataType === DataType.FLOAT32 + ? ([-65536, 65536] as [number, number]) + : defaultDataTypeRange[dataType]; + this.setTrackableValue(maxRange, maxRange); + display.scheduleRedraw(); + } } } diff --git a/src/widget/layer_control_channel_invlerp.ts b/src/widget/layer_control_channel_invlerp.ts index 90eac95222..a575fec714 100644 --- a/src/widget/layer_control_channel_invlerp.ts +++ b/src/widget/layer_control_channel_invlerp.ts @@ -17,7 +17,7 @@ import type { CoordinateSpaceCombiner } from "#src/coordinate_transform.js"; import type { UserLayer } from "#src/layer/index.js"; import { Position } from "#src/navigation_state.js"; -import type { WatchableValueInterface } from "#src/trackable_value.js"; +import { type WatchableValueInterface } from "#src/trackable_value.js"; import { arraysEqual } from "#src/util/array.js"; import type { DataType } from "#src/util/data_type.js"; import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; diff --git a/src/widget/layer_control_property.ts b/src/widget/layer_control_property.ts new file mode 100644 index 0000000000..ed28d47ea2 --- /dev/null +++ b/src/widget/layer_control_property.ts @@ -0,0 +1,143 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { UserLayer } from "#src/layer/index.js"; +import type { WatchableValueInterface } from "#src/trackable_value.js"; +import { DataType } from "#src/util/data_type.js"; +import { RefCounted } from "#src/util/disposable.js"; +import type { + AvailableSegmentProperties, + SegmentPropertyReference, +} from "#src/webgl/shader_ui_controls.js"; +import type { LayerControlFactory } from "#src/widget/layer_control.js"; + +type SegmentPropertyOption = SegmentPropertyReference & { + dataType?: DataType; +}; + +function encodeSegmentPropertyReference(ref: SegmentPropertyReference) { + return JSON.stringify([ref.type, ref.id]); +} + +function decodeSegmentPropertyReference( + value: string, +): SegmentPropertyReference | undefined { + try { + const [type, id] = JSON.parse(value) as unknown[]; + if ( + (type === "tag" || type === "numerical" || type === "string") && + typeof id === "string" + ) { + return { type, id }; + } + } catch { + // Fall through to undefined. + } + return undefined; +} + +function segmentPropertyReferencesEqual( + a: SegmentPropertyReference | undefined, + b: SegmentPropertyReference | undefined, +) { + return a?.type === b?.type && a?.id === b?.id; +} + +export class SegmentPropertySelectWidget extends RefCounted { + element = document.createElement("select"); + + constructor( + segmentProperties: AvailableSegmentProperties, + public model: WatchableValueInterface, + ) { + super(); + const { element } = this; + element.classList.add("neuroglancer-select-widget"); + + const maybeAddGroup = (label: string, values: SegmentPropertyOption[]) => { + if (values.length) { + const optGroup = document.createElement("optgroup"); + optGroup.label = `${label} properties`; + element.appendChild(optGroup); + for (const value of values) { + const option = document.createElement("option"); + option.value = encodeSegmentPropertyReference(value); + option.textContent = + value.dataType === undefined + ? value.id + : `${value.id} (${DataType[value.dataType].toLowerCase()})`; + optGroup.appendChild(option); + } + } + }; + + maybeAddGroup( + "tag", + segmentProperties.tags.map((id) => ({ type: "tag", id })), + ); + maybeAddGroup( + "numerical", + [...segmentProperties.numericalProperties].map(([id, dataType]) => ({ + type: "numerical", + id, + dataType, + })), + ); + maybeAddGroup( + "string", + segmentProperties.stringProperties.map((id) => ({ type: "string", id })), + ); + + this.registerDisposer(model.changed.add(() => this.updateView())); + this.registerEventListener(element, "change", () => this.updateModel()); + this.updateView(); + } + + private updateView() { + if (this.model.value === undefined) { + this.element.selectedIndex = -1; + return; + } + this.element.value = encodeSegmentPropertyReference(this.model.value); + } + + private updateModel() { + const value = decodeSegmentPropertyReference(this.element.value); + if (!segmentPropertyReferencesEqual(value, this.model.value)) { + this.model.value = value; + } + } +} + +export function propertyLayerControl( + getter: (layer: LayerType) => { + segmentProperties: AvailableSegmentProperties; + watchableValue: WatchableValueInterface< + SegmentPropertyReference | undefined + >; + }, +): LayerControlFactory { + return { + makeControl: (layer, context) => { + const { segmentProperties, watchableValue } = getter(layer); + const control = context.registerDisposer( + new SegmentPropertySelectWidget(segmentProperties, watchableValue), + ); + return { control, controlElement: control.element }; + }, + activateTool: () => {}, + }; +} diff --git a/src/widget/layer_control_property_invlerp.ts b/src/widget/layer_control_property_invlerp.ts index 463ba0dbd8..92076f80ef 100644 --- a/src/widget/layer_control_property_invlerp.ts +++ b/src/widget/layer_control_property_invlerp.ts @@ -16,7 +16,11 @@ import type { UserLayer } from "#src/layer/index.js"; import type { WatchableValueInterface } from "#src/trackable_value.js"; -import { makeCachedDerivedWatchableValue } from "#src/trackable_value.js"; +import { + makeCachedDerivedWatchableValue, + makeCachedLazyDerivedWatchableValue, +} from "#src/trackable_value.js"; +import type { TypedNumberArray } from "#src/util/array.js"; import { DataType } from "#src/util/data_type.js"; import { convertDataTypeInterval, @@ -40,6 +44,7 @@ export function propertyInvlerpLayerControl( getter: (layer: LayerType) => { watchableValue: WatchableValueInterface; properties: PropertiesSpecification; + values?: Map>; histogramSpecifications: HistogramSpecifications; histogramIndex: number; legendShaderOptions: LegendShaderOptions | undefined; @@ -50,6 +55,7 @@ export function propertyInvlerpLayerControl( const { watchableValue, properties, + values, histogramSpecifications, legendShaderOptions, histogramIndex, @@ -93,27 +99,42 @@ export function propertyInvlerpLayerControl( updateView(); options.labelContainer.appendChild(propertySelectElement); } - const derivedWatchableValue = { - changed: watchableValue.changed, - get value(): InvlerpParameters { - let { dataType, window, range } = watchableValue.value; - if (range === undefined) { - range = defaultDataTypeRange[dataType]; - } - return { - window: normalizeDataTypeInterval(window ?? range), - range, - }; - }, - set value(newValue: InvlerpParameters) { - const { window, range } = newValue; - watchableValue.value = { ...watchableValue.value, window, range }; - }, - }; + const derivedWatchableValue: WatchableValueInterface = + { + changed: watchableValue.changed, + get value() { + let { dataType, window, range } = watchableValue.value; + if (range === undefined) { + range = defaultDataTypeRange[dataType]; + } + return { + window: normalizeDataTypeInterval(window ?? range), + range, + }; + }, + set value(newValue: InvlerpParameters) { + const { window, range } = newValue; + watchableValue.value = { ...watchableValue.value, window, range }; + }, + }; + const derivedDataTypeWatchable = makeCachedDerivedWatchableValue( (p) => p.dataType, [watchableValue], ); + const derivedPropertyWatchable = makeCachedDerivedWatchableValue( + (p) => p.property, + [watchableValue], + ); + const derivedValuesWatchable = makeCachedLazyDerivedWatchableValue( + (property) => { + if (values) { + return values.get(property); + } + return undefined; + }, + derivedPropertyWatchable, + ); const control = context.registerDisposer( new VariableDataTypeInvlerpWidget( options.visibility, @@ -123,6 +144,7 @@ export function propertyInvlerpLayerControl( histogramSpecifications, histogramIndex, legendShaderOptions, + derivedValuesWatchable, ), ); return { control, controlElement: control.element }; diff --git a/src/widget/shader_code_widget.ts b/src/widget/shader_code_widget.ts index 05aed00769..bb64ac4a41 100644 --- a/src/widget/shader_code_widget.ts +++ b/src/widget/shader_code_widget.ts @@ -57,7 +57,7 @@ const SHADER_UPDATE_DELAY = 500; interface ShaderCodeState { shaderError: WatchableShaderError; shaderControlState?: ShaderControlState; - fragmentMain: WatchableValue; + fragment: WatchableValue; sourceStringNumber?: number; } @@ -70,7 +70,7 @@ export class ShaderCodeWidget extends RefCounted { private debouncedValueUpdater = debounce(() => { this.changingValue = true; try { - this.state.fragmentMain.value = this.textEditor.getValue(); + this.state.fragment.value = this.textEditor.getValue(); } finally { this.changingValue = false; } @@ -79,7 +79,7 @@ export class ShaderCodeWidget extends RefCounted { constructor(public state: ShaderCodeState) { super(); this.textEditor = CodeMirror((_element) => {}, { - value: this.state.fragmentMain.value, + value: this.state.fragment.value, mode: "glsl", gutters: ["CodeMirror-lint-markers"], }); @@ -88,9 +88,9 @@ export class ShaderCodeWidget extends RefCounted { this.debouncedValueUpdater(); }); this.registerDisposer( - this.state.fragmentMain.changed.add(() => { + this.state.fragment.changed.add(() => { if (!this.changingValue) { - this.textEditor.setValue(this.state.fragmentMain.value); + this.textEditor.setValue(this.state.fragment.value); } }), ); @@ -216,13 +216,14 @@ export function makeShaderCodeWidgetTopRow( href: string; }, className: string, + title = "Shader", ) { const spacer = document.createElement("div"); spacer.style.flex = "1"; const topRow = document.createElement("div"); topRow.className = className; - topRow.appendChild(document.createTextNode("Shader")); + topRow.appendChild(document.createTextNode(title)); topRow.appendChild(spacer); layer.registerDisposer( diff --git a/src/widget/shader_controls.ts b/src/widget/shader_controls.ts index fa4922bdcc..67f8889d1d 100644 --- a/src/widget/shader_controls.ts +++ b/src/widget/shader_controls.ts @@ -47,6 +47,7 @@ import { import { channelInvlerpLayerControl } from "#src/widget/layer_control_channel_invlerp.js"; import { checkboxLayerControl } from "#src/widget/layer_control_checkbox.js"; import { colorLayerControl } from "#src/widget/layer_control_color.js"; +import { propertyLayerControl } from "#src/widget/layer_control_property.js"; import { propertyInvlerpLayerControl } from "#src/widget/layer_control_property_invlerp.js"; import { rangeLayerControl } from "#src/widget/layer_control_range.js"; import { selectLayerControl } from "#src/widget/layer_control_select.js"; @@ -70,7 +71,7 @@ export interface ShaderControlsOptions { function getShaderLayerControlFactory( layerShaderControls: LayerShaderControls, controlId: string, -): LayerControlFactory | undefined { +): LayerControlFactory | undefined { const { shaderControlState } = layerShaderControls; const controlState = shaderControlState.state.get(controlId); if (controlState === undefined) return undefined; @@ -108,6 +109,7 @@ function getShaderLayerControlFactory( case "propertyInvlerp": { return propertyInvlerpLayerControl(() => ({ properties: control.properties, + values: control.values, watchableValue: controlState.trackable, histogramSpecifications: shaderControlState.histogramSpecifications, histogramIndex: calculateHistogramIndex(), @@ -125,6 +127,12 @@ function getShaderLayerControlFactory( histogramIndex: calculateHistogramIndex(), })); } + case "property": { + return propertyLayerControl(() => ({ + segmentProperties: control.segmentProperties, + watchableValue: controlState.trackable, + })); + } } function calculateHistogramIndex(controlType: string = control.type) { @@ -161,20 +169,20 @@ function getShaderLayerControlDefinition( getter: (layer: LayerType) => LayerShaderControls, toolId: string, controlId: string, -): LayerControlDefinition { +): LayerControlDefinition { return { label: controlId, toolJson: shaderControlToolJson(controlId, toolId), makeControl: (layer, context, options) => { const layerShaderControls = getter(layer); - return getShaderLayerControlFactory( + return getShaderLayerControlFactory( layerShaderControls, controlId, )!.makeControl(layer, context, options); }, activateTool: (activation, control) => { const layerShaderControls = getter(activation.tool.layer); - return getShaderLayerControlFactory( + return getShaderLayerControlFactory( layerShaderControls, controlId, )!.activateTool(activation, control); diff --git a/src/widget/transfer_function.ts b/src/widget/transfer_function.ts index 4cc788f727..22039398ea 100644 --- a/src/widget/transfer_function.ts +++ b/src/widget/transfer_function.ts @@ -1184,7 +1184,7 @@ out_color = tempColor * alpha; ); const histogramTextureUnit = lineShader.textureUnit( histogramSamplerTextureUnit, - ); + )!; gl.uniform1f( lineShader.uniform("uBoundsFraction"), getIntervalBoundsEffectiveFraction( From 234e312c6faadaf655c4c9b3908b498a507f01be Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Fri, 17 Jul 2026 14:08:57 -0400 Subject: [PATCH 02/16] cleanup --- src/layer/segmentation/index.ts | 9 +++++---- src/mesh/frontend.ts | 6 ++---- src/segmentation_display_state/frontend.ts | 15 +-------------- src/skeleton/frontend.ts | 3 +-- src/sliceview/volume/segmentation_renderlayer.ts | 1 - 5 files changed, 9 insertions(+), 25 deletions(-) diff --git a/src/layer/segmentation/index.ts b/src/layer/segmentation/index.ts index f6032dab98..42f372f3fd 100644 --- a/src/layer/segmentation/index.ts +++ b/src/layer/segmentation/index.ts @@ -144,11 +144,13 @@ import { makeWatchableShaderError, parameterizedEmitterDependentShaderGetter, } from "#src/webgl/dynamic_shader.js"; +import type { ShaderModule } from "#src/webgl/shader.js"; import { ShaderControlState } from "#src/webgl/shader_ui_controls.js"; import type { DependentViewContext } from "#src/widget/dependent_view_widget.js"; import { registerLayerShaderControlsTool } from "#src/widget/shader_controls.js"; const MAX_LAYER_BAR_UI_INDICATOR_COLORS = 6; +const emptySegmentColorShaderModule: ShaderModule = () => {}; export class SegmentationUserLayerGroupState extends RefCounted @@ -671,12 +673,12 @@ vColor = segmentColorUserShader(uint64_t(uID)); ); }; - context = () => {}; // TEMP how to get rid of this? - private tempColor = vec4.create(); getShaderSegmentColor = (id: bigint) => { - const { shader, parameters } = this.getSegmentColorShader(this.context); + const { shader, parameters } = this.getSegmentColorShader( + emptySegmentColorShaderModule, + ); if (shader === null) return; shader.bind(); const { gl } = shader; @@ -697,7 +699,6 @@ vColor = segmentColorUserShader(uint64_t(uID)); ); gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); const data = new Uint8Array(4); - // TODO can I read straight to float? gl.readPixels( 0, 0, diff --git a/src/mesh/frontend.ts b/src/mesh/frontend.ts index 3f6ff9f758..fb7acfd378 100644 --- a/src/mesh/frontend.ts +++ b/src/mesh/frontend.ts @@ -558,9 +558,8 @@ export class MeshLayer extends PerspectiveViewRenderLayer { + (objectId, pickIndex) => { const key = getObjectKey(objectId); const manifestChunk = manifestChunks.get(key); ++totalChunks; @@ -937,9 +936,8 @@ export class MultiscaleMeshLayer extends PerspectiveViewRenderLayer { + (objectId, pickIndex) => { const key = getObjectKey(objectId); const manifestChunk = chunks.get(key); ++totalManifestChunks; diff --git a/src/segmentation_display_state/frontend.ts b/src/segmentation_display_state/frontend.ts index e0f08408c9..09f601549c 100644 --- a/src/segmentation_display_state/frontend.ts +++ b/src/segmentation_display_state/frontend.ts @@ -1174,31 +1174,18 @@ export class SegmentationLayerSharedObject extends Base { export function forEachVisibleSegmentToDraw( displayState: SegmentationDisplayState3D, renderLayer: RenderLayer, - emitColor: boolean, pickIDs: PickIDManager | undefined, callback: ( objectId: bigint, - color: vec4 | undefined, pickIndex: number | undefined, rootObjectId: bigint, ) => void, ) { - emitColor; - // const alpha = Math.min(1, displayState.objectAlpha.value); - // const baseSegmentColoring = displayState.baseSegmentColoring.value; forEachVisibleSegment( displayState.segmentationGroupState.value, (objectId, rootObjectId) => { const pickIndex = pickIDs?.registerUint64(renderLayer, objectId); - const color = undefined; - // const color = emitColor - // ? getObjectColor( - // displayState, - // baseSegmentColoring ? objectId : rootObjectId, - // alpha, - // ) - // : undefined; - callback(objectId, color, pickIndex, rootObjectId); + callback(objectId, pickIndex, rootObjectId); }, ); } diff --git a/src/skeleton/frontend.ts b/src/skeleton/frontend.ts index fd09bff904..8f06b014b8 100644 --- a/src/skeleton/frontend.ts +++ b/src/skeleton/frontend.ts @@ -676,9 +676,8 @@ export class SkeletonLayer extends RefCounted { forEachVisibleSegmentToDraw( displayState, layer, - renderContext.emitColor, renderContext.emitPickID ? renderContext.pickIDs : undefined, - (objectId, _color, pickIndex) => { + (objectId, pickIndex) => { const key = getObjectKey(objectId); const skeleton = skeletons.get(key); if ( diff --git a/src/sliceview/volume/segmentation_renderlayer.ts b/src/sliceview/volume/segmentation_renderlayer.ts index f49d323c4c..f9795dba97 100644 --- a/src/sliceview/volume/segmentation_renderlayer.ts +++ b/src/sliceview/volume/segmentation_renderlayer.ts @@ -300,7 +300,6 @@ uint64_t getMappedObjectId(uint64_t value) { } else if (!has) { alpha = uNotSelectedAlpha; } - //alpha = 1.0; `; fragmentMain += ` vec4 rgba = segmentColorUserShader(valueForColor, has ? 0.5 : 0.75); From a4c88c3c2195072335ca90a0f4434c6c5e0af4ba Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Mon, 20 Jul 2026 11:11:00 -0400 Subject: [PATCH 03/16] fix: getBaseObjectColor should not add hover highlight --- src/layer/segmentation/index.ts | 8 +- .../segment_color_shader.browser_test.ts | 87 +++++++++++-------- src/segment_color.ts | 5 +- src/segmentation_display_state/frontend.ts | 62 +------------ .../volume/segmentation_renderlayer.ts | 1 + 5 files changed, 63 insertions(+), 100 deletions(-) diff --git a/src/layer/segmentation/index.ts b/src/layer/segmentation/index.ts index 42f372f3fd..c2b4473e05 100644 --- a/src/layer/segmentation/index.ts +++ b/src/layer/segmentation/index.ts @@ -675,7 +675,7 @@ vColor = segmentColorUserShader(uint64_t(uID)); private tempColor = vec4.create(); - getShaderSegmentColor = (id: bigint) => { + getShaderBaseSegmentColor = (id: bigint) => { const { shader, parameters } = this.getSegmentColorShader( emptySegmentColorShaderModule, ); @@ -691,6 +691,12 @@ vColor = segmentColorUserShader(uint64_t(uID)); gl, shader, parameters.shaderBuilderState, + undefined, + { + segmentDefaultColor: this.segmentDefaultColor.value, + segmentStatedColors: this.segmentStatedColors.value, + hoverHighlight: false, + }, ); gl.uniform2ui( shader.uniform(`uID`), diff --git a/src/layer/segmentation/segment_color_shader.browser_test.ts b/src/layer/segmentation/segment_color_shader.browser_test.ts index ffc2a56571..633cb343f3 100644 --- a/src/layer/segmentation/segment_color_shader.browser_test.ts +++ b/src/layer/segmentation/segment_color_shader.browser_test.ts @@ -67,14 +67,13 @@ const compareWithCPUHash = ( objectId: bigint, ) => { const outColor = - segmentationUserLayer.displayState.getShaderSegmentColor(objectId); + segmentationUserLayer.displayState.getShaderBaseSegmentColor(objectId); const colorGroupState = segmentationUserLayer.displayState.segmentationColorGroupState.value; const outColorCPU = vec4.create(); colorGroupState.segmentColorHash.compute(outColorCPU, objectId); expect(outColor).toBeDefined(); expect(outColor!.length).toBe(4); - // console.log("comparing colors", outColor!.join(','), outColorCPU.join(',')); for (let i = 0; i < 4; ++i) { expect(outColor![i]).toBeCloseTo(outColorCPU[i]); } @@ -114,7 +113,7 @@ const expectColor = ( expect([...color]).toEqual(expected.map((x) => expect.closeTo(x))); }; -describe("getShaderSegmentColor", () => { +describe("getShaderBaseSegmentColor", () => { it("default shader, return hash", () => { const segmentationUserLayer = setupSegmentationLayer(); const objectId = 1n; @@ -132,7 +131,7 @@ describe("getShaderSegmentColor", () => { return vec3(1.0, 0.0, 0.0); }`; const outColor = - segmentationUserLayer.displayState.getShaderSegmentColor(1n); + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n); expectColor(outColor!, [1.0, 0.0, 0.0, 0.0]); }); @@ -143,7 +142,7 @@ describe("getShaderSegmentColor", () => { return vec4(0.0, 0.0, 0.0, 0.5); }`; const outColor = - segmentationUserLayer.displayState.getShaderSegmentColor(1n); + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n); expectColor(outColor!, [0.0, 0.0, 0.0, 0.5]); }); @@ -154,12 +153,10 @@ describe("getShaderSegmentColor", () => { BigInt(packColor(vec4.fromValues(0.25, 0.5, 0.75, 0.5))), ); - expectColor(segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, [ - 0.25, - 0.5, - 0.75, - 128 / 255, - ]); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [0.25, 0.5, 0.75, 128 / 255], + ); }); it("treats rgb mapped segment colors as having undefined alpha", () => { @@ -174,11 +171,25 @@ describe("getShaderSegmentColor", () => { }`; expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [1.0, 0.0, 0.0, 0.75], ); }); + it("does not apply hover highlighting to offscreen color lookups", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.segmentStatedColors.value.set( + 1n, + BigInt(packColor(vec3.fromValues(1.0, 0.0, 0.0))), + ); + segmentationUserLayer.displayState.segmentSelectionState.set(1n); + + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + }); + it("returns undefined if segment properties have not been loaded and shader uses properties", () => { const segmentationUserLayer = setupSegmentationLayer(); segmentationUserLayer.displayState.fragmentSegmentColor.value = ` @@ -191,7 +202,7 @@ describe("getShaderSegmentColor", () => { }`; expect( - segmentationUserLayer.displayState.getShaderSegmentColor(1n), + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n), ).toBeUndefined(); }); @@ -226,15 +237,15 @@ describe("getShaderSegmentColor", () => { id: "color", }); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [1.0, 0.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, [0.0, 1.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(3n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(3n)!, [0.0, 0.0, 1.0, 0.0], ); }); @@ -270,7 +281,7 @@ describe("getShaderSegmentColor", () => { id: "color", }); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [0.0, 1.0, 0.0, 0.0], ); }); @@ -300,7 +311,7 @@ describe("getShaderSegmentColor", () => { id: "color", }); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [0.0, 1.0, 0.0, 0.0], ); }); @@ -343,19 +354,19 @@ vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { id: "blue", }); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [1.0, 0.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, [0.0, 0.0, 1.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(3n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(3n)!, [0.0, 1.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(0n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(0n)!, [0.0, 1.0, 0.0, 0.0], ); }); @@ -388,15 +399,15 @@ vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { return vec3(0.3, 0.6, 0.9); }`; expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [1.0, 0.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, [0.0, 0.0, 1.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(3n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(3n)!, [0.0, 1.0, 0.0, 0.0], ); }); @@ -424,7 +435,7 @@ vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { return vec3(0.0, 0.0, 0.0); }`; expect( - segmentationUserLayer.displayState.getShaderSegmentColor(1n), + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n), ).toBeUndefined(); const shaderError = segmentationUserLayer.displayState.shaderError.value; expect(shaderError).toBeInstanceOf(ShaderCompilationError); @@ -479,11 +490,11 @@ vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { id: "prop1", }); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [0.0, 0.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, [1.0, 0.0, 0.0, 0.0], ); }); @@ -512,11 +523,11 @@ vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { return vec3(0.0, 0.0, 0.0); }`; expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [0.0, 0.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, [1.0, 0.0, 0.0, 0.0], ); }); @@ -545,7 +556,7 @@ vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { return vec3(0.0, 0.0, 0.0); }`; expect( - segmentationUserLayer.displayState.getShaderSegmentColor(1n), + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n), ).toBeUndefined(); const shaderError = segmentationUserLayer.displayState.shaderError.value; expect(shaderError).toBeInstanceOf(ShaderCompilationError); @@ -597,15 +608,15 @@ vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { return vec3(0.5, 0.5, 0.5); }`; expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [1.0, 0.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, [0.0, 1.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(3n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(3n)!, [0.0, 0.0, 1.0, 0.0], ); }); @@ -632,11 +643,11 @@ vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { return vec3(normalized(), 0.0, 0.0); }`; expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [0.0, 0.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, [0.5, 0.0, 0.0, 0.0], ); }); @@ -667,11 +678,11 @@ vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { id: "prop1", }); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(1n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, [0.0, 0.0, 0.0, 0.0], ); expectColor( - segmentationUserLayer.displayState.getShaderSegmentColor(2n)!, + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, [0.75, 0.0, 0.0, 0.0], ); }); diff --git a/src/segment_color.ts b/src/segment_color.ts index fd65a7b063..e5c57daebf 100644 --- a/src/segment_color.ts +++ b/src/segment_color.ts @@ -266,6 +266,7 @@ export interface SegmentationColorUserShaderManagerParameters { export interface SegmentationColorUserShaderManagerInputs { segmentDefaultColor: vec3 | vec4 | undefined; segmentStatedColors: Uint64Map; + hoverHighlight: boolean; } export class SegmentColorUserShaderManager extends RefCounted { @@ -348,6 +349,7 @@ export class SegmentColorUserShaderManager extends RefCounted { return { segmentDefaultColor: this.displayState.segmentDefaultColor.value, segmentStatedColors: this.displayState.segmentStatedColors.value, + hoverHighlight: this.displayState.hoverHighlight.value, }; } @@ -856,8 +858,7 @@ vec4 segmentColorUserShader(uint64_t segmentId) { let selectedSegmentHigh = 0; const { segmentSelectionState } = this.displayState; const hasSelectedSegment = - segmentSelectionState.hasSelectedSegment && - displayState.hoverHighlight.value; + segmentSelectionState.hasSelectedSegment && inputs.hoverHighlight; gl.uniform1ui( shader.uniform("uHasSelectedSegment"), hasSelectedSegment ? 1 : 0, diff --git a/src/segmentation_display_state/frontend.ts b/src/segmentation_display_state/frontend.ts index 09f601549c..8cc80ac96d 100644 --- a/src/segmentation_display_state/frontend.ts +++ b/src/segmentation_display_state/frontend.ts @@ -212,7 +212,7 @@ export interface SegmentationDisplayState { selectSegment: (id: bigint, pin: boolean | "toggle" | "force-unpin") => void; filterBySegmentLabel: (id: bigint) => void; moveToSegment: (id: bigint) => void; - getShaderSegmentColor: (id: bigint) => Float32Array | undefined; + getShaderBaseSegmentColor: (id: bigint) => Float32Array | undefined; // Indirect properties hideSegmentZero: WatchableValueInterface; @@ -1063,64 +1063,8 @@ export function getBaseObjectColor( color.fill(1); return color; } - const { getShaderSegmentColor } = displayState; - return getShaderSegmentColor(objectId) ?? color; - // const colorGroupState = displayState.segmentationColorGroupState.value; - // const { segmentStatedColors } = colorGroupState; - // let statedColor: bigint | undefined; - // if ( - // segmentStatedColors.size !== 0 && - // (statedColor = colorGroupState.segmentStatedColors.get(objectId)) !== - // undefined - // ) { - // // If displayState maps the ID to a color, use it - // color[0] = Number(statedColor & 0x0000ffn) / 255.0; - // color[1] = (Number(statedColor & 0x00ff00n) >>> 8) / 255.0; - // color[2] = (Number(statedColor & 0xff0000n) >>> 16) / 255.0; - // return getShaderSegmentColor(objectId, color); - // } - // const segmentDefaultColor = colorGroupState.segmentDefaultColor.value; - // if (segmentDefaultColor !== undefined) { - // color[0] = segmentDefaultColor[0]; - // color[1] = segmentDefaultColor[1]; - // color[2] = segmentDefaultColor[2]; - // return getShaderSegmentColor(objectId, color); - // } - // colorGroupState.segmentColorHash.compute(color, objectId); // no need to do hash if we do it in the shader - // return getShaderSegmentColor(objectId, color); -} - -/** - * Returns the alpha-premultiplied color to use. - */ -export function getObjectColor( - displayState: SegmentationDisplayState, - objectId: bigint, - alpha = 1, -) { - const color = tempColor; - color[3] = alpha; - getBaseObjectColor(displayState, objectId, color); - // TODO move saturation to color shader - // let saturation = displayState.saturation.value; - // if ( - // displayState.hoverHighlight.value && - // displayState.segmentSelectionState.isSelected(objectId) - // ) { - // if (saturation > 0.5) { - // saturation = saturation -= 0.5; - // } else { - // saturation += 0.5; - // } - // } - // for (let i = 0; i < 3; ++i) { - // color[i] = color[i] * saturation + (1 - saturation); - // } - - // color[0] *= alpha; - // color[1] *= alpha; - // color[2] *= alpha; - return color; + const { getShaderBaseSegmentColor } = displayState; + return getShaderBaseSegmentColor(objectId) ?? color; } export function sendVisibleSegmentsState( diff --git a/src/sliceview/volume/segmentation_renderlayer.ts b/src/sliceview/volume/segmentation_renderlayer.ts index f9795dba97..2466e2ea7d 100644 --- a/src/sliceview/volume/segmentation_renderlayer.ts +++ b/src/sliceview/volume/segmentation_renderlayer.ts @@ -377,6 +377,7 @@ uint64_t getMappedObjectId(uint64_t value) { { segmentDefaultColor: activeSegmentDefaultColor, segmentStatedColors: activeSegmentStatedColors, + hoverHighlight: displayState.hoverHighlight.value, }, ); } From bb0946ea68201e648fb6773879c04615830e224f Mon Sep 17 00:00:00 2001 From: Chris Jordan Date: Mon, 20 Jul 2026 12:48:04 -0400 Subject: [PATCH 04/16] cleanup --- src/layer/segmentation/index.ts | 8 ++++---- src/sliceview/volume/segmentation_renderlayer.ts | 3 +-- src/webgl/texture_access.ts | 5 +---- src/widget/invlerp.ts | 4 ++-- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/layer/segmentation/index.ts b/src/layer/segmentation/index.ts index c2b4473e05..691b1a2787 100644 --- a/src/layer/segmentation/index.ts +++ b/src/layer/segmentation/index.ts @@ -427,7 +427,7 @@ class LinkedSegmentationGroupState< } } -export const DEFAULT_FRAGMENT_SEGMENT_COLOR = ` +export const DEFAULT_USER_MAIN_SEGMENT_COLOR = ` vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { return color; } @@ -579,7 +579,7 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { ), undefined, {}, - DEFAULT_FRAGMENT_SEGMENT_COLOR, + DEFAULT_USER_MAIN_SEGMENT_COLOR, ), ); @@ -614,7 +614,7 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { skeletonRenderingOptions = new SkeletonRenderingOptions(); shaderError = makeWatchableShaderError(); fragmentSegmentColor = makeTrackableFragmentMain( - DEFAULT_FRAGMENT_SEGMENT_COLOR, + DEFAULT_USER_MAIN_SEGMENT_COLOR, ); renderScaleHistogram = new RenderScaleHistogram(); @@ -651,7 +651,7 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { encodeParameters: (p) => { return `${p.shaderBuilderState.key}/${JSON.stringify(p.segmentColorParameters)}/${JSON.stringify(p.segmentColorProperties.map(encodeSegmentPropertyShaderDefinition))}`; }, - shaderError: this.layer.displayState.shaderError, // TODO can I reuse this? + shaderError: this.layer.displayState.shaderError, defineShader: (builder, { shaderBuilderState }) => { this.offscreenSegmentationColorUserShader.defineShader( builder, diff --git a/src/sliceview/volume/segmentation_renderlayer.ts b/src/sliceview/volume/segmentation_renderlayer.ts index 2466e2ea7d..4afb362e4b 100644 --- a/src/sliceview/volume/segmentation_renderlayer.ts +++ b/src/sliceview/volume/segmentation_renderlayer.ts @@ -160,7 +160,6 @@ export class SegmentationRenderLayer extends SliceViewVolumeRenderLayer x !== undefined, [displayState.highlightColor], @@ -228,7 +227,7 @@ export class SegmentationRenderLayer extends SliceViewVolumeRenderLayer Date: Wed, 22 Jul 2026 12:36:48 -0400 Subject: [PATCH 05/16] add batched GPU evaluation of segmentation colors segment_list: add prefetchBaseObjectColors - computes colors before rows are created add updateMany - recomputes colors for all currently rendered rows following display-state changes virtual_list: add optional VirtualListSource.prefetch(start, end) callback. It is called before missing rows in that interval are constructed --- src/layer/segmentation/index.ts | 71 ++++++++---- .../segment_color_shader.browser_test.ts | 19 ++++ src/segmentation_display_state/frontend.ts | 101 +++++++++++++++++- src/ui/segment_list.ts | 49 +++++---- src/widget/virtual_list.ts | 2 + 5 files changed, 200 insertions(+), 42 deletions(-) diff --git a/src/layer/segmentation/index.ts b/src/layer/segmentation/index.ts index 691b1a2787..6ebd6d3f7f 100644 --- a/src/layer/segmentation/index.ts +++ b/src/layer/segmentation/index.ts @@ -59,7 +59,7 @@ import type { import { augmentSegmentId, bindSegmentListWidth, - getBaseObjectColor, + getBaseObjectColors, makeSegmentWidget, maybeAugmentSegmentId, registerCallbackWhenSegmentationDisplayStateChanged, @@ -659,11 +659,12 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { shaderBuilderState, ); builder.addAttribute("highp vec4", "aVertexPosition"); - builder.addUniform("highp uvec2", "uID"); + builder.addAttribute("highp uvec2", "aID"); builder.addVarying("highp vec4", "vColor"); const vertexMain = ` -gl_Position = aVertexPosition; -vColor = segmentColorUserShader(uint64_t(uID)); +gl_Position = vec4(aVertexPosition.xy, 0.0, 1.0); +gl_PointSize = 1.0; +vColor = segmentColorUserShader(uint64_t(aID)); `; builder.addVertexMain(vertexMain); builder.addOutputBuffer("vec4", "out_fragColor", 0); @@ -675,18 +676,48 @@ vColor = segmentColorUserShader(uint64_t(uID)); private tempColor = vec4.create(); - getShaderBaseSegmentColor = (id: bigint) => { + getShaderBaseSegmentColors = ( + ids: readonly bigint[], + colors = new Float32Array(ids.length * 4), + ) => { + const numIds = ids.length; + if (numIds === 0) return colors; const { shader, parameters } = this.getSegmentColorShader( emptySegmentColorShaderModule, ); if (shader === null) return; shader.bind(); const { gl } = shader; + const canvas = gl.canvas as OffscreenCanvas; + if (canvas.width !== numIds || canvas.height !== 1) { + canvas.width = numIds; + canvas.height = 1; + } + gl.viewport(0, 0, numIds, 1); + + const positions = new Float32Array(numIds * 2); + const idsData = new Uint32Array(numIds * 2); + for (let i = 0; i < numIds; ++i) { + const id = ids[i]; + positions[2 * i] = (2 * (i + 0.5)) / numIds - 1; + positions[2 * i + 1] = 0; + idsData[2 * i] = Number(id & 0xffffffffn); + idsData[2 * i + 1] = Number(id >> 32n); + } const positionBuffer = GLBuffer.fromData( gl, - new Float32Array([1, 1, -1, 1, 1, -1, -1, -1]), + positions, + undefined, + WebGL2RenderingContext.STREAM_DRAW, ); positionBuffer.bindToVertexAttrib(shader.attribute("aVertexPosition"), 2); + const idBuffer = GLBuffer.fromData( + gl, + idsData, + undefined, + WebGL2RenderingContext.STREAM_DRAW, + ); + idBuffer.bindToVertexAttribI(shader.attribute("aID"), 2); this.offscreenSegmentationColorUserShader.enable( gl, shader, @@ -698,27 +729,29 @@ vColor = segmentColorUserShader(uint64_t(uID)); hoverHighlight: false, }, ); - gl.uniform2ui( - shader.uniform(`uID`), - Number(id & 0xffffffffn), - Number(id >> 32n), - ); - gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); - const data = new Uint8Array(4); + gl.drawArrays(gl.POINTS, 0, numIds); + const data = new Uint8Array(4 * numIds); gl.readPixels( 0, 0, - 1, + numIds, 1, WebGL2RenderingContext.RGBA, WebGL2RenderingContext.UNSIGNED_BYTE, data, ); for (let i = 0; i < data.length; i++) { - this.tempColor[i] = data[i] / 255.0; + colors[i] = data[i] / 255.0; } this.offscreenSegmentationColorUserShader.disable(gl, shader); - return this.tempColor; + positionBuffer.dispose(); + idBuffer.dispose(); + return colors; + }; + + getShaderBaseSegmentColor = (id: bigint) => { + const colors = this.getShaderBaseSegmentColors([id], this.tempColor); + return colors === undefined ? undefined : this.tempColor; }; linkedSegmentationGroup: LinkedLayerGroup; @@ -1552,9 +1585,9 @@ export class SegmentationUserLayer extends Base { } const visibleSegments = [...visibleSegmentsSet]; - const colors = visibleSegments.map((id) => { - // here we can do a batch get of colors using the segment color shader instead of one at a time - const color = getCssColor(getBaseObjectColor(displayState, id)); + const baseColors = getBaseObjectColors(displayState, visibleSegments); + const colors = visibleSegments.map((id, index) => { + const color = getCssColor(baseColors.subarray(4 * index, 4 * index + 4)); return { color, id }; }); diff --git a/src/layer/segmentation/segment_color_shader.browser_test.ts b/src/layer/segmentation/segment_color_shader.browser_test.ts index 633cb343f3..2174a159e1 100644 --- a/src/layer/segmentation/segment_color_shader.browser_test.ts +++ b/src/layer/segmentation/segment_color_shader.browser_test.ts @@ -176,6 +176,25 @@ describe("getShaderBaseSegmentColor", () => { ); }); + it("gets multiple colors with a single offscreen lookup", () => { + const segmentationUserLayer = setupSegmentationLayer(); + const { displayState } = segmentationUserLayer; + displayState.segmentStatedColors.value.set( + 1n, + BigInt(packColor(vec3.fromValues(1.0, 0.0, 0.0))), + ); + displayState.segmentStatedColors.value.set( + 2n, + BigInt(packColor(vec3.fromValues(0.0, 1.0, 0.0))), + ); + displayState.segmentDefaultColor.value = vec3.fromValues(0.0, 0.0, 1.0); + + const colors = displayState.getShaderBaseSegmentColors([1n, 2n, 3n])!; + expectColor(colors.subarray(0, 4) as vec4, [1.0, 0.0, 0.0, 0.0]); + expectColor(colors.subarray(4, 8) as vec4, [0.0, 1.0, 0.0, 0.0]); + expectColor(colors.subarray(8, 12) as vec4, [0.0, 0.0, 1.0, 0.0]); + }); + it("does not apply hover highlighting to offscreen color lookups", () => { const segmentationUserLayer = setupSegmentationLayer(); segmentationUserLayer.displayState.segmentStatedColors.value.set( diff --git a/src/segmentation_display_state/frontend.ts b/src/segmentation_display_state/frontend.ts index 8cc80ac96d..159350d3d4 100644 --- a/src/segmentation_display_state/frontend.ts +++ b/src/segmentation_display_state/frontend.ts @@ -213,6 +213,10 @@ export interface SegmentationDisplayState { filterBySegmentLabel: (id: bigint) => void; moveToSegment: (id: bigint) => void; getShaderBaseSegmentColor: (id: bigint) => Float32Array | undefined; + getShaderBaseSegmentColors: ( + ids: readonly bigint[], + colors?: Float32Array, + ) => Float32Array | undefined; // Indirect properties hideSegmentZero: WatchableValueInterface; @@ -413,6 +417,8 @@ const segmentWidgetTemplateWithUnmapped = (() => { export type SegmentWidgetTemplate = typeof segmentWidgetTemplate; +type SegmentColorMap = ReadonlyMap; + interface SegmentWidgetWithExtraColumnsTemplate extends SegmentWidgetTemplate { numericalPropertyIndices: number[]; } @@ -623,6 +629,7 @@ export class SegmentWidgetFactory