diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst index 4230a8380c..a645a53f7b 100644 --- a/docs/user-guide/index.rst +++ b/docs/user-guide/index.rst @@ -12,6 +12,8 @@ Topics to segmentation layers. - :doc:`annotation_shaders` — writing custom GLSL shader code to drive the visual appearance of annotations from their properties. +- :doc:`segmentation_shaders` — writing custom GLSL shader code to color + segmentation layers from segment ids and segment properties. .. toctree:: :hidden: @@ -19,3 +21,4 @@ Topics navigation annotations annotation_shaders + segmentation_shaders diff --git a/docs/user-guide/segmentation_shaders.rst b/docs/user-guide/segmentation_shaders.rst new file mode 100644 index 0000000000..422eb11e87 --- /dev/null +++ b/docs/user-guide/segmentation_shaders.rst @@ -0,0 +1,171 @@ +Segmentation Shaders +-------------------- + +Segmentation layers can use custom GLSL code to change how segments are colored. +The technical reference for the shader API is in +`the segmentation rendering guide `_. +This page is a gentler introduction to the same ideas. + +The Segment Color Function +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A segmentation shader defines a ``segmentColor`` function. The default shader +keeps Neuroglancer's existing segment color unchanged: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return color; + } + +The ``color`` argument is the color Neuroglancer would normally use for the +segment. It may come from the segment color hash, the layer's default segment +color, or an explicit color assigned to that segment. The ``isStated`` argument +is ``true`` when ``color`` came from an explicit segment color. The +``hasProperties`` argument is ``true`` when Neuroglancer found segment property +data for the current segment. + +To make every visible segment red, return a red ``vec3``: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return vec3(1.0, 0.0, 0.0); + } + +If you want to preserve explicit per-segment colors but recolor everything +else, use ``isStated``: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (isStated) { + return color; + } + return vec3(0.0, 0.6, 1.0); + } + +Opacity +~~~~~~~ + +Use a ``vec4`` return type to set opacity. The alpha channel is the fourth +component. Returning a negative alpha leaves the layer opacity unchanged. + +.. code-block:: text + + vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + if (isStated) { + return color; + } + return vec4(color.rgb, 0.35); + } + +Segment Properties +~~~~~~~~~~~~~~~~~~ + +Segment properties can drive color choices. If a segment property map is +available, a shader can read tags and properties directly by name: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + if (tag("axon")) { + return vec3(1.0, 0.4, 0.0); + } + if (prop("size") > 100u) { + return vec3(1.0, 1.0, 0.0); + } + return color; + } + +``tag("axon")`` returns ``true`` when the current segment has that tag. +``prop("size")`` reads a numerical property named ``size``. For string +properties, compare the value with a string literal: + +.. code-block:: text + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + if (prop("class") == "interneuron") { + return vec3(0.0, 1.0, 0.6); + } + return color; + } + +Property Controls +~~~~~~~~~~~~~~~~~ + +A ``property`` UI control lets the user choose which segment property a shader +uses. The control can be filtered to tags, numerical properties, or string +properties. + +.. code-block:: glsl + + #uicontrol property selectedTag(type="tag") + #uicontrol property selectedSize(type="number") + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + if (selectedTag) { + return vec3(1.0, 0.0, 0.0); + } + if (selectedSize > 100u) { + return vec3(1.0, 1.0, 0.0); + } + return color; + } + +Use ``type="string"`` for a string property picker. ``type="number"`` and +``type="numerical"`` both select numerical properties. + +Data Mapping +~~~~~~~~~~~~ + +For continuous numerical properties, an ``invlerp`` control maps a property +range to ``0-1``. The user can adjust the selected property and range from the +shader UI. + +.. code-block:: glsl + + #uicontrol float intensity invlerp(property="size", range=[0, 1000]) + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + return intensity() * vec3(1.0, 0.2, 0.0); + } + +If ``property`` is omitted, Neuroglancer selects the first available numerical +segment property as the default. ``range`` controls the data values mapped to +``0`` and ``1``. ``window`` can be added to control the range shown by the UI +widget. + +Colormaps +~~~~~~~~~ + +The remapped value from an ``invlerp`` control can be passed to a colormap: + +.. code-block:: glsl + + #uicontrol float intensity invlerp(property="size", range=[0, 1000]) + + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5, 0.5, 0.5); + } + if (isStated) { + return color; + } + return colormapJet(intensity()); + } + +This pattern is useful when a numerical segment property should control color +continuously while still respecting explicitly assigned segment colors. \ No newline at end of file diff --git a/src/annotation/type_handler.browser_test.ts b/src/annotation/type_handler.browser_test.ts new file mode 100644 index 0000000000..126e97b8e6 --- /dev/null +++ b/src/annotation/type_handler.browser_test.ts @@ -0,0 +1,107 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "#src/annotation/bounding_box.js"; +import "#src/annotation/ellipsoid.js"; +import "#src/annotation/line.js"; +import "#src/annotation/point.js"; +import "#src/annotation/polyline.js"; + +import { describe, expect, it } from "vitest"; +import type { AnnotationPropertySpec } from "#src/annotation/index.js"; +import { AnnotationType } from "#src/annotation/index.js"; +import { + type AnnotationShaderGetter, + getAnnotationTypeRenderHandler, +} from "#src/annotation/type_handler.js"; +import { WatchableValue } from "#src/trackable_value.js"; +import { initializeWebGL } from "#src/webgl/context.js"; +import { + makeTrackableFragmentMain, + makeWatchableShaderError, +} from "#src/webgl/dynamic_shader.js"; +import { + getFallbackBuilderState, + parseShaderUiControls, + ShaderControlState, +} from "#src/webgl/shader_ui_controls.js"; + +describe("annotation property shaders", () => { + it("compiles a shader that colors by boolean property", () => { + const canvas = document.createElement("canvas"); + const gl = initializeWebGL(canvas); + const fragmentMain = makeTrackableFragmentMain(` +void main() { + if (prop_highlight() != 0u) { + setColor(vec3(1.0, 0.0, 0.0)); + } else { + setColor(vec3(0.0, 0.0, 1.0)); + } +} +`); + const shaderControlState = new ShaderControlState(fragmentMain); + const fallbackShaderParameters = new WatchableValue( + getFallbackBuilderState(parseShaderUiControls(fragmentMain.value)), + ); + const shaderError = makeWatchableShaderError(); + const properties: AnnotationPropertySpec[] = [ + { + identifier: "highlight", + description: undefined, + type: "bool", + default: 0, + }, + ]; + const renderHandler = getAnnotationTypeRenderHandler(AnnotationType.POINT); + const renderHelper = new renderHandler.perspectiveViewRenderHelper( + gl, + AnnotationType.POINT, + /*rank=*/ 3, + properties, + shaderControlState, + fallbackShaderParameters, + shaderError, + ); + renderHelper.targetIsSliceView = false; + renderHelper.pickIdsPerInstance = renderHandler.pickIdsPerInstance; + + const shaderGetter = ( + renderHelper as typeof renderHelper & { + shaderGetter3d: AnnotationShaderGetter; + } + ).shaderGetter3d; + const shaderResult = shaderGetter((builder) => { + builder.addOutputBuffer("vec4", "out_color", 0); + builder.addFragmentCode(` +void emit(vec4 color, highp uint pickId) { + out_color = color; +} +`); + }); + + expect(shaderError.value).toBeNull(); + expect(shaderResult.shader).not.toBeNull(); + expect(shaderResult.shader!.vertexSource).toContain( + "highp uint prop_highlight()", + ); + expect(shaderResult.shader!.vertexSource).toContain( + "if (prop_highlight() != 0u)", + ); + + renderHelper.dispose(); + shaderControlState.dispose(); + }); +}); diff --git a/src/annotation/type_handler.ts b/src/annotation/type_handler.ts index c8fae8d7c4..408173f0cb 100644 --- a/src/annotation/type_handler.ts +++ b/src/annotation/type_handler.ts @@ -42,6 +42,7 @@ import { parameterizedEmitterDependentShaderGetter, shaderCodeWithLineDirective, } from "#src/webgl/dynamic_shader.js"; +import { copyHistogramToCPU } from "#src/webgl/empirical_cdf.js"; import { defineInvlerpShaderFunction, enableLerpShaderFunction, @@ -268,10 +269,10 @@ class AnnotationRenderHelperBase extends RefCounted { protected defineProperties( builder: ShaderBuilder, - referencedProperties: number[], + annotationPropertyIndices: number[], ) { const { properties, rank } = this; - for (const i of referencedProperties) { + for (const i of annotationPropertyIndices) { const property = properties[i]; const handler = annotationPropertyTypeRenderHandlers[property.type]; handler.defineShader(builder, property.identifier, rank); @@ -279,7 +280,7 @@ class AnnotationRenderHelperBase extends RefCounted { const { propertyOffsets } = this; const { propertyGroupBytes, propertyGroupCumulativeBytes } = this; builder.addInitializer((shader) => { - const binders = referencedProperties.map( + const binders = annotationPropertyIndices.map( (i) => shader.vertexShaderInputBinders[`prop_${properties[i].identifier}`], ); @@ -293,7 +294,7 @@ class AnnotationRenderHelperBase extends RefCounted { bind(stride: number, offset: number) { for (let i = 0; i < numProperties; ++i) { const { group, offset: propertyOffset } = - propertyOffsets[referencedProperties[i]]; + propertyOffsets[annotationPropertyIndices[i]]; binders[i].bind( /*stride=*/ propertyGroupBytes[group], /*offset=*/ offset + @@ -349,8 +350,11 @@ export abstract class AnnotationRenderHelper extends AnnotationRenderHelperBase parameters: ShaderControlsBuilderState, ) => { const { rank, properties } = this; - const referencedProperties: number[] = []; - const controlsReferencedProperties = parameters.referencedProperties; + const annotationPropertyIndices: number[] = []; + const controlAnnotationProperties = + parameters.propertyReferences.source === "annotation" + ? parameters.propertyReferences.references + : []; const processedCode = parameters.parseResult.code; for ( let i = 0, numProperties = properties.length; @@ -360,14 +364,14 @@ export abstract class AnnotationRenderHelper extends AnnotationRenderHelperBase const property = properties[i]; const functionName = `prop_${property.identifier}`; if ( - !controlsReferencedProperties.includes(property.identifier) && + !controlAnnotationProperties.includes(property.identifier) && !processedCode.match(new RegExp(`\\b${functionName}\\b`)) ) { continue; } - referencedProperties.push(i); + annotationPropertyIndices.push(i); } - this.defineProperties(builder, referencedProperties); + this.defineProperties(builder, annotationPropertyIndices); builder.addUniform("highp vec3", "uColor"); builder.addUniform("highp uint", "uSelectedIndex"); builder.addVarying("highp vec4", "vColor"); @@ -444,7 +448,7 @@ float getMaxSubspaceClipCoefficient(float modelPointA[${this.rank}], float mode } `); - addControlsToBuilder(parameters, builder); + addControlsToBuilder(parameters, builder, /*fragment=*/ false); builder.addVertexCode(` const bool PROJECTION_VIEW = ${!this.targetIsSliceView}; bool ng_discardValue; @@ -776,21 +780,12 @@ gl_PointSize = 1.0; } gl.drawArrays(WebGL2RenderingContext.POINTS, 0, context.count); if (DEBUG_HISTOGRAMS) { - const tempBuffer = new Float32Array(256 * 4); - gl.readPixels( - 0, - 0, - 256, - 1, - WebGL2RenderingContext.RGBA, - WebGL2RenderingContext.FLOAT, - tempBuffer, + const histogram = copyHistogramToCPU(gl); + console.log( + "histogram property:", + propertyIdentifier, + histogram.join(" "), ); - const tempBuffer2 = new Float32Array(256); - for (let j = 0; j < 256; ++j) { - tempBuffer2[j] = tempBuffer[j * 4]; - } - console.log("histogram", tempBuffer2.join(" ")); } binder.disable(); break; diff --git a/src/gpu_hash/shader.ts b/src/gpu_hash/shader.ts index af44a0b97b..9140bdf899 100644 --- a/src/gpu_hash/shader.ts +++ b/src/gpu_hash/shader.ts @@ -119,7 +119,10 @@ export class HashSetShaderManager { this.readTable = prefix + "_readTable"; } - defineShader(builder: ShaderBuilder) { + defineShader(builder: ShaderBuilder, fragment = true) { + const addCode = fragment + ? builder.addFragmentCode.bind(builder) + : builder.addVertexCode.bind(builder); const { hashSeedsName, samplerName, numAlternatives, hashKeyMask } = this; builder.addUniform("highp uint", hashSeedsName, numAlternatives); builder.addUniform("highp uint", hashKeyMask); @@ -128,11 +131,11 @@ export class HashSetShaderManager { samplerName, this.textureUnitSymbol, ); - builder.addFragmentCode(glsl_hashCombine); - builder.addFragmentCode(glsl_uint64); - builder.addFragmentCode(glsl_equalUint64); + addCode(glsl_hashCombine); + addCode(glsl_uint64); + addCode(glsl_equalUint64); this.accessHelper.defineShader(builder); - builder.addFragmentCode( + addCode( this.accessHelper.getAccessor( this.readTable, this.samplerName, @@ -159,7 +162,7 @@ bool ${this.hasFunctionName}(uint64_t x) { return false; } `; - builder.addFragmentCode(s); + addCode(s); } get hasFunctionName() { @@ -172,7 +175,7 @@ bool ${this.hasFunctionName}(uint64_t x) { hashTable: GPUHashTable, ) { hashTable.copyToGPU(); - const textureUnit = shader.textureUnit(this.textureUnitSymbol); + const textureUnit = shader.textureUnit(this.textureUnitSymbol)!; gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + textureUnit); gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, hashTable.texture); gl.uniform1ui( @@ -186,15 +189,18 @@ bool ${this.hasFunctionName}(uint64_t x) { } disable(gl: GL, shader: ShaderProgram) { - const textureUnit = shader.textureUnit(this.textureUnitSymbol); + const textureUnit = shader.textureUnit(this.textureUnitSymbol)!; gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + textureUnit); gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, null); } } export class HashMapShaderManager extends HashSetShaderManager { - defineShader(builder: ShaderBuilder) { - super.defineShader(builder); + defineShader(builder: ShaderBuilder, fragment = true) { + const addCode = fragment + ? builder.addFragmentCode.bind(builder) + : builder.addVertexCode.bind(builder); + super.defineShader(builder, fragment); const { numAlternatives, hashSeedsName, hashKeyMask } = this; let s = ` bool ${this.getFunctionName}(uint64_t x, out uint64_t value) { @@ -215,7 +221,7 @@ bool ${this.getFunctionName}(uint64_t x, out uint64_t value) { return false; } `; - builder.addFragmentCode(s); + addCode(s); } get getFunctionName() { diff --git a/src/layer/annotation/index.ts b/src/layer/annotation/index.ts index 55c1a33463..ee119452f2 100644 --- a/src/layer/annotation/index.ts +++ b/src/layer/annotation/index.ts @@ -849,7 +849,7 @@ export class AnnotationUserLayer extends Base { function makeShaderCodeWidget(layer: AnnotationUserLayer) { return new ShaderCodeWidget({ shaderError: layer.annotationDisplayState.shaderError, - fragmentMain: layer.annotationDisplayState.shader, + fragment: layer.annotationDisplayState.shader, shaderControlState: layer.annotationDisplayState.shaderControls, }); } diff --git a/src/layer/image/index.ts b/src/layer/image/index.ts index b58b7cbaa0..802f43128a 100644 --- a/src/layer/image/index.ts +++ b/src/layer/image/index.ts @@ -471,7 +471,7 @@ export class ImageUserLayer extends Base { function makeShaderCodeWidget(layer: ImageUserLayer) { return new ShaderCodeWidget({ shaderError: layer.shaderError, - fragmentMain: layer.fragmentMain, + fragment: layer.fragmentMain, shaderControlState: layer.shaderControlState, }); } diff --git a/src/layer/index.ts b/src/layer/index.ts index fad0e1400e..0d3fb57d9d 100644 --- a/src/layer/index.ts +++ b/src/layer/index.ts @@ -364,6 +364,9 @@ export class UserLayer extends RefCounted { get isReady() { return this.loadingCounter === 0; } + isReadyWatchable: WatchableValueInterface = new WatchableValue( + false, + ); tabs = this.registerDisposer(new TabSpecification()); panels = new UserLayerSidePanelsState(this); @@ -379,6 +382,11 @@ export class UserLayer extends RefCounted { constructor(public managedLayer: Borrowed) { super(); + this.registerDisposer( + this.readyStateChanged.add(() => { + this.isReadyWatchable.value = this.isReady; + }), + ); this.toolBinder = this.registerDisposer( new LayerToolBinder(this, this.manager.root.toolBinder), ); diff --git a/src/layer/segmentation/index.ts b/src/layer/segmentation/index.ts index 8fc24b8fd4..3c32d42090 100644 --- a/src/layer/segmentation/index.ts +++ b/src/layer/segmentation/index.ts @@ -45,7 +45,13 @@ import { RenderScaleHistogram, trackableRenderScaleTarget, } from "#src/render_scale_statistics.js"; -import { getCssColor, SegmentColorHash } from "#src/segment_color.js"; +import { + DEFAULT_USER_MAIN_SEGMENT_COLOR, + encodeSegmentPropertyShaderDefinition, + getCssColor, + SegmentColorHash, + SegmentColorUserShaderManager, +} from "#src/segment_color.js"; import type { SegmentationColorGroupState, SegmentationDisplayState, @@ -54,7 +60,7 @@ import type { import { augmentSegmentId, bindSegmentListWidth, - getBaseObjectColor, + getBaseObjectColors, makeSegmentWidget, maybeAugmentSegmentId, registerCallbackWhenSegmentationDisplayStateChanged, @@ -92,6 +98,7 @@ import type { WatchableValueInterface, } from "#src/trackable_value.js"; import { + AggregateWatchableValue, IndirectTrackableValue, IndirectWatchableValue, makeCachedDerivedWatchableValue, @@ -109,6 +116,7 @@ import { DisplayOptionsTab } from "#src/ui/segmentation_display_options_tab.js"; import { Uint64Map } from "#src/uint64_map.js"; import { Uint64OrderedSet } from "#src/uint64_ordered_set.js"; import { Uint64Set } from "#src/uint64_set.js"; +import type { TypedArray } from "#src/util/array.js"; import { gatherUpdate } from "#src/util/array.js"; import { packColor, @@ -119,7 +127,8 @@ import { } from "#src/util/color.js"; import type { Borrowed, Owned } from "#src/util/disposable.js"; import { RefCounted } from "#src/util/disposable.js"; -import type { vec3, vec4 } from "#src/util/geom.js"; +import type { vec3 } from "#src/util/geom.js"; +import { vec4 } from "#src/util/geom.js"; import { parseArray, parseUint64, @@ -129,11 +138,26 @@ import { verifyString, } from "#src/util/json.js"; import { Signal } from "#src/util/signal.js"; -import { makeWatchableShaderError } from "#src/webgl/dynamic_shader.js"; +import { GLBuffer } from "#src/webgl/buffer.js"; +import { initializeWebGL } from "#src/webgl/context.js"; +import type { WatchableShaderError } from "#src/webgl/dynamic_shader.js"; +import { + makeAggregateWatchableShaderError, + makeTrackableFragmentMain, + makeWatchableShaderError, + parameterizedEmitterDependentShaderGetter, +} from "#src/webgl/dynamic_shader.js"; +import type { ShaderModule } from "#src/webgl/shader.js"; +import { + getFallbackBuilderState, + parseShaderUiControls, + ShaderControlState, +} from "#src/webgl/shader_ui_controls.js"; import type { DependentViewContext } from "#src/widget/dependent_view_widget.js"; import { registerLayerShaderControlsTool } from "#src/widget/shader_controls.js"; const MAX_LAYER_BAR_UI_INDICATOR_COLORS = 6; +const emptySegmentColorShaderModule: ShaderModule = () => {}; export class SegmentationUserLayerGroupState extends RefCounted @@ -411,11 +435,22 @@ class LinkedSegmentationGroupState< } class SegmentationUserLayerDisplayState implements SegmentationDisplayState { + private getSegmentColorShader; + + segmentColorShaderControlState: ShaderControlState; + segmentationColorUserShader: SegmentColorUserShaderManager; + constructor(public layer: SegmentationUserLayer) { // Even though `SegmentationUserLayer` assigns this to its `displayState` property, redundantly // assign it here first in order to allow it to be accessed by `segmentationGroupState`. layer.displayState = this; + this.shaderError = makeAggregateWatchableShaderError(this.layer, [ + this.volumeShaderError, + this.meshShaderError, + this.offscreenShaderError, + ]); + this.linkedSegmentationGroup = layer.registerDisposer( new LinkedLayerGroup( layer.manager.rootLayers, @@ -520,8 +555,67 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { (group) => group.segmentPropertyMap, ), ); + + this.segmentColorShaderControlState = this.layer.registerDisposer( + new ShaderControlState( + this.fragmentSegmentColor, + makeCachedLazyDerivedWatchableValue( + (segmentPropertyMap, isReady) => { + const properties = new Map(); + const values = new Map>(); + if (segmentPropertyMap === undefined) { + // dont return a non null before layer.isReady to prevent losing shader control state during loading process + return isReady ? {} : null; + } + for (const property of segmentPropertyMap.numericalProperties) { + properties.set(property.id, property.dataType); + values.set(property.id, property.values); + } + const getPropertyValueExpression = (property: string) => { + const identifier = + SegmentColorUserShaderManager.getNumericalPropertyShaderIdentifier( + segmentPropertyMap, + property, + ); + if (identifier === undefined) { + throw new Error(`Unknown numerical property: ${property}`); + } + return identifier; + }; + + return { + properties, + values, + propertySource: "segment" as const, + getPropertyValueExpression, + segmentPropertyMap, + }; + }, + this.segmentationGroupState.value.segmentPropertyMap, + this.layer.isReadyWatchable, + ), + undefined, + {}, + DEFAULT_USER_MAIN_SEGMENT_COLOR, + ), + ); + + // 2d/3d share GL context + // the offscreen canvas is a separate context that doesn't share textures + this.segmentationColorUserShader = new SegmentColorUserShaderManager( + this, + this.layer.manager.chunkManager.chunkQueueManager.gl, + ); + + this.offscreenGL = initializeWebGL(new OffscreenCanvas(1, 1)); + this.offscreenSegmentationColorUserShader = + new SegmentColorUserShaderManager(this, this.offscreenGL); + this.getSegmentColorShader = this.makeSegmentColorShaderGetter(); } + offscreenGL; + offscreenSegmentationColorUserShader; + segmentSelectionState = new SegmentSelectionState(); selectedAlpha = trackableAlphaValue(0.5); saturation = trackableAlphaValue(1.0); @@ -535,7 +629,14 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { objectAlpha = trackableAlphaValue(1.0); ignoreNullVisibleSet = new TrackableBoolean(true, true); skeletonRenderingOptions = new SkeletonRenderingOptions(); - shaderError = makeWatchableShaderError(); + volumeShaderError = makeWatchableShaderError(); + meshShaderError = makeWatchableShaderError(); + offscreenShaderError = makeWatchableShaderError(); + shaderError: WatchableShaderError; + fragmentSegmentColor = makeTrackableFragmentMain( + DEFAULT_USER_MAIN_SEGMENT_COLOR, + ); + renderScaleHistogram = new RenderScaleHistogram(); renderScaleTarget = trackableRenderScaleTarget(1); selectSegment: (id: bigint, pin: boolean | "toggle") => void; @@ -551,6 +652,145 @@ class SegmentationUserLayerDisplayState implements SegmentationDisplayState { this.layer.moveToSegment(id); }; + makeSegmentColorShaderGetter = () => { + const parameters = this.layer.registerDisposer( + new AggregateWatchableValue(() => ({ + segmentColorParameters: + this.segmentationColorUserShader.shaderParameters, + segmentColorProperties: + this.offscreenSegmentationColorUserShader.usedProperties, + shaderBuilderState: this.segmentColorShaderControlState.builderState, + })), + ); + const fallbackParameters = new WatchableValue({ + segmentColorParameters: + this.segmentationColorUserShader.shaderParameters.value, + segmentColorProperties: [], + shaderBuilderState: getFallbackBuilderState( + parseShaderUiControls(DEFAULT_USER_MAIN_SEGMENT_COLOR), + ), + }); + return parameterizedEmitterDependentShaderGetter( + this.layer, + this.offscreenGL, + { + memoizeKey: `segmentation/ColorShader`, + parameters, + fallbackParameters, + encodeParameters: (p) => { + return `${p.shaderBuilderState.key}/${JSON.stringify(p.segmentColorParameters)}/${JSON.stringify(p.segmentColorProperties.map(encodeSegmentPropertyShaderDefinition))}`; + }, + shaderError: this.layer.displayState.offscreenShaderError, + defineShader: ( + builder, + { segmentColorParameters, shaderBuilderState }, + ) => { + this.offscreenSegmentationColorUserShader.defineShader( + builder, + /*fragment=*/ false, + shaderBuilderState, + segmentColorParameters, + ); + builder.addAttribute("highp vec4", "aVertexPosition"); + builder.addAttribute("highp uvec2", "aID"); + builder.addVarying("highp vec4", "vColor"); + const vertexMain = ` +gl_Position = vec4(aVertexPosition.xy, 0.0, 1.0); +gl_PointSize = 1.0; +vColor = segmentColorUserShader(uint64_t(aID)); +`; + builder.addVertexMain(vertexMain); + builder.addOutputBuffer("vec4", "out_fragColor", 0); + builder.setFragmentMain("out_fragColor = vColor;"); + }, + }, + ); + }; + + private tempColor = vec4.create(); + + getShaderBaseSegmentColors = ( + ids: readonly bigint[], + colors = new Float32Array(ids.length * 4), + ) => { + const numIds = ids.length; + if (numIds === 0) return colors; + const { shader, parameters } = this.getSegmentColorShader( + emptySegmentColorShaderModule, + ); + if (shader === null) return; + shader.bind(); + const { gl } = shader; + const canvas = gl.canvas as OffscreenCanvas; + if (canvas.width !== numIds || canvas.height !== 1) { + canvas.width = numIds; + canvas.height = 1; + } + gl.viewport(0, 0, numIds, 1); + + const positions = new Float32Array(numIds * 2); + const idsData = new Uint32Array(numIds * 2); + for (let i = 0; i < numIds; ++i) { + const id = ids[i]; + positions[2 * i] = (2 * (i + 0.5)) / numIds - 1; + positions[2 * i + 1] = 0; + idsData[2 * i] = Number(id & 0xffffffffn); + idsData[2 * i + 1] = Number(id >> 32n); + } + const positionBuffer = GLBuffer.fromData( + gl, + positions, + undefined, + WebGL2RenderingContext.STREAM_DRAW, + ); + positionBuffer.bindToVertexAttrib(shader.attribute("aVertexPosition"), 2); + const idBuffer = GLBuffer.fromData( + gl, + idsData, + undefined, + WebGL2RenderingContext.STREAM_DRAW, + ); + idBuffer.bindToVertexAttribI(shader.attribute("aID"), 2); + this.offscreenSegmentationColorUserShader.enable( + gl, + shader, + parameters.shaderBuilderState, + parameters.segmentColorParameters, + { + segmentDefaultColor: this.segmentDefaultColor.value, + segmentStatedColors: this.segmentStatedColors.value, + hoverHighlight: false, + }, + ); + gl.drawArrays(gl.POINTS, 0, numIds); + const data = new Uint8Array(4 * numIds); + gl.readPixels( + 0, + 0, + numIds, + 1, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + data, + ); + for (let i = 0; i < data.length; i++) { + colors[i] = data[i] / 255.0; + } + this.offscreenSegmentationColorUserShader.disable( + gl, + shader, + parameters.segmentColorParameters, + ); + positionBuffer.dispose(); + idBuffer.dispose(); + return colors; + }; + + getShaderBaseSegmentColor = (id: bigint) => { + const colors = this.getShaderBaseSegmentColors([id], this.tempColor); + return colors === undefined ? undefined : this.tempColor; + }; + linkedSegmentationGroup: LinkedLayerGroup; linkedSegmentationColorGroup: LinkedLayerGroup; originalSegmentationGroupState: SegmentationUserLayerGroupState; @@ -689,6 +929,12 @@ export class SegmentationUserLayer extends Base { this.displayState.linkedSegmentationGroup.changed.add(() => this.updateDataSubsourceActivations(), ); + this.displayState.fragmentSegmentColor.changed.add( + this.specificationChanged.dispatch, + ); + this.displayState.segmentColorShaderControlState.changed.add( + this.specificationChanged.dispatch, + ); this.tabs.add("rendering", { label: "Render", order: -100, @@ -996,6 +1242,9 @@ export class SegmentationUserLayer extends Base { this.displayState.ignoreNullVisibleSet.restoreState( specification[json_keys.IGNORE_NULL_VISIBLE_SET_JSON_KEY], ); + this.displayState.fragmentSegmentColor.restoreState( + specification[json_keys.SEGMENT_COLOR_SHADER_JSON_KEY], + ); const { skeletonRenderingOptions } = this.displayState; skeletonRenderingOptions.restoreState( @@ -1040,6 +1289,9 @@ export class SegmentationUserLayer extends Base { this.displayState.segmentationColorGroupState.value.restoreState( specification, ); + this.displayState.segmentColorShaderControlState.restoreState( + specification[json_keys.SHADER_CONTROLS_JSON_KEY], + ); } toJSON() { @@ -1068,6 +1320,8 @@ export class SegmentationUserLayer extends Base { this.displayState.renderScaleTarget.toJSON(); x[json_keys.CROSS_SECTION_RENDER_SCALE_JSON_KEY] = this.sliceViewRenderScaleTarget.toJSON(); + x[json_keys.SEGMENT_COLOR_SHADER_JSON_KEY] = + this.displayState.fragmentSegmentColor.toJSON(); const { linkedSegmentationGroup, linkedSegmentationColorGroup } = this.displayState; @@ -1091,6 +1345,8 @@ export class SegmentationUserLayer extends Base { this.displayState.segmentationColorGroupState.value.toJSON(), ); } + x[json_keys.SHADER_CONTROLS_JSON_KEY] = + this.displayState.segmentColorShaderControlState.toJSON(); return x; } @@ -1366,8 +1622,9 @@ export class SegmentationUserLayer extends Base { } const visibleSegments = [...visibleSegmentsSet]; - const colors = visibleSegments.map((id) => { - const color = getCssColor(getBaseObjectColor(displayState, id)); + const baseColors = getBaseObjectColors(displayState, visibleSegments); + const colors = visibleSegments.map((id, index) => { + const color = getCssColor(baseColors.subarray(4 * index, 4 * index + 4)); return { color, id }; }); @@ -1403,6 +1660,10 @@ registerLayerTypeDetector((subsource) => { return undefined; }); +registerLayerShaderControlsTool(SegmentationUserLayer, (layer) => ({ + shaderControlState: layer.displayState.segmentColorShaderControlState, +})); + registerLayerShaderControlsTool( SegmentationUserLayer, (layer) => ({ diff --git a/src/layer/segmentation/json_keys.ts b/src/layer/segmentation/json_keys.ts index 47d603dab4..faf5c6397d 100644 --- a/src/layer/segmentation/json_keys.ts +++ b/src/layer/segmentation/json_keys.ts @@ -14,6 +14,7 @@ export const COLOR_SEED_JSON_KEY = "colorSeed"; export const SEGMENT_STATED_COLORS_JSON_KEY = "segmentColors"; export const MESH_RENDER_SCALE_JSON_KEY = "meshRenderScale"; export const CROSS_SECTION_RENDER_SCALE_JSON_KEY = "crossSectionRenderScale"; +export const SEGMENT_COLOR_SHADER_JSON_KEY = "segmentColorShader"; export const SKELETON_RENDERING_JSON_KEY = "skeletonRendering"; export const SKELETON_SHADER_JSON_KEY = "skeletonShader"; export const SKELETON_CODE_VISIBLE_KEY = "codeVisible"; @@ -26,3 +27,4 @@ export const SEGMENT_DEFAULT_COLOR_JSON_KEY = "segmentDefaultColor"; export const ANCHOR_SEGMENT_JSON_KEY = "anchorSegment"; export const SKELETON_RENDERING_SHADER_CONTROL_TOOL_ID = "skeletonShaderControl"; +export const SHADER_CONTROLS_JSON_KEY = "shaderControls"; diff --git a/src/layer/segmentation/rendering.md b/src/layer/segmentation/rendering.md new file mode 100644 index 0000000000..ced46e48a7 --- /dev/null +++ b/src/layer/segmentation/rendering.md @@ -0,0 +1,141 @@ +# Segmentation rendering + +Segmentation rendering can be customized using GLSL shader code. The shader is +applied after Neuroglancer has determined the base segment color, including +explicit per-segment colors and the default segment color. + +## Shader language + +The shader code must conform to the OpenGL ES Shading Language (GLSL) version 3.0, specified at . + +You may find the WebGL reference card helpful: . + +## UI Controls + +[UI control directives](../sliceview/image_layer_rendering.md#ui-controls) are +supported. Segmentation shaders also support segment property controls: + +```glsl +#uicontrol property selectedTag(type="tag") +#uicontrol property selectedNumber(type="number") +#uicontrol property selectedString(type="string") +``` + +The `type` parameter filters the property picker. Valid values are `tag`, +`number`, `numerical`, and `string`. In shader code, a tag property control is a +`bool`; numerical property controls have the GLSL type corresponding to the +selected property data type; string property controls are string values and may +be compared with string literals. + +Numerical segment properties can also be used with `invlerp` controls: + +```glsl +#uicontrol float intensity invlerp(property="size", range=[0, 100]) +``` + +If `property` is omitted, the control defaults to the first available numerical +segment property and the selected property may be changed from the UI. + +## API + +### Format of user shader + +The basic user shader is in the form of: + +```glsl +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return color; +} +``` + +- `color` is the color that would be displayed if you chose not to override it. + It includes the segment color hash, the default segment color, or a stated + per-segment color. + +- `hasProperties` indicates that segment property data was found for the target + segment. Check this before relying on property values if your shader needs a + fallback for segments without property rows. + +- `isStated` indicates that `color` came from an explicit stated segment color. + +A `vec4` version of `segmentColor` can also be used to override opacity. A +negative alpha means that the shader does not override the opacity determined by +the segmentation layer. + +```glsl +vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + return color; +} +``` + +### Accessing segmentation properties + +To retrieve a numerical or string property named `myProperty`, use the syntax +`prop("myProperty")`. + +To check if a tag named `myTag` is enabled for a segment, use the syntax + +```glsl +if (tag("myTag")) { + // do something +} +``` + +The helper functions require a segment property map. If the named tag or +property does not exist, shader compilation reports an error. + +### Examples + +```glsl +#uicontrol float property1 invlerp(property="myNumericalProperty", window=[1, 10]) +vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec4(0.5, 0.5, 0.5, 1.0); + } + if (isStated) { + return color; + } + vec4 newColor = vec4(0.0, 0.0, 0.0, 1.0); + newColor.rgb = colormapJet(property1()); + + if (tag("lot-of-axon")) { + return vec4(1.0, 1.0, 1.0, 1.0); + } + + newColor.a = 1.0; + + return newColor; +} +``` + +```glsl +#uicontrol property redTag(type="tag") +#uicontrol property size(type="number") +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.5); + } + if (redTag) { + return vec3(1.0, 0.0, 0.0); + } + if (size > 50u) { + return vec3(1.0, 1.0, 0.0); + } + return color.rgb; +} +``` + +```glsl +vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec4(0.5, 0.5, 0.5, 1.0); + } + if (isStated) { + return color; + } + if (prop("NAis") > 10u) { + return vec4(1.0, 1.0, 1.0, 1.0); + } + return color; +} +``` diff --git a/src/layer/segmentation/segment_color_shader.browser_test.ts b/src/layer/segmentation/segment_color_shader.browser_test.ts new file mode 100644 index 0000000000..53cee08cc0 --- /dev/null +++ b/src/layer/segmentation/segment_color_shader.browser_test.ts @@ -0,0 +1,798 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect } from "vitest"; +import { DisplayContext } from "#src/display_context.js"; +import { makeLayer } from "#src/layer/index.js"; +import type { SegmentationUserLayer } from "#src/layer/segmentation/index.js"; +import "#layer/segmentation"; +import { + type InlineSegmentPropertyMap, + PreprocessedSegmentPropertyMap, + SegmentPropertyMap, +} from "#src/segmentation_display_state/property_map.js"; +import { packColor } from "#src/util/color.js"; +import { DataType } from "#src/util/data_type.js"; +import { vec3, vec4 } from "#src/util/geom.js"; +import { Viewer } from "#src/viewer.js"; +import { ShaderCompilationError } from "#src/webgl/shader.js"; +import type { SegmentPropertyReference } from "#src/webgl/shader_ui_controls.js"; + +const setupSegmentationLayer = () => { + const target = document.createElement("div"); + const display = new DisplayContext(target); + const viewer = new Viewer(display); + return makeLayer(viewer.layerSpecification, "test", { type: "segmentation" }) + .layer! as SegmentationUserLayer; +}; + +const setSegmentPropertyMap = ( + segmentationUserLayer: SegmentationUserLayer, + inlineProperties: InlineSegmentPropertyMap, +) => { + segmentationUserLayer.displayState.segmentPropertyMap.value = + new PreprocessedSegmentPropertyMap( + new SegmentPropertyMap({ inlineProperties }), + ); +}; + +const setSegmentPropertyControl = ( + segmentationUserLayer: SegmentationUserLayer, + controlName: string, + value: SegmentPropertyReference, +) => { + const controlState = + segmentationUserLayer.displayState.segmentColorShaderControlState.state.get( + controlName, + ); + expect(controlState).toBeDefined(); + controlState!.trackable.value = value; +}; + +const compareWithCPUHash = ( + segmentationUserLayer: SegmentationUserLayer, + objectId: bigint, +) => { + const outColor = + segmentationUserLayer.displayState.getShaderBaseSegmentColor(objectId); + const colorGroupState = + segmentationUserLayer.displayState.segmentationColorGroupState.value; + const outColorCPU = vec4.create(); + colorGroupState.segmentColorHash.compute(outColorCPU, objectId); + expect(outColor).toBeDefined(); + expect(outColor!.length).toBe(4); + for (let i = 0; i < 4; ++i) { + expect(outColor![i]).toBeCloseTo(outColorCPU[i]); + } +}; + +expect.extend({ + toBeCloseToFoo(received: number[] | Float32Array, expected: number[]) { + if (received.length !== expected.length) { + return { + pass: false, + message: () => + `Expected array length ${expected.length} but received ${received.length}`, + }; + } + for (let i = 0; i < received.length; ++i) { + if (Math.abs(received[i] - expected[i]) > 1e-6) { + return { + pass: false, + message: () => + `Expected element ${i} to be close to ${expected[i]} but received ${received[i]}`, + }; + } + } + return { + pass: true, + message: () => "Arrays are close", + }; + }, +}); + +const expectColor = ( + color: vec4, + expected: [number, number, number, number], +) => { + expect(color).toBeDefined(); + expect(color!.length).toBe(4); + expect([...color]).toEqual(expected.map((x) => expect.closeTo(x))); +}; + +describe("getShaderBaseSegmentColor", () => { + it("default shader, return hash", () => { + const segmentationUserLayer = setupSegmentationLayer(); + const objectId = 1n; + compareWithCPUHash(segmentationUserLayer, objectId); + }); + it("default shader, random segment id", () => { + const segmentationUserLayer = setupSegmentationLayer(); + const objectId = BigInt(Math.floor(Math.random() * 100000)); + compareWithCPUHash(segmentationUserLayer, objectId); + }); + it("red shader", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return vec3(1.0, 0.0, 0.0); + }`; + const outColor = + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n); + expectColor(outColor!, [1.0, 0.0, 0.0, 0.0]); + }); + + it.each([ + "highp vec3 segmentColor", + "vec3 segmentColor ", + "vec3\nsegmentColor\n", + ])("supports %s declarations", (declaration) => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +${declaration}(vec3 color, bool hasProperties, bool isStated) { + return vec3(1.0, 0.0, 0.0); +}`; + + const outColor = + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n); + + expectColor(outColor!, [1.0, 0.0, 0.0, 0.0]); + }); + + it("alpha shader", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` + vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + return vec4(0.0, 0.0, 0.0, 0.5); + }`; + const outColor = + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n); + expectColor(outColor!, [0.0, 0.0, 0.0, 0.5]); + }); + + it("preserves alpha from mapped segment colors", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.segmentStatedColors.value.set( + 1n, + BigInt(packColor(vec4.fromValues(0.25, 0.5, 0.75, 0.5))), + ); + + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [0.25, 0.5, 0.75, 128 / 255], + ); + }); + + it("treats rgb mapped segment colors as having undefined alpha", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.segmentStatedColors.value.set( + 1n, + BigInt(packColor(vec3.fromValues(1.0, 0.0, 0.0))), + ); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` + vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + return vec4(color.rgb, color.a < 0.0 ? 0.75 : 0.25); + }`; + + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.75], + ); + }); + + it("gets multiple colors with a single offscreen lookup", () => { + const segmentationUserLayer = setupSegmentationLayer(); + const { displayState } = segmentationUserLayer; + displayState.segmentStatedColors.value.set( + 1n, + BigInt(packColor(vec3.fromValues(1.0, 0.0, 0.0))), + ); + displayState.segmentStatedColors.value.set( + 2n, + BigInt(packColor(vec3.fromValues(0.0, 1.0, 0.0))), + ); + displayState.segmentDefaultColor.value = vec3.fromValues(0.0, 0.0, 1.0); + + const colors = displayState.getShaderBaseSegmentColors([1n, 2n, 3n])!; + expectColor(colors.subarray(0, 4) as vec4, [1.0, 0.0, 0.0, 0.0]); + expectColor(colors.subarray(4, 8) as vec4, [0.0, 1.0, 0.0, 0.0]); + expectColor(colors.subarray(8, 12) as vec4, [0.0, 0.0, 1.0, 0.0]); + }); + + it("does not apply hover highlighting to offscreen color lookups", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.segmentStatedColors.value.set( + 1n, + BigInt(packColor(vec3.fromValues(1.0, 0.0, 0.0))), + ); + segmentationUserLayer.displayState.segmentSelectionState.set(1n); + + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + }); + + it("uses the fallback shader if segment properties have not been loaded", () => { + const segmentationUserLayer = setupSegmentationLayer(); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property redTag(type="tag") + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (redTag) { + return vec3(1.0, 0.0, 0.0); + } + return vec3(0.0, 0.0, 0.0); + }`; + + expect( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n), + ).toBeDefined(); + }); + + it("colors by string properties", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "color", + type: "string", + values: ["red", "green"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property colorProperty(type="string") + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.0, 0.0, 1.0); + } + if (colorProperty == "red") { + return vec3(1.0, 0.0, 0.0); + } + if (colorProperty == "green") { + return vec3(0.0, 1.0, 0.0); + } + return vec3(0.5, 0.5, 0.5); + }`; + setSegmentPropertyControl(segmentationUserLayer, "colorProperty", { + type: "string", + id: "color", + }); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, + [0.0, 1.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(3n)!, + [0.0, 0.0, 1.0, 0.0], + ); + }); + + it("colors by string literals with ids greater than 255", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "color", + type: "string", + values: ["target"], + }, + ], + }); + const precedingLiterals = Array.from( + { length: 256 }, + (_, index) => `"unused${index}"`, + ).join(" "); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property colorProperty(type="string") + /* ${precedingLiterals} */ + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (colorProperty == "target") { + return vec3(1.0, 0.0, 0.0); + } + return vec3(0.0, 0.0, 1.0); + }`; + setSegmentPropertyControl(segmentationUserLayer, "colorProperty", { + type: "string", + id: "color", + }); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + }); + + it("defaults invalid string property control state", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "color", + type: "string", + values: ["red"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property colorProperty(type="string") + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (colorProperty == "red") { + return vec3(0.0, 1.0, 0.0); + } + return vec3(0.5, 0.5, 0.5); + }`; + const controlState = + segmentationUserLayer.displayState.segmentColorShaderControlState.state.get( + "colorProperty", + ); + expect(controlState).toBeDefined(); + controlState!.trackable.restoreState({ type: "string", id: "foo" }); + expect(controlState!.trackable.value).toEqual({ + type: "string", + id: "color", + }); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [0.0, 1.0, 0.0, 0.0], + ); + }); + + it("handles unmatched string property value", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "color", + type: "string", + values: ["foo"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property colorProperty(type="string") + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (colorProperty != "red") { + return vec3(0.0, 1.0, 0.0); + } + return vec3(0.5, 0.5, 0.5); + }`; + setSegmentPropertyControl(segmentationUserLayer, "colorProperty", { + type: "string", + id: "color", + }); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [0.0, 1.0, 0.0, 0.0], + ); + }); + + it("colors by tag", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "tag1", + type: "tags", + tags: ["red", "blue"], + tagDescriptions: ["red", "blue"], + values: ["\u0000", "\u0001"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property redTag(type="tag") +#uicontrol property blueTag(type="tag") +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.0, 1.0, 0.0); + } + if (redTag) { + return vec3(1.0, 0.0, 0.0); + } + if (blueTag) { + return vec3(0.0, 0.0, 1.0); + } + return vec3(0.3, 0.6, 0.9); +}`; + setSegmentPropertyControl(segmentationUserLayer, "redTag", { + type: "tag", + id: "red", + }); + setSegmentPropertyControl(segmentationUserLayer, "blueTag", { + type: "tag", + id: "blue", + }); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, + [0.0, 0.0, 1.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(3n)!, + [0.0, 1.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(0n)!, + [0.0, 1.0, 0.0, 0.0], + ); + }); + + it("colors by tag helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "tag1", + type: "tags", + tags: ["red", "blue"], + tagDescriptions: ["red", "blue"], + values: ["\u0000", "\u0001"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.0, 1.0, 0.0); + } + if (tag("red")) { + return vec3(1.0, 0.0, 0.0); + } + if (tag("blue")) { + return vec3(0.0, 0.0, 1.0); + } + return vec3(0.3, 0.6, 0.9); +}`; + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, + [0.0, 0.0, 1.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(3n)!, + [0.0, 1.0, 0.0, 0.0], + ); + }); + + it("reports shader error for missing tag helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "tag1", + type: "tags", + tags: ["red"], + tagDescriptions: ["red"], + values: ["\u0000"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (tag("blue")) { + return vec3(0.0, 0.0, 1.0); + } + float otherError = missingValue; + return vec3(0.0, 0.0, 0.0); +}`; + expect( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n), + ).toBeDefined(); + const shaderError = segmentationUserLayer.displayState.shaderError.value; + expect(shaderError).toBeInstanceOf(ShaderCompilationError); + expect(shaderError!.message).toContain( + `'tag("blue")' : tag does not exist`, + ); + expect(shaderError!.message).not.toContain( + `'' : 'tag("blue")' : tag does not exist`, + ); + expect(shaderError!.message).not.toContain( + "no matching overloaded function found", + ); + expect(shaderError!.message).toContain("missingValue"); + expect( + (shaderError as ShaderCompilationError).errorMessages.find((x) => + x.message.includes(`'tag("blue")' : tag does not exist`), + )?.line, + ).toBe(2); + expect( + (shaderError as ShaderCompilationError).errorMessages.find((x) => + x.message.includes("missingValue"), + )?.line, + ).toBe(5); + }); + + it("colors by numerical property uint8", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.UINT8, + values: new Uint8Array([0, 50]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property prop1Property(type="number") +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (prop1Property == 50u) { + return vec3(1.0, 0.0, 0.0); + } + return vec3(0.0, 0.0, 0.0); +}`; + setSegmentPropertyControl(segmentationUserLayer, "prop1Property", { + type: "numerical", + id: "prop1", + }); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [0.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, + [1.0, 0.0, 0.0, 0.0], + ); + }); + + it("preserves integer and zero numerical property values", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "score", + type: "number", + dataType: DataType.UINT32, + description: undefined, + values: Uint32Array.of(4_000_000_001, 0), + bounds: [0, 4_000_000_001], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property scoreProperty(type="number") +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) return vec3(0.0, 0.0, 1.0); + if (scoreProperty == 4000000001u) return vec3(1.0, 0.0, 0.0); + if (scoreProperty == 0u) return vec3(0.0, 1.0, 0.0); + return vec3(0.0, 0.0, 0.0); +}`; + setSegmentPropertyControl(segmentationUserLayer, "scoreProperty", { + type: "numerical", + id: "score", + }); + + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, + [0.0, 1.0, 0.0, 0.0], + ); + }); + + it("colors by numerical property helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.UINT8, + values: new Uint8Array([0, 50]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (prop("prop1") == 50u) { + return vec3(1.0, 0.0, 0.0); + } + return vec3(0.0, 0.0, 0.0); +}`; + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [0.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, + [1.0, 0.0, 0.0, 0.0], + ); + }); + + it("reports shader error for missing property helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.UINT8, + values: new Uint8Array([0]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (prop("prop2") == 50u) { + return vec3(1.0, 0.0, 0.0); + } + float otherError = missingValue; + return vec3(0.0, 0.0, 0.0); +}`; + expect( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n), + ).toBeDefined(); + const shaderError = segmentationUserLayer.displayState.shaderError.value; + expect(shaderError).toBeInstanceOf(ShaderCompilationError); + expect(shaderError!.message).toContain( + `'prop("prop2")' : property does not exist`, + ); + expect(shaderError!.message).not.toContain( + `'' : 'prop("prop2")' : property does not exist`, + ); + expect(shaderError!.message).not.toContain( + "no matching overloaded function found", + ); + expect(shaderError!.message).toContain("missingValue"); + expect( + (shaderError as ShaderCompilationError).errorMessages.find((x) => + x.message.includes(`'prop("prop2")' : property does not exist`), + )?.line, + ).toBe(2); + expect( + (shaderError as ShaderCompilationError).errorMessages.find((x) => + x.message.includes("missingValue"), + )?.line, + ).toBe(5); + }); + + it("colors by string property helper", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "color", + type: "string", + values: ["red", "green"], + }, + ], + }); + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` + vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + if (!hasProperties) { + return vec3(0.0, 0.0, 1.0); + } + if (prop("color") == "red") { + return vec3(1.0, 0.0, 0.0); + } + if (prop("color") == "green") { + return vec3(0.0, 1.0, 0.0); + } + return vec3(0.5, 0.5, 0.5); + }`; + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [1.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, + [0.0, 1.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(3n)!, + [0.0, 0.0, 1.0, 0.0], + ); + }); + + it("colors by numerical property invlerp", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.UINT8, + values: new Uint8Array([0, 50]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol invlerp normalized(property="prop1", range=[0, 100]) +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return vec3(normalized(), 0.0, 0.0); +}`; + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [0.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, + [0.5, 0.0, 0.0, 0.0], + ); + }); + + it("colors by numerical property float", () => { + const segmentationUserLayer = setupSegmentationLayer(); + setSegmentPropertyMap(segmentationUserLayer, { + ids: new BigUint64Array([1n, 2n]), + properties: [ + { + id: "prop1", + type: "number", + dataType: DataType.FLOAT32, + values: new Float32Array([0, 0.75]), + description: "prop1", + bounds: [0, 100], + }, + ], + }); + + segmentationUserLayer.displayState.fragmentSegmentColor.value = ` +#uicontrol property prop1Property(type="number") +vec3 segmentColor(vec3 color, bool hasProperties, bool isStated) { + return vec3(prop1Property, 0.0, 0.0); +}`; + setSegmentPropertyControl(segmentationUserLayer, "prop1Property", { + type: "numerical", + id: "prop1", + }); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(1n)!, + [0.0, 0.0, 0.0, 0.0], + ); + expectColor( + segmentationUserLayer.displayState.getShaderBaseSegmentColor(2n)!, + [0.75, 0.0, 0.0, 0.0], + ); + }); +}); diff --git a/src/layer/segmentation/style.css b/src/layer/segmentation/style.css index 3d379ba8de..4d70b02599 100644 --- a/src/layer/segmentation/style.css +++ b/src/layer/segmentation/style.css @@ -22,14 +22,17 @@ .neuroglancer-segmentation-rendering-tab .neuroglancer-shader-code-widget { height: 6em; + resize: vertical; } -.neuroglancer-segmentation-dropdown-skeleton-shader-header { +.neuroglancer-segmentation-dropdown-shader-header { display: flex; flex-direction: row; align-items: center; } +.neuroglancer-segmentation-layer-segment-color-shader-overlay + .neuroglancer-shader-code-widget, .neuroglancer-segmentation-layer-skeleton-shader-overlay .neuroglancer-shader-code-widget { width: 80vw; diff --git a/src/layer/single_mesh/index.ts b/src/layer/single_mesh/index.ts index cbd65286a4..1279851b7f 100644 --- a/src/layer/single_mesh/index.ts +++ b/src/layer/single_mesh/index.ts @@ -143,7 +143,7 @@ export class SingleMeshUserLayer extends UserLayer { function makeShaderCodeWidget(layer: SingleMeshUserLayer) { return new ShaderCodeWidget({ - fragmentMain: layer.displayState.fragmentMain, + fragment: layer.displayState.fragmentMain, shaderError: layer.displayState.shaderError, shaderControlState: layer.displayState.shaderControlState, }); diff --git a/src/mesh/frontend.ts b/src/mesh/frontend.ts index bd6ded8562..55ebf741eb 100644 --- a/src/mesh/frontend.ts +++ b/src/mesh/frontend.ts @@ -43,6 +43,12 @@ import type { import { PerspectiveViewRenderLayer } from "#src/perspective_view/render_layer.js"; import type { ThreeDimensionalRenderLayerAttachmentState } from "#src/renderlayer.js"; import { update3dRenderLayerAttachment } from "#src/renderlayer.js"; +import type { SegmentationColorUserShaderManagerParameters } from "#src/segment_color.js"; +import { + DEFAULT_USER_MAIN_SEGMENT_COLOR, + encodeSegmentPropertyShaderDefinition, + SegmentColorShaderManager, +} from "#src/segment_color.js"; import { forEachVisibleSegment, getObjectKey, @@ -54,9 +60,12 @@ import { SegmentationLayerSharedObject, } from "#src/segmentation_display_state/frontend.js"; import type { WatchableValueInterface } from "#src/trackable_value.js"; -import { makeCachedDerivedWatchableValue } from "#src/trackable_value.js"; +import { + AggregateWatchableValue, + makeCachedDerivedWatchableValue, + WatchableValue, +} from "#src/trackable_value.js"; import type { Borrowed, RefCounted } from "#src/util/disposable.js"; -import type { vec4 } from "#src/util/geom.js"; import { getFrustumPlanes, mat3, @@ -68,8 +77,14 @@ import { import * as matrix from "#src/util/matrix.js"; import { GLBuffer } from "#src/webgl/buffer.js"; import type { GL } from "#src/webgl/context.js"; +import type { WatchableShaderError } from "#src/webgl/dynamic_shader.js"; import { parameterizedEmitterDependentShaderGetter } from "#src/webgl/dynamic_shader.js"; import type { ShaderBuilder, ShaderProgram } from "#src/webgl/shader.js"; +import { glsl_uint64 } from "#src/webgl/shader_lib.js"; +import { + getFallbackBuilderState, + parseShaderUiControls, +} from "#src/webgl/shader_ui_controls.js"; import type { RPC } from "#src/worker_rpc.js"; import { registerSharedObjectOwner } from "#src/worker_rpc.js"; @@ -179,7 +194,7 @@ function getFloatPositionHandler( defineShader: (builder: ShaderBuilder) => { builder.addAttribute("highp vec3", "aVertexPosition"); builder.addVertexCode( - "highp vec3 getVertexPosition() { return aVertexPosition; }", + "highp vec3 getVertexPosition() { return aVertexPosition; }\n", ); }, bind( @@ -265,14 +280,18 @@ export class MeshShaderManager { } } - setColor(gl: GL, shader: ShaderProgram, color: vec4) { - gl.uniform4fv(shader.uniform("uColor"), color); - } - setPickID(gl: GL, shader: ShaderProgram, pickID: number) { gl.uniform1ui(shader.uniform("uPickID"), pickID); } + setID(gl: GL, shader: ShaderProgram, id: bigint) { + gl.uniform2ui( + shader.uniform(`uID`), + Number(id & 0xffffffffn), + Number(id >> 32n), + ); + } + beginModel( gl: GL, shader: ShaderProgram, @@ -343,30 +362,81 @@ export class MeshShaderManager { this.drawFragmentHelper(gl, shader, fragmentChunk, indexBegin, indexEnd); } - endLayer(gl: GL, shader: ShaderProgram) { + endLayer( + gl: GL, + shader: ShaderProgram, + displayState: MeshDisplayState, + segmentColorParameters: SegmentationColorUserShaderManagerParameters, + ) { + displayState.segmentationColorUserShader.disable( + gl, + shader, + segmentColorParameters, + ); this.vertexPositionHandler.endLayer(gl, shader); gl.disableVertexAttribArray(shader.attribute("aVertexNormal")); } makeGetter(layer: RefCounted & { gl: GL; displayState: MeshDisplayState }) { - const silhouetteRenderingEnabled = layer.registerDisposer( - makeCachedDerivedWatchableValue( - (x) => x > 0, - [layer.displayState.silhouetteRendering], - ), + const parameters = layer.registerDisposer( + new AggregateWatchableValue((refCounted) => ({ + segmentColorParameters: + layer.displayState.segmentationColorUserShader.shaderParameters, + segmentColorProperties: + layer.displayState.segmentationColorUserShader.usedProperties, + shaderBuilderState: + layer.displayState.segmentColorShaderControlState.builderState, + silhouetteRenderingEnabled: refCounted.registerDisposer( + makeCachedDerivedWatchableValue( + (x) => x > 0, + [layer.displayState.silhouetteRendering], + ), + ), + })), ); + const fallbackParameters = new WatchableValue({ + segmentColorParameters: + layer.displayState.segmentationColorUserShader.shaderParameters.value, + segmentColorProperties: [], + shaderBuilderState: getFallbackBuilderState( + parseShaderUiControls(DEFAULT_USER_MAIN_SEGMENT_COLOR), + ), + silhouetteRenderingEnabled: + layer.displayState.silhouetteRendering.value > 0, + }); + return parameterizedEmitterDependentShaderGetter(layer, layer.gl, { memoizeKey: `mesh/MeshShaderManager/${this.fragmentRelativeVertices}/${this.vertexPositionFormat}`, - parameters: silhouetteRenderingEnabled, - defineShader: (builder, silhouetteRenderingEnabled) => { + parameters, + encodeParameters: (p) => { + return `${p.shaderBuilderState.key}/${JSON.stringify(p.segmentColorParameters)}/${JSON.stringify(p.segmentColorProperties.map(encodeSegmentPropertyShaderDefinition))}/${p.silhouetteRenderingEnabled}`; + }, + fallbackParameters, + shaderError: layer.displayState.meshShaderError, + defineShader: ( + builder, + { + segmentColorParameters, + shaderBuilderState, + silhouetteRenderingEnabled, + }, + ) => { this.vertexPositionHandler.defineShader(builder); + layer.displayState.segmentationColorUserShader.defineShader( + builder, + /*fragment=*/ false, + shaderBuilderState, + segmentColorParameters, + ); builder.addAttribute("highp vec2", "aVertexNormal"); builder.addVarying("highp vec4", "vColor"); builder.addUniform("highp vec4", "uLightDirection"); - builder.addUniform("highp vec4", "uColor"); builder.addUniform("highp mat3", "uNormalMatrix"); builder.addUniform("highp mat4", "uModelViewProjection"); builder.addUniform("highp uint", "uPickID"); + builder.addUniform("highp uvec2", "uID"); + builder.addUniform("highp float", "uAlpha"); + if (silhouetteRenderingEnabled) { builder.addUniform("highp float", "uSilhouettePower"); } @@ -375,6 +445,7 @@ export class MeshShaderManager { builder.addUniform("highp vec3", "uFragmentShape"); } builder.addVertexCode(glsl_decodeNormalOctahedronSnorm8); + builder.addVertexCode(glsl_uint64); let vertexMain = ""; if (this.fragmentRelativeVertices) { vertexMain += ` @@ -393,7 +464,12 @@ vec3 origNormal = decodeNormalOctahedronSnorm8(aVertexNormal); vec3 normal = normalize(uNormalMatrix * (normalMultiplier * origNormal)); float absCosAngle = abs(dot(normal, uLightDirection.xyz)); float lightingFactor = absCosAngle + uLightDirection.w; -vColor = vec4(lightingFactor * uColor.rgb, uColor.a); +vColor = segmentColorUserShader(uint64_t(uID)); +float alpha = uAlpha; +if (vColor.a >= 0.0) { + alpha *= vColor.a; +} +vColor = vec4(lightingFactor * vColor.rgb, alpha); `; if (silhouetteRenderingEnabled) { vertexMain += ` @@ -401,6 +477,8 @@ vColor *= pow(1.0 - absCosAngle, uSilhouettePower); `; } builder.setVertexMain(vertexMain); + const shaderManager = new SegmentColorShaderManager("getColor"); + shaderManager.defineShader(builder); builder.setFragmentMain("emit(vColor, uPickID);"); }, }); @@ -409,6 +487,7 @@ vColor *= pow(1.0 - absCosAngle, uSilhouettePower); export interface MeshDisplayState extends SegmentationDisplayState3D { silhouetteRendering: WatchableValueInterface; + shaderError: WatchableShaderError; } export class MeshLayer extends PerspectiveViewRenderLayer { @@ -490,11 +569,17 @@ export class MeshLayer extends PerspectiveViewRenderLayer { + (objectId, pickIndex) => { const key = getObjectKey(objectId); const manifestChunk = manifestChunks.get(key); ++totalChunks; if (manifestChunk === undefined) return; ++presentChunks; if (renderContext.emitColor) { - meshShaderManager.setColor(gl, shader, color!); + gl.uniform1f( + shader.uniform("uAlpha"), + displayState.objectAlpha.value, + ); } if (renderContext.emitPickID) { meshShaderManager.setPickID(gl, shader, pickIndex!); } + meshShaderManager.setID(gl, shader, objectId); totalChunks += manifestChunk.fragmentIds.length; for (const fragmentId of manifestChunk.fragmentIds) { @@ -550,7 +638,12 @@ export class MeshLayer extends PerspectiveViewRenderLayer { + (objectId, pickIndex) => { const key = getObjectKey(objectId); const manifestChunk = chunks.get(key); ++totalManifestChunks; @@ -895,11 +993,16 @@ export class MultiscaleMeshLayer extends PerspectiveViewRenderLayer> 24; value.r = float((uintValue & 0x0000ffu)) / 255.0; value.g = float((uintValue & 0x00ff00u) >> 8) / 255.0; value.b = float((uintValue & 0xff0000u) >> 16) / 255.0; - value.a = float((uintValue & 0xff000000u) >> 24) / 255.0; + value.a = alpha == 0u ? -1.0 : float(alpha) / 255.0; return true; } return false; } `; - builder.addFragmentCode(s); + addCode(s); } get getFunctionName() { @@ -179,3 +227,828 @@ bool ${this.getFunctionName}(uint64_t x, out vec4 value) { this.hashMapShaderManager.disable(gl, shader); } } + +interface SegmentPropertyShaderData { + texture: WebGLTexture; + dataType: DataType; + sourceValues: unknown; + stringLiteralIds: ShaderStringLiteralIdMap | undefined; +} + +export interface SegmentPropertyShaderDefinition { + identifier: string; + dataType: DataType; +} + +export function encodeSegmentPropertyShaderDefinition( + definition: SegmentPropertyShaderDefinition, +) { + return `${definition.identifier}:${definition.dataType}`; +} + +interface SegmentPropertyReferencesResult { + segmentProperties: SegmentPropertyReference[]; + allPropertiesFound: boolean; + errors: SegmentPropertyReferenceError[]; +} + +interface SegmentPropertyReferenceError { + line: number; + message: string; +} + +const segmentPropertyHelperCallPattern = + /\b(tag|prop)\s*\(\s*string_t\(\s*(\d+)u\s*\)\s*\)/g; +const vec3SegmentColorFunctionPattern = /\bvec3\s+segmentColor\s*\(/; + +function getLineNumberAtIndex(code: string, index: number) { + return Math.max(0, code.substring(0, index).split("\n").length - 1); +} + +export interface SegmentationColorUserShaderManagerParameters { + hasSegmentDefaultColor: boolean; + hasSegmentStatedColors: boolean; +} + +export interface SegmentationColorUserShaderManagerInputs { + segmentDefaultColor: vec3 | vec4 | undefined; + segmentStatedColors: Uint64Map; + hoverHighlight: boolean; +} + +export class SegmentColorUserShaderManager extends RefCounted { + private static getPropertyShaderIdentifier( + type: SegmentPropertyReference["type"], + index: number, + ) { + return `${type}${index}`; + } + + static getNumericalPropertyShaderIdentifier( + segmentPropertyMap: PreprocessedSegmentPropertyMap, + propertyId: string, + ) { + const index = segmentPropertyMap.numericalProperties.findIndex( + (property) => property.id === propertyId, + ); + return index === -1 + ? undefined + : this.getPropertyShaderIdentifier("numerical", index); + } + + protected segmentColorShaderManager = new SegmentColorShaderManager( + "segmentColorHash", + ); + protected hashMapManager = new HashMapShaderManager("SegmentToPropertyIndex"); + protected segmentStatedColorShaderManager = + new SegmentStatedColorShaderManager("segmentStatedColor"); + + private gpuSegmentStatedColorHashTable: + | GPUHashTable + | undefined; + + public shaderParameters: AggregateWatchableValue; + public usedProperties: WatchableValueInterface< + readonly SegmentPropertyShaderDefinition[] + >; + + private segmentPropertyIndexMap = new HashMapUint64(); + private segmentPropertyIndexMapSource: + | PreprocessedSegmentPropertyMap + | undefined; + private segmentPropertyShaderData = new Map< + string, + SegmentPropertyShaderData + >(); + + constructor( + private displayState: SegmentationDisplayState, + private gl: GL, + ) { + super(); + this.shaderParameters = this.registerDisposer( + new AggregateWatchableValue((refCounted) => ({ + hasSegmentDefaultColor: refCounted.registerDisposer( + makeCachedDerivedWatchableValue( + (segmentDefaultColor) => { + return segmentDefaultColor !== undefined; + }, + [displayState.segmentDefaultColor], + ), + ), + hasSegmentStatedColors: refCounted.registerDisposer( + makeCachedDerivedWatchableValue( + (segmentStatedColors: Uint64Map) => { + return segmentStatedColors.size !== 0; + }, + [displayState.segmentStatedColors], + ), + ), + })), + ); + + this.usedProperties = this.registerDisposer( + makeCachedLazyDerivedWatchableValue( + (builderState, segmentPropertyMap) => { + const { segmentProperties } = this.getSegmentPropertyReferences( + builderState, + segmentPropertyMap, + ); + const { definitions } = this.getSegmentPropertyShaderDefinitions( + segmentProperties, + segmentPropertyMap, + ); + return definitions; + }, + this.displayState.segmentColorShaderControlState.builderState, + this.displayState.segmentationGroupState.value.segmentPropertyMap, + ), + ); + } + + private getSegmentPropertyMap() { + return this.displayState.segmentationGroupState.value.segmentPropertyMap + .value; + } + + private getDefaultInputs(): SegmentationColorUserShaderManagerInputs { + return { + segmentDefaultColor: this.displayState.segmentDefaultColor.value, + segmentStatedColors: this.displayState.segmentStatedColors.value, + hoverHighlight: this.displayState.hoverHighlight.value, + }; + } + + private getSegmentPropertyShaderDefinition( + prop: SegmentPropertyReference, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + ): SegmentPropertyShaderDefinition | undefined { + if (prop.type === "tag") { + const { tags } = segmentPropertyMap; + if (!tags) return undefined; + const index = tags.tags.indexOf(prop.id); + return index === -1 + ? undefined + : { + identifier: + SegmentColorUserShaderManager.getPropertyShaderIdentifier( + "tag", + index, + ), + dataType: DataType.UINT8, + }; + } + if (prop.type === "numerical") { + const index = segmentPropertyMap.numericalProperties.findIndex( + (p) => p.id === prop.id, + ); + if (index === -1) return undefined; + return { + identifier: SegmentColorUserShaderManager.getPropertyShaderIdentifier( + "numerical", + index, + ), + dataType: segmentPropertyMap.numericalProperties[index].dataType, + }; + } + const index = segmentPropertyMap.strings.findIndex((p) => p.id === prop.id); + return index === -1 + ? undefined + : { + identifier: SegmentColorUserShaderManager.getPropertyShaderIdentifier( + "string", + index, + ), + dataType: DataType.UINT8, + }; + } + + private getSegmentPropertyReferenceForHelperCall( + helperName: string, + id: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + ): SegmentPropertyReference | undefined { + if (helperName === "tag") { + return segmentPropertyMap.tags?.tags.includes(id) + ? { type: "tag", id } + : undefined; + } + if (segmentPropertyMap.numericalProperties.some((p) => p.id === id)) { + return { type: "numerical", id }; + } + if (segmentPropertyMap.strings.some((p) => p.id === id)) { + return { type: "string", id }; + } + return undefined; + } + + private getSegmentPropertyReferences( + builderState: ShaderControlsBuilderState, + segmentPropertyMap: PreprocessedSegmentPropertyMap | undefined, + ): SegmentPropertyReferencesResult { + const segmentProperties = + builderState.propertyReferences.source === "segment" + ? [...builderState.propertyReferences.references] + : []; + const errors: SegmentPropertyReferenceError[] = []; + let allPropertiesFound = true; + const stringLiteralValues = new Map( + [...builderState.parseResult.preprocessing.stringLiteralIds].map( + ([value, id]) => [id, value], + ), + ); + const addReference = (reference: SegmentPropertyReference) => { + if ( + !segmentProperties.some( + (x) => x.type === reference.type && x.id === reference.id, + ) + ) { + segmentProperties.push(reference); + } + }; + for (const match of builderState.parseResult.code.matchAll( + segmentPropertyHelperCallPattern, + )) { + const helperName = match[1]; + const line = getLineNumberAtIndex( + builderState.parseResult.code, + match.index ?? 0, + ); + const propertyId = stringLiteralValues.get(Number(match[2])); + if (propertyId === undefined) { + errors.push({ + line, + message: `'${helperName}(unknown)' : property string literal does not exist`, + }); + continue; + } + if (segmentPropertyMap === undefined) { + allPropertiesFound = false; + continue; + } + const reference = this.getSegmentPropertyReferenceForHelperCall( + helperName, + propertyId, + segmentPropertyMap, + ); + if (reference === undefined) { + errors.push({ + line, + message: + helperName === "tag" + ? `'tag(${JSON.stringify(propertyId)})' : tag does not exist` + : `'prop(${JSON.stringify(propertyId)})' : property does not exist`, + }); + continue; + } + addReference(reference); + } + return { segmentProperties, allPropertiesFound, errors }; + } + + private rewriteSegmentPropertyHelperCalls( + code: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + stringLiteralIds: ShaderStringLiteralIdMap, + ) { + const stringLiteralValues = new Map( + [...stringLiteralIds].map(([value, id]) => [id, value]), + ); + return code.replace( + segmentPropertyHelperCallPattern, + (match, helperName: string, literalId: string) => { + const propertyId = stringLiteralValues.get(Number(literalId)); + if (propertyId === undefined) return match; + const reference = this.getSegmentPropertyReferenceForHelperCall( + helperName, + propertyId, + segmentPropertyMap, + ); + if (reference === undefined) return match; + const definition = this.getSegmentPropertyShaderDefinition( + reference, + segmentPropertyMap, + ); + if (definition === undefined) return match; + switch (reference.type) { + case "tag": + return `(${definition.identifier} == 1u)`; + case "string": + return `string_t(${definition.identifier})`; + case "numerical": + return definition.identifier; + } + }, + ); + } + + private getSegmentPropertyShaderDefinitions( + segmentProperties: SegmentPropertyReference[], + segmentPropertyMap: PreprocessedSegmentPropertyMap | undefined, + ) { + const definitions = new Map(); + let allPropertiesFound = true; + if (segmentProperties.length !== 0 && segmentPropertyMap === undefined) { + allPropertiesFound = false; + } + if (segmentPropertyMap !== undefined) { + for (const prop of segmentProperties) { + const definition = this.getSegmentPropertyShaderDefinition( + prop, + segmentPropertyMap, + ); + if (definition === undefined) { + allPropertiesFound = false; + continue; + } + definitions.set(definition.identifier, definition); + } + } + return { definitions: [...definitions.values()], allPropertiesFound }; + } + + private updateSegmentPropertyIndexMap( + segmentPropertyMap: PreprocessedSegmentPropertyMap | undefined, + ) { + if (this.segmentPropertyIndexMapSource === segmentPropertyMap) return; + this.segmentPropertyIndexMap.clear(); + this.segmentPropertyIndexMapSource = segmentPropertyMap; + const inlineProperties = + segmentPropertyMap?.segmentPropertyMap.inlineProperties; + if (inlineProperties === undefined) return; + for (let i = 0; i < inlineProperties.ids.length; i++) { + this.segmentPropertyIndexMap.set(inlineProperties.ids[i], BigInt(i)); + } + } + + private deleteSegmentPropertyTexture(identifier: string) { + const data = this.segmentPropertyShaderData.get(identifier); + if (data === undefined) return; + this.gl.deleteTexture(data.texture); + this.segmentPropertyShaderData.delete(identifier); + } + + private updateShaderData( + identifier: string, + values: TypedNumberArray, + dataType: DataType, + sourceValues: unknown = values, + stringLiteralIds?: ShaderStringLiteralIdMap, + ) { + const existing = this.segmentPropertyShaderData.get(identifier); + if ( + existing !== undefined && + existing.dataType === dataType && + existing.sourceValues === sourceValues && + existing.stringLiteralIds === stringLiteralIds + ) { + return; + } + this.deleteSegmentPropertyTexture(identifier); + this.segmentPropertyShaderData.set(identifier, { + ...createSegmentPropertyTextureData(values, this.gl, dataType), + sourceValues, + stringLiteralIds, + }); + } + + private tagPropertyToShaderData( + id: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + ) { + const { tags } = segmentPropertyMap; + if (!tags) return; + const index = tags.tags.indexOf(id); + if (index === -1) return; + const { values } = tags; + const propertyShaderIdentifier = + SegmentColorUserShaderManager.getPropertyShaderIdentifier("tag", index); + const codeUnit = String.fromCharCode(index); + const valuesForTag = values.map((x) => (x.includes(codeUnit) ? 1 : 0)); + this.updateShaderData( + propertyShaderIdentifier, + new Uint8Array(valuesForTag), + DataType.UINT8, + values, + ); + return propertyShaderIdentifier; + } + + private numericPropertyToShaderData( + id: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + ) { + const { numericalProperties } = segmentPropertyMap; + const index = numericalProperties.findIndex((p) => p.id === id); + if (index === -1) return; + const property = numericalProperties[index]; + const propertyShaderIdentifier = + SegmentColorUserShaderManager.getPropertyShaderIdentifier( + "numerical", + index, + ); + this.updateShaderData( + propertyShaderIdentifier, + property.values, + property.dataType, + ); + return propertyShaderIdentifier; + } + + private stringPropertyToShaderData( + id: string, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + stringLiteralIds: ShaderStringLiteralIdMap, + ) { + const { strings } = segmentPropertyMap; + const index = strings.findIndex((p) => p.id === id); + if (index === -1) return; + const property = strings[index]; + const propertyShaderIdentifier = + SegmentColorUserShaderManager.getPropertyShaderIdentifier( + "string", + index, + ); + this.updateShaderData( + propertyShaderIdentifier, + new Uint32Array(property.values.map((x) => stringLiteralIds.get(x) ?? 0)), + DataType.UINT32, + property.values, + stringLiteralIds, + ); + return propertyShaderIdentifier; + } + + private updateSegmentPropertyTexture( + prop: SegmentPropertyReference, + segmentPropertyMap: PreprocessedSegmentPropertyMap, + stringLiteralIds: ShaderStringLiteralIdMap, + ) { + switch (prop.type) { + case "tag": + return this.tagPropertyToShaderData(prop.id, segmentPropertyMap); + case "numerical": + return this.numericPropertyToShaderData(prop.id, segmentPropertyMap); + case "string": + return this.stringPropertyToShaderData( + prop.id, + segmentPropertyMap, + stringLiteralIds, + ); + } + } + + private updateSegmentPropertyTextures( + builderState: ShaderControlsBuilderState, + ) { + const activeIdentifiers = new Set(); + const { parseResult } = builderState; + const segmentPropertyMap = this.getSegmentPropertyMap(); + const { segmentProperties } = this.getSegmentPropertyReferences( + builderState, + segmentPropertyMap, + ); + this.updateSegmentPropertyIndexMap(segmentPropertyMap); + if (segmentPropertyMap !== undefined) { + const { + preprocessing: { stringLiteralIds }, + } = parseResult; + for (const prop of segmentProperties) { + const identifier = this.updateSegmentPropertyTexture( + prop, + segmentPropertyMap, + stringLiteralIds, + ); + if (typeof identifier === "string") { + activeIdentifiers.add(identifier); + } + } + } + for (const identifier of this.segmentPropertyShaderData.keys()) { + if (!activeIdentifiers.has(identifier)) { + this.deleteSegmentPropertyTexture(identifier); + } + } + return activeIdentifiers; + } + + private getMappedIdColor( + builder: ShaderBuilder, + fragment: boolean, + shaderParameters: SegmentationColorUserShaderManagerParameters, + ) { + const { hasSegmentStatedColors, hasSegmentDefaultColor } = shaderParameters; + let getMappedIdColor = `vec4 getMappedIdColor(uint64_t value, out bool isStated) { +`; + if (hasSegmentStatedColors) { + this.segmentStatedColorShaderManager.defineShader(builder, fragment); + getMappedIdColor += ` + vec4 rgba; + if (${this.segmentStatedColorShaderManager.getFunctionName}(value, rgba)) { + isStated = true; + return rgba; + } +`; + } + if (hasSegmentDefaultColor) { + builder.addUniform("highp vec4", "uSegmentDefaultColor"); + getMappedIdColor += ` return uSegmentDefaultColor; +`; + } else { + this.segmentColorShaderManager.defineShader(builder, fragment); + getMappedIdColor += ` return vec4(segmentColorHash(value), -1.0); +`; + } + getMappedIdColor += ` +} +`; + return getMappedIdColor; + } + + defineShader( + builder: ShaderBuilder, + fragment: boolean, + builderState: ShaderControlsBuilderState = this.displayState + .segmentColorShaderControlState.builderState.value, + shaderParameters: SegmentationColorUserShaderManagerParameters = this + .shaderParameters.value, + ) { + builder.addUniform("highp float", "uSaturation"); + builder.addUniform("bool", "uHasSelectedSegment"); + builder.addUniform("highp uvec2", "uSelectedSegment"); + const addCode = fragment + ? builder.addFragmentCode.bind(builder) + : builder.addVertexCode.bind(builder); + const { hashMapManager } = this; + hashMapManager.defineShader(builder, fragment); + addCode(this.getMappedIdColor(builder, fragment, shaderParameters)); + const { parseResult } = builderState; + const segmentPropertyMap = this.getSegmentPropertyMap(); + const { + segmentProperties, + allPropertiesFound: allReferencedPropertiesFound, + errors, + } = this.getSegmentPropertyReferences(builderState, segmentPropertyMap); + const { definitions, allPropertiesFound } = + this.getSegmentPropertyShaderDefinitions( + segmentProperties, + segmentPropertyMap, + ); + let userCode = + allReferencedPropertiesFound && allPropertiesFound + ? parseResult.code + : ""; + if (errors.length !== 0 && segmentPropertyMap !== undefined) { + userCode = `${errors + .map(({ line, message }) => `#line ${line} 1\n#error ${message}\n`) + .join("")} +bool tag(string_t value) { + return false; +} +highp uint prop(string_t value) { + return 0u; +} +#line 0 1 +${parseResult.code}`; + } + if (userCode && segmentPropertyMap !== undefined) { + userCode = this.rewriteSegmentPropertyHelperCalls( + userCode, + segmentPropertyMap, + parseResult.preprocessing.stringLiteralIds, + ); + } + if (userCode) { + addCode(glsl_COLORMAPS); + if (parseResult.preprocessing.stringLiteralIds.size !== 0) { + addCode(glsl_string); + } + for (const { identifier, dataType } of definitions) { + const accessHelper = new OneDimensionalTextureAccessHelper( + `segmentproperty_${identifier}`, + ); + builder.addTextureSampler( + `${getSamplerPrefixForDataType(dataType)}sampler2D`, + `${identifier}_sampler`, + Symbol.for(identifier), + ); + accessHelper.defineShader(builder); + addCode( + accessHelper.getAccessor( + `${identifier}_read`, + `${identifier}_sampler`, + dataType, + ), + ); + addCode(`highp ${getShaderOutputType(dataType)} ${identifier};\n`); + } + addControlsToBuilder(builderState, builder, fragment); + const loadSegmentPropertiesCode = ` +bool loadSegmentProperties(uint64_t id) { + uint64_t propertyIndex_64; + if (!${hashMapManager.getFunctionName}(id, propertyIndex_64)) { + return false; + } + uint propertyIndex = propertyIndex_64.value[0]; + ${definitions + .map(({ identifier, dataType }) => { + return ` + ${identifier} = ${identifier}_read(propertyIndex)${dataType === DataType.FLOAT32 ? "" : ".value"}; +`; + }) + .join("\n")} + return true; +}`; + addCode(loadSegmentPropertiesCode); + addCode(shaderCodeWithLineDirective(userCode)); + if (vec3SegmentColorFunctionPattern.test(userCode)) { + addCode(` +vec4 segmentColor(vec4 color, bool hasProperties, bool isStated) { + return vec4(segmentColor(color.rgb, hasProperties, isStated), color.a); +}`); + } + } + addCode(` +vec4 segmentColorUserShader(uint64_t segmentId, float adjustment) { + float alpha = -1.0; // negative = undefined + bool isStated = false; + vec4 color = getMappedIdColor(segmentId, isStated); + alpha = color.a; + float saturation = uSaturation; + if (uHasSelectedSegment && uSelectedSegment == segmentId.value) { + if (saturation > adjustment) { + saturation -= adjustment; + } else { + saturation += adjustment; + } + } +${ + userCode + ? ` + bool hasProperties = loadSegmentProperties(segmentId); + color = segmentColor(color, hasProperties, isStated); +` + : "" +} + return vec4(mix(vec3(1.0,1.0,1.0), vec3(color), saturation), color.a); +} +vec4 segmentColorUserShader(uint64_t segmentId) { + return segmentColorUserShader(segmentId, 0.5); +} +`); + } + + enable( + gl: GL, + shader: ShaderProgram, + builderState: ShaderControlsBuilderState, + shaderParameters: SegmentationColorUserShaderManagerParameters = this + .shaderParameters.value, + inputs: SegmentationColorUserShaderManagerInputs = this.getDefaultInputs(), + ) { + const { parseResult } = builderState; + const { displayState } = this; + let selectedSegmentLow = 0; + let selectedSegmentHigh = 0; + const { segmentSelectionState } = this.displayState; + const hasSelectedSegment = + segmentSelectionState.hasSelectedSegment && inputs.hoverHighlight; + gl.uniform1ui( + shader.uniform("uHasSelectedSegment"), + hasSelectedSegment ? 1 : 0, + ); + if (hasSelectedSegment) { + const seg = displayState.baseSegmentHighlighting.value + ? segmentSelectionState.baseSelectedSegment + : segmentSelectionState.selectedSegment; + selectedSegmentLow = Number(seg & 0xffffffffn); + selectedSegmentHigh = Number(seg >> 32n); + // only update when we have a selected segment since we ignore when uHasSelectedSegment is false + gl.uniform2ui( + shader.uniform("uSelectedSegment"), + selectedSegmentLow, + selectedSegmentHigh, + ); + } + gl.uniform1f(shader.uniform("uSaturation"), displayState.saturation.value); + const { hasSegmentDefaultColor, hasSegmentStatedColors } = shaderParameters; + if (hasSegmentDefaultColor) { + const { segmentDefaultColor } = inputs; + if (segmentDefaultColor) { + const [r, g, b, a] = segmentDefaultColor; + gl.uniform4f( + shader.uniform("uSegmentDefaultColor"), + r, + g, + b, + a === undefined ? -1.0 : a, + ); + } + } else { + const { + segmentColorHash: { value: segmentColorHash }, + } = displayState; + this.segmentColorShaderManager.enable(gl, shader, segmentColorHash); + } + setControlsInShader( + gl, + shader, + this.displayState.segmentColorShaderControlState, + parseResult, + ); + const activeSegmentPropertyIdentifiers = + this.updateSegmentPropertyTextures(builderState); + this.hashMapManager.enable( + gl, + shader, + GPUHashTable.get(this.gl, this.segmentPropertyIndexMap), + ); + for (const identifier of activeSegmentPropertyIdentifiers) { + const { texture } = this.segmentPropertyShaderData.get(identifier)!; + const textureUnit = shader.textureUnit(Symbol.for(identifier)); + if (textureUnit !== undefined) { + gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + textureUnit); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, texture); + } + } + if (hasSegmentStatedColors) { + const { segmentStatedColors } = inputs; + let { gpuSegmentStatedColorHashTable } = this; + if ( + gpuSegmentStatedColorHashTable === undefined || + gpuSegmentStatedColorHashTable.hashTable !== + segmentStatedColors.hashTable + ) { + gpuSegmentStatedColorHashTable?.dispose(); + this.gpuSegmentStatedColorHashTable = gpuSegmentStatedColorHashTable = + GPUHashTable.get(gl, segmentStatedColors.hashTable); + } + this.segmentStatedColorShaderManager.enable( + gl, + shader, + gpuSegmentStatedColorHashTable, + ); + } + } + + disable( + gl: GL, + shader: ShaderProgram, + shaderParameters: SegmentationColorUserShaderManagerParameters = this + .shaderParameters.value, + ) { + this.hashMapManager.disable(gl, shader); + const { hasSegmentStatedColors } = shaderParameters; + if (hasSegmentStatedColors) { + this.segmentStatedColorShaderManager.disable(gl, shader); + } + } + + disposed() { + for (const { texture } of this.segmentPropertyShaderData.values()) { + this.gl.deleteTexture(texture); + } + this.segmentPropertyShaderData.clear(); + this.gpuSegmentStatedColorHashTable?.dispose(); + } +} + +function createSegmentPropertyTextureData( + values: TypedNumberArray, + gl: GL, + dataType: DataType, +) { + const texture = gl.createTexture(); + { + const textureFormat = computeTextureFormat( + new TextureFormat(), + dataType, + 1, + ); + gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + gl.tempTextureUnit); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, texture); + setOneDimensionalTextureData(gl, textureFormat, values); + gl.bindTexture(WebGL2RenderingContext.TEXTURE_2D, null); + } + return { + texture, + dataType, + }; +} + +function getShaderOutputType(ioType: DataType): string { + switch (ioType) { + case DataType.UINT8: + case DataType.UINT16: + case DataType.UINT32: + return "uint"; + case DataType.INT8: + case DataType.INT16: + case DataType.INT32: + return "int"; + case DataType.FLOAT32: + return "float"; + case DataType.UINT64: + return "uint64_t"; + } +} diff --git a/src/segmentation_display_state/frontend.ts b/src/segmentation_display_state/frontend.ts index 8ffdebbad5..9a4bbcab50 100644 --- a/src/segmentation_display_state/frontend.ts +++ b/src/segmentation_display_state/frontend.ts @@ -23,7 +23,10 @@ import type { PickIDManager } from "#src/object_picking.js"; import type { WatchableRenderLayerTransform } from "#src/render_coordinate_transform.js"; import type { RenderScaleHistogram } from "#src/render_scale_statistics.js"; import type { RenderLayer } from "#src/renderlayer.js"; -import type { SegmentColorHash } from "#src/segment_color.js"; +import type { + SegmentColorHash, + SegmentColorUserShaderManager, +} from "#src/segment_color.js"; import { getCssColor } from "#src/segment_color.js"; import type { VisibleSegmentsState } from "#src/segmentation_display_state/base.js"; import { @@ -61,6 +64,8 @@ import { kOneVec, vec4 } from "#src/util/geom.js"; import { parseUint64 } from "#src/util/json.js"; import { NullarySignal } from "#src/util/signal.js"; import { withSharedVisibility } from "#src/visibility_priority/frontend.js"; +import type { WatchableShaderError } from "#src/webgl/dynamic_shader.js"; +import type { ShaderControlState } from "#src/webgl/shader_ui_controls.js"; import { makeCopyButton } from "#src/widget/copy_button.js"; import { makeEyeButton } from "#src/widget/eye_button.js"; import { makeFilterButton } from "#src/widget/filter_button.js"; @@ -193,6 +198,7 @@ export interface SegmentationColorGroupState { } export interface SegmentationDisplayState { + layer: SegmentationUserLayer; segmentSelectionState: SegmentSelectionState; saturation: TrackableAlphaValue; hoverHighlight: WatchableValueInterface; @@ -200,10 +206,21 @@ export interface SegmentationDisplayState { baseSegmentHighlighting: WatchableValueInterface; segmentationGroupState: WatchableValueInterface; segmentationColorGroupState: WatchableValueInterface; + segmentColorShaderControlState: ShaderControlState; + segmentationColorUserShader: SegmentColorUserShaderManager; + fragmentSegmentColor: WatchableValueInterface; + volumeShaderError: WatchableShaderError; + meshShaderError: WatchableShaderError; + offscreenShaderError: WatchableShaderError; selectSegment: (id: bigint, pin: boolean | "toggle" | "force-unpin") => void; filterBySegmentLabel: (id: bigint) => void; moveToSegment: (id: bigint) => void; + getShaderBaseSegmentColor: (id: bigint) => Float32Array | undefined; + getShaderBaseSegmentColors: ( + ids: readonly bigint[], + colors?: Float32Array, + ) => Float32Array | undefined; // Indirect properties hideSegmentZero: WatchableValueInterface; @@ -404,6 +421,8 @@ const segmentWidgetTemplateWithUnmapped = (() => { export type SegmentWidgetTemplate = typeof segmentWidgetTemplate; +type SegmentColorMap = ReadonlyMap; + interface SegmentWidgetWithExtraColumnsTemplate extends SegmentWidgetTemplate { numericalPropertyIndices: number[]; } @@ -614,6 +633,7 @@ export class SegmentWidgetFactory