diff --git a/README.md b/README.md index cedced004d..8f5a243a3a 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ The "frontend" UI thread handles user actions and rendering, while the "backend" - [Compressed segmentation format](src/sliceview/compressed_segmentation/README.md) - [Data chunk management](src/chunk_manager/) - [On-GPU hashing](src/gpu_hash/) +- [Volume rendering](src/volume_rendering/) +- [Screen-space ambient occlusion (SSAO) rendering](src/ssao/) # Building diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst index 4230a8380c..a9b31734fc 100644 --- a/docs/user-guide/index.rst +++ b/docs/user-guide/index.rst @@ -12,6 +12,8 @@ Topics to segmentation layers. - :doc:`annotation_shaders` — writing custom GLSL shader code to drive the visual appearance of annotations from their properties. +- :doc:`rendering` — perspective-view rendering modes, including + screen-space ambient occlusion. .. toctree:: :hidden: @@ -19,3 +21,4 @@ Topics navigation annotations annotation_shaders + rendering diff --git a/docs/user-guide/rendering.rst b/docs/user-guide/rendering.rst new file mode 100644 index 0000000000..2f2217058d --- /dev/null +++ b/docs/user-guide/rendering.rst @@ -0,0 +1,39 @@ +Rendering +========= + +Neuroglancer offers several rendering modes for displaying data in the +perspective (3-D) view. + +Volume rendering +---------------- + +TODO + +.. _ssao: + +Screen-space ambient occlusion (SSAO) +------------------------------------- + +SSAO simulates shadows on 3-D mesh surfaces by darkening crevices and +concavities where ambient light would be occluded. It adds depth cues that help +you perceive shapes, and makes the display more appealing. + +.. image:: https://storage.googleapis.com/neuroglancer-doc-videos/0/user-guide/ssao/ssao-off.png + :alt: A mesh rendered without SSAO. + :width: 45% + +.. image:: https://storage.googleapis.com/neuroglancer-doc-videos/0/user-guide/ssao/ssao-on.png + :alt: The same mesh rendered with SSAO, showing shading in crevices. + :width: 45% + +Use the :kbd:`q` key to toggle SSAO on and off. The settings panel has three +controls for SSAO: + +- **Enable SSAO (shadows)**: the :kbd:`q` toggle. +- **SSAO intensity**: a slider; higher values give darker shadow effects. +- **SSAO radius**: a slider; higher values give broader, softer shadows. + +SSAO applies only to mesh surfaces (segmentation layers with object surface +meshes). It does not apply to other opaque geometry like skeletons and +annotations. SSAO is disabled in any perspective view that contains a +volume-rendering layer. diff --git a/python/neuroglancer/__init__.py b/python/neuroglancer/__init__.py index c216904838..e64ac54f90 100644 --- a/python/neuroglancer/__init__.py +++ b/python/neuroglancer/__init__.py @@ -118,5 +118,6 @@ row_layout, # noqa: F401 column_layout, # noqa: F401 LayerGroupViewer, # noqa: F401 + Ssao, # noqa: F401 ViewerState, # noqa: F401 ) diff --git a/python/neuroglancer/viewer_state.py b/python/neuroglancer/viewer_state.py index 5eee031c80..fe321a3cb2 100644 --- a/python/neuroglancer/viewer_state.py +++ b/python/neuroglancer/viewer_state.py @@ -1879,6 +1879,16 @@ class DataSelectionState(SidePanelLocation): layers = wrapped_property("layers", LayerSelectedValues) +@export +class Ssao(JsonObjectWrapper): + __slots__ = () + supports_validation = True + + enabled = wrapped_property("enabled", optional(bool, False)) + intensity = wrapped_property("intensity", optional(float, 1.8)) + radius = wrapped_property("radius", optional(float, 0.05)) + + @export class ViewerState(JsonObjectWrapper): """Complete Neuroglancer viewer state. @@ -1928,6 +1938,7 @@ class ViewerState(JsonObjectWrapper): "showAxisLines", optional(bool, True) ) wire_frame = wireFrame = wrapped_property("wireFrame", optional(bool, False)) + ssao = wrapped_property("ssao", Ssao) enable_adaptive_downsampling = enableAdaptiveDownsampling = wrapped_property( "enableAdaptiveDownsampling", optional(bool, True) ) diff --git a/src/data_panel_layout.ts b/src/data_panel_layout.ts index f64f989f05..2ca78f27f1 100644 --- a/src/data_panel_layout.ts +++ b/src/data_panel_layout.ts @@ -42,6 +42,7 @@ import type { RenderLayerRole } from "#src/renderlayer.js"; import { SliceView } from "#src/sliceview/frontend.js"; import type { SliceViewerState } from "#src/sliceview/panel.js"; import { SliceViewPanel } from "#src/sliceview/panel.js"; +import type { TrackableSSAO } from "#src/ssao/trackable_ssao_params.js"; import { TrackableBoolean } from "#src/trackable_boolean.js"; import type { WatchableSet, @@ -101,6 +102,7 @@ export interface ViewerUIState crossSectionBackgroundColor: TrackableRGB; perspectiveViewBackgroundColor: TrackableRGB; hideCrossSectionBackground3D: TrackableBoolean; + ssao: TrackableSSAO; pickRadius: TrackableValue; } @@ -184,6 +186,7 @@ export function getCommonViewerState(viewer: ViewerUIState) { visibility: viewer.visibility, scaleBarOptions: viewer.scaleBarOptions, hideCrossSectionBackground3D: viewer.hideCrossSectionBackground3D, + ssao: viewer.ssao, pickRadius: viewer.pickRadius, }; } diff --git a/src/layer_group_viewer.ts b/src/layer_group_viewer.ts index fa68aacdd4..0e42f0c154 100644 --- a/src/layer_group_viewer.ts +++ b/src/layer_group_viewer.ts @@ -51,6 +51,7 @@ import { WatchableDisplayDimensionRenderInfo, } from "#src/navigation_state.js"; import type { RenderLayerRole } from "#src/renderlayer.js"; +import type { TrackableSSAO } from "#src/ssao/trackable_ssao_params.js"; import { TrackableBoolean } from "#src/trackable_boolean.js"; import type { TrackableValue, @@ -104,6 +105,7 @@ export interface LayerGroupViewerState { crossSectionBackgroundColor: TrackableRGB; perspectiveViewBackgroundColor: TrackableRGB; hideCrossSectionBackground3D: TrackableBoolean; + ssao: TrackableSSAO; pickRadius: TrackableValue; } @@ -391,6 +393,9 @@ export class LayerGroupViewer extends RefCounted { get scaleBarOptions() { return this.viewerState.scaleBarOptions; } + get ssao() { + return this.viewerState.ssao; + } layerPanel: LayerBar | undefined; layout: DataPanelLayoutContainer; toolBinder: LocalToolBinder; diff --git a/src/layer_groups_layout.ts b/src/layer_groups_layout.ts index e50f4ff388..7c440153f4 100644 --- a/src/layer_groups_layout.ts +++ b/src/layer_groups_layout.ts @@ -421,6 +421,7 @@ function getCommonViewerState(viewer: Viewer) { crossSectionBackgroundColor: viewer.crossSectionBackgroundColor, perspectiveViewBackgroundColor: viewer.perspectiveViewBackgroundColor, hideCrossSectionBackground3D: viewer.hideCrossSectionBackground3D, + ssao: viewer.ssao, pickRadius: viewer.uiConfiguration.pickRadius, }; } diff --git a/src/mesh/frontend.ts b/src/mesh/frontend.ts index bd6ded8562..d27375f981 100644 --- a/src/mesh/frontend.ts +++ b/src/mesh/frontend.ts @@ -36,6 +36,7 @@ import { validateOctree, } from "#src/mesh/multiscale.js"; import type { PerspectivePanel } from "#src/perspective_view/panel.js"; +import { perspectivePanelEmitWithNormals } from "#src/perspective_view/panel.js"; import type { PerspectiveViewReadyRenderContext, PerspectiveViewRenderContext, @@ -294,6 +295,16 @@ export class MeshShaderManager { mat3.invert(tempMat3, tempMat3); mat3.transpose(tempMat3, tempMat3); gl.uniformMatrix3fv(shader.uniform("uNormalMatrix"), false, tempMat3); + if (renderContext.emitter === perspectivePanelEmitWithNormals) { + // Combined model→view normal transform for SSAO. Uses `modelMat` + // directly, bypassing `uNormalMatrix`'s `canonicalVoxelFactors` scaling, + // which gives wrong oblique normals on anisotropic data. + mat4.multiply(tempMat4, projectionParameters.viewMatrix, modelMat); + mat4.invert(tempMat4, tempMat4); + mat3FromMat4(tempMat3, tempMat4); + mat3.transpose(tempMat3, tempMat3); + gl.uniformMatrix3fv(shader.uniform("uViewNormalMatrix"), false, tempMat3); + } } drawFragmentHelper( @@ -358,7 +369,8 @@ export class MeshShaderManager { return parameterizedEmitterDependentShaderGetter(layer, layer.gl, { memoizeKey: `mesh/MeshShaderManager/${this.fragmentRelativeVertices}/${this.vertexPositionFormat}`, parameters: silhouetteRenderingEnabled, - defineShader: (builder, silhouetteRenderingEnabled) => { + defineShader: (builder, silhouetteRenderingEnabled, _, emitter) => { + const emitNormals = emitter === perspectivePanelEmitWithNormals; this.vertexPositionHandler.defineShader(builder); builder.addAttribute("highp vec2", "aVertexNormal"); builder.addVarying("highp vec4", "vColor"); @@ -367,6 +379,12 @@ export class MeshShaderManager { builder.addUniform("highp mat3", "uNormalMatrix"); builder.addUniform("highp mat4", "uModelViewProjection"); builder.addUniform("highp uint", "uPickID"); + if (emitNormals) { + builder.addVarying("highp vec3", "vViewNormal"); + builder.addUniform("highp mat3", "uViewNormalMatrix"); + // Hover-highlighted segments opt out of AO by writing the zero-RGB sentinel to `vViewNormal`; see below. + builder.addUniform("highp float", "uHighlighted"); + } if (silhouetteRenderingEnabled) { builder.addUniform("highp float", "uSilhouettePower"); } @@ -395,13 +413,26 @@ float absCosAngle = abs(dot(normal, uLightDirection.xyz)); float lightingFactor = absCosAngle + uLightDirection.w; vColor = vec4(lightingFactor * uColor.rgb, uColor.a); `; + if (emitNormals) { + // Bypass `normal` (post-uNormalMatrix); `uViewNormalMatrix` + // already encodes the full model→view transform. Multiply by + // (1 - uHighlighted) so hovered segments emit the zero sentinel. + vertexMain += ` +vViewNormal = (1.0 - uHighlighted) * + normalize(uViewNormalMatrix * (normalMultiplier * origNormal)); +`; + } if (silhouetteRenderingEnabled) { vertexMain += ` vColor *= pow(1.0 - absCosAngle, uSilhouettePower); `; } builder.setVertexMain(vertexMain); - builder.setFragmentMain("emit(vColor, uPickID);"); + builder.setFragmentMain( + emitNormals + ? "emit(vColor, uPickID, vViewNormal);" + : "emit(vColor, uPickID);", + ); }, }); } @@ -516,6 +547,15 @@ export class MeshLayer extends PerspectiveViewRenderLayer >; private hasVolumeRendering = false; + private ssaoVolumeWarned = false; get rpc() { return this.sharedObject.rpc!; @@ -324,9 +391,17 @@ export class PerspectivePanel extends RenderedDataPanel { ); private axesLineHelper = this.registerDisposer(AxesLineHelper.get(this.gl)); - protected offscreenFramebuffer = this.registerDisposer( - new FramebufferConfiguration(this.gl, { - colorBuffers: [ + // The NORMAL color attachment is added the first time SSAO is enabled, and + // stays for the lifetime of the panel (no thrashing on toggle). + private hasNormalAttachment = false; + private offscreenFramebuffer_: + | FramebufferConfiguration + | undefined; + + // Lazy: allocates on first access; rebuilt by enableNormalAttachment. + protected get offscreenFramebuffer() { + if (this.offscreenFramebuffer_ === undefined) { + const colorBuffers: TextureBuffer[] = [ new TextureBuffer( this.gl, WebGL2RenderingContext.RGBA8, @@ -345,10 +420,45 @@ export class PerspectivePanel extends RenderedDataPanel { WebGL2RenderingContext.RED, WebGL2RenderingContext.FLOAT, ), - ], - depthBuffer: new DepthStencilRenderbuffer(this.gl), - }), - ); + ]; + if (this.hasNormalAttachment) { + // Packed normals + sentinel alpha live in [0, 1]; 8 bits/channel is + // plenty after the *2-1 decode in the GTAO shader. + colorBuffers.push( + new TextureBuffer( + this.gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + ), + ); + } + this.offscreenFramebuffer_ = this.registerDisposer( + new FramebufferConfiguration(this.gl, { + colorBuffers, + depthBuffer: new DepthStencilRenderbuffer(this.gl), + }), + ); + } + return this.offscreenFramebuffer_; + } + + // Both rebuilt together: transparentConfiguration holds an addRef on + // offscreenFramebuffer's depth buffer, so replacing only one would unshare them. + private enableNormalAttachment() { + if (this.hasNormalAttachment) return; + this.hasNormalAttachment = true; + if (this.offscreenFramebuffer_ !== undefined) { + this.unregisterDisposer(this.offscreenFramebuffer_); + this.offscreenFramebuffer_.dispose(); + this.offscreenFramebuffer_ = undefined; + } + if (this.transparentConfiguration_ !== undefined) { + this.unregisterDisposer(this.transparentConfiguration_); + this.transparentConfiguration_.dispose(); + this.transparentConfiguration_ = undefined; + } + } protected transparentConfiguration_: | FramebufferConfiguration @@ -389,6 +499,8 @@ export class PerspectivePanel extends RenderedDataPanel { OffscreenCopyHelper.get(this.gl, defineMaxProjectionToPickCopyShader, 3), ); + private ssaoManager = this.registerDisposer(new SSAOManager(this.gl)); + private sharedObject: PerspectiveViewState; private scaleBars = this.registerDisposer( @@ -588,6 +700,7 @@ export class PerspectivePanel extends RenderedDataPanel { this.registerDisposer( viewer.wireFrame.changed.add(() => this.scheduleRedraw()), ); + this.registerDisposer(viewer.ssao.changed.add(() => this.scheduleRedraw())); this.registerDisposer( viewer.hideCrossSectionBackground3D.changed.add(() => this.scheduleRedraw(), @@ -897,11 +1010,22 @@ export class PerspectivePanel extends RenderedDataPanel { } const gl = this.gl; + // ssaoActive (set below after the opaque walk) may be false when + // ssaoRequested is true, if volume rendering is present. + const ssaoRequested = this.viewer.ssao.enabled.value; + if (ssaoRequested && !this.hasNormalAttachment) { + this.enableNormalAttachment(); + } + const { offscreenFramebuffer } = this; + const needsNormalMask = !ssaoRequested && this.hasNormalAttachment; const disablePicking = () => { - gl.drawBuffers(this.offscreenFramebuffer.singleAttachmentList); + gl.drawBuffers(offscreenFramebuffer.singleAttachmentList); }; const bindFramebuffer = () => { - this.offscreenFramebuffer.bind(width, height); + offscreenFramebuffer.bind(width, height); + if (needsNormalMask) { + gl.drawBuffers(kOpaqueDrawBuffersNoNormal); + } }; bindFramebuffer(); gl.disable(gl.SCISSOR_TEST); @@ -975,6 +1099,14 @@ export class PerspectivePanel extends RenderedDataPanel { kZeroVec4, ); + if (ssaoRequested) { + gl.clearBufferfv( + WebGL2RenderingContext.COLOR, + OffscreenTextures.NORMAL, + kZeroVec4, + ); + } + gl.enable(gl.DEPTH_TEST); const projectionParameters = this.projectionParameters.value; @@ -997,7 +1129,9 @@ export class PerspectivePanel extends RenderedDataPanel { ambientLighting: ambient, directionalLighting: directional, pickIDs: pickingData.pickIDs, - emitter: perspectivePanelEmit, + emitter: ssaoRequested + ? perspectivePanelEmitWithNormals + : perspectivePanelEmit, emitColor: true, emitPickID: true, alreadyEmittedPickID: false, @@ -1043,8 +1177,48 @@ export class PerspectivePanel extends RenderedDataPanel { } } this.hasVolumeRendering = hasVolumeRendering; + // Mixed mesh + volume scenes are not yet supported; the opaque pass and + // NORMAL writes already happened from the with-normals emitter (minor + // waste, not a correctness issue). Notify once per panel lifetime via + // both the console and a dismissable status banner. + if (ssaoRequested && hasVolumeRendering && !this.ssaoVolumeWarned) { + const message = + "SSAO is not supported with volume rendering and has been disabled for this view."; + console.warn(`Neuroglancer: ${message}`); + StatusMessage.showTemporaryMessage(message, /*closeAfter=*/ 8000); + this.ssaoVolumeWarned = true; + } + const ssaoActive = ssaoRequested && !hasVolumeRendering; this.drawSliceViews(renderContext); + if (ssaoActive) { + const depthTexture = + offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture; + const normalTexture = + offscreenFramebuffer.colorBuffers[OffscreenTextures.NORMAL].texture; + offscreenFramebuffer.unbind(); + + // Clamp to the slider range; URL state could otherwise supply a + // negative or absurd value and produce NaNs or inverted sampling. + // Passed straight through; the shader normalizes by wClip so the + // slider's screen-space effect stays consistent across scales. + const radius = Math.min( + SSAO_RADIUS_RANGE.max, + Math.max(SSAO_RADIUS_RANGE.min, this.viewer.ssao.radius.value), + ); + + this.ssaoManager.render( + width, + height, + depthTexture, + normalTexture, + projectionParameters.projectionMat, + radius, + ); + + bindFramebuffer(); + } + if (hasAnnotation) { // Render annotations with blending enabled. @@ -1073,7 +1247,7 @@ export class PerspectivePanel extends RenderedDataPanel { gl.disable(WebGL2RenderingContext.BLEND); } - if (this.viewer.showAxisLines.value) { + if (!ssaoActive && this.viewer.showAxisLines.value) { this.drawAxisLines(); } @@ -1188,7 +1362,7 @@ export class PerspectivePanel extends RenderedDataPanel { for (const [renderLayer, attachment] of visibleLayers) { if (renderLayer.isVolumeRendering) { renderContext.depthBufferTexture = - this.offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture; + offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture; const isVolumeProjectionLayer = isProjectionLayer( renderLayer as VolumeRenderingRenderLayer, @@ -1316,7 +1490,7 @@ export class PerspectivePanel extends RenderedDataPanel { WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA, WebGL2RenderingContext.SRC_ALPHA, ); - this.offscreenFramebuffer.bindSingle(OffscreenTextures.COLOR); + offscreenFramebuffer.bindSingle(OffscreenTextures.COLOR); this.transparencyCopyHelper.draw( transparentConfiguration.colorBuffers[0].texture, transparentConfiguration.colorBuffers[1].texture, @@ -1398,39 +1572,82 @@ export class PerspectivePanel extends RenderedDataPanel { this.viewer.showScaleBar.value && this.viewer.orthographicProjection.value ) { - // Only modify color buffer. - gl.drawBuffers([gl.COLOR_ATTACHMENT0]); - - gl.disable(WebGL2RenderingContext.DEPTH_TEST); - gl.enable(WebGL2RenderingContext.BLEND); - gl.blendFunc( - WebGL2RenderingContext.SRC_ALPHA, - WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA, - ); - const { scaleBars } = this; - const options = this.viewer.scaleBarOptions.value; - scaleBars.draw( - this.renderViewport, - this.navigationState.displayDimensionRenderInfo.value, - this.navigationState.relativeDisplayScales.value, - this.navigationState.zoomFactor.value / - this.renderViewport.logicalHeight, - options, - ); - gl.disable(WebGL2RenderingContext.BLEND); + // Works when SSAO is on, because `scaleBars.draw` sets its own + // (panel-local) viewport. + this.drawScaleBarOverlay(/*toScreen=*/ false); } - this.offscreenFramebuffer.unbind(); + offscreenFramebuffer.unbind(); // Draw the texture over the whole viewport. this.setGLClippedViewport(); - this.offscreenCopyHelper.draw( - this.offscreenFramebuffer.colorBuffers[OffscreenTextures.COLOR].texture, - ); + this.compositeAndOverlay(ssaoActive, offscreenFramebuffer); return true; } + // Final compositing of the offscreen color buffer to the canvas, plus the + // post-composite axis-line overlay for the SSAO path. The non-SSAO path + // renders axis lines into the offscreen FB before the copy. + private compositeAndOverlay( + ssaoActive: boolean, + offscreenFramebuffer: FramebufferConfiguration, + ) { + const gl = this.gl; + if (!ssaoActive) { + this.offscreenCopyHelper.draw( + offscreenFramebuffer.colorBuffers[OffscreenTextures.COLOR].texture, + ); + return; + } + // Clamp to the slider range; pow(ao, intensity) with a negative or + // absurd exponent would otherwise NaN or wash out the composite. + const intensity = Math.min( + SSAO_INTENSITY_RANGE.max, + Math.max(SSAO_INTENSITY_RANGE.min, this.viewer.ssao.intensity.value), + ); + this.ssaoManager.drawComposite( + offscreenFramebuffer.colorBuffers[OffscreenTextures.COLOR].texture, + offscreenFramebuffer.colorBuffers[OffscreenTextures.NORMAL].texture, + intensity, + ); + if (this.viewer.showAxisLines.value) { + gl.disable(WebGL2RenderingContext.DEPTH_TEST); + this.drawAxisLines(/*toScreen=*/ true); + } + } + + private drawScaleBarOverlay(toScreen: boolean) { + const gl = this.gl; + if (!toScreen) { + // Restrict writes to the color buffer so the scale bar doesn't pollute + // Z / PICK / NORMAL when drawn into the offscreen FB. + gl.drawBuffers([gl.COLOR_ATTACHMENT0]); + } + gl.disable(WebGL2RenderingContext.DEPTH_TEST); + gl.enable(WebGL2RenderingContext.BLEND); + gl.blendFunc( + WebGL2RenderingContext.SRC_ALPHA, + WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA, + ); + const { scaleBars } = this; + const options = this.viewer.scaleBarOptions.value; + scaleBars.draw( + this.renderViewport, + this.navigationState.displayDimensionRenderInfo.value, + this.navigationState.relativeDisplayScales.value, + this.navigationState.zoomFactor.value / this.renderViewport.logicalHeight, + options, + ); + gl.disable(WebGL2RenderingContext.BLEND); + } + protected drawSliceViews(renderContext: PerspectiveViewRenderContext) { - const { sliceViewRenderHelper } = this; + const { sliceViewRenderHelper, gl } = this; + // SliceViewRenderHelper's shader uses perspectivePanelEmit (3 outputs); + // mask ATT3 if the FB has a NORMAL attachment from SSAO so drawBuffers + // matches the shader output count. + if (this.hasNormalAttachment) { + gl.drawBuffers(kOpaqueDrawBuffersNoNormal); + } const { lightDirection, ambientLighting, @@ -1483,7 +1700,7 @@ export class PerspectivePanel extends RenderedDataPanel { } } - protected drawAxisLines() { + protected drawAxisLines(toScreen = false) { const { zoomFactor: { value: zoom }, } = this.viewer.navigationState; @@ -1497,7 +1714,9 @@ export class PerspectivePanel extends RenderedDataPanel { 4; const axisLength = zoom * axisRatio; const { gl } = this; - gl.drawBuffers([gl.COLOR_ATTACHMENT0]); + if (!toScreen) { + gl.drawBuffers([gl.COLOR_ATTACHMENT0]); + } this.axesLineHelper.draw( computeAxisLineMatrix(projectionParameters, axisLength), /*blend=*/ false, diff --git a/src/ssao/README.md b/src/ssao/README.md new file mode 100644 index 0000000000..041c11c3cd --- /dev/null +++ b/src/ssao/README.md @@ -0,0 +1,89 @@ +# Screen-Space Ambient Occlusion (SSAO) + +SSAO simulates shadows on 3D mesh surfaces by darkening crevices and concavities +where ambient light would be occluded. It adds depth cues that help us perceive +shapes, and it makes the display more appealing. SSAO is an efficient +post-processing effect applied to the perspective view after opaque geometry is +drawn. + +## Overview + +This implementation uses the Ground Truth Ambient Occlusion (GTAO) algorithm +presented by Jimenez et al. at SIGGRAPH 2016 +(https://www.activision.com/cdn/research/PracticalRealtimeStrategiesTRfinal.pdf). + +The algorithm does the following for each pixel: + +1. Reconstruct the view-space position from the depth buffer. +2. Read the view-space normal from the normal buffer. +3. March in 4 directions (`NUM_DIRECTIONS`) × 8 steps (`NUM_STEPS`) to find the + maximum horizon angle in each direction. +4. Integrate occlusion from the horizon angles relative to the surface normal. + +Then the raw ambient occlusion is bilaterally blurred (depth-aware, 5-tap +kernel, two passes) and composited with the color buffer using +`color.rgb * pow(ao, intensity)`. + +## Scope + +SSAO is limited to mesh surfaces because only `MeshLayer` and +`MultiscaleMeshLayer` supply a view-space normal via the three-argument +`emit(color, pickId, viewNormal)` form. Annotations are omitted from SSAO because +they are glyphs for which 3D shading makes little sense. Skeletons are omitted +because they are rendered as lines without the surface normals needed for SSAO. +Single meshes are omitted because their source files (OBJ, VTK) treat normals as +optional; a future extension could handle the case where normals are present, sharing +code with the other mesh layers. The omitted types call the two-argument +`emit(color, pickId)` form, which writes the zero sentinel `vec4(0)` to the NORMAL +attachment. The SSAO shader treats a zero-RGB normal as a no-AO sentinel, so those +pixels render at full brightness. Highlighted (hovered) mesh segments use the same +zero-RGB sentinel so they also render without darkening. + +This limitation for annotations is enforced in the compositing stage, where the +shader skips the AO multiplication for any pixel whose NORMAL value is the +zero-RGB sentinel. The two-argument emit writes `vec4(0, 0, 0, 1)` to NORMAL; +alpha must be 1 here for the blend mode `(SRC_ALPHA, ONE_MINUS_SRC_ALPHA)` to +fully overwrite the underlying mesh normal with the sentinel. The sentinel +lands at every annotation pixel (including explicitly translucent pixels and +anti-aliased edges). A consequence is that translucent annotations also do not +receive darkening at the covered pixels, since the sentinel takes precedence +of the underlying mesh's normal. In practice, pixels affected this way should +be limited. A fix could be to apply AO to the COLOR buffer before annotations +render. + +Ordinary translucent volume layers do not interact with SSAO because the SSAO +pass runs after opaque geometry but before OIT/transparency. Max-projection +volume layers do write to the Z buffer, though, and the combination of that +volume rendering and SSAO has not been validated. + +## Integration points + +- **`perspective_view/panel.ts`**: Selects the `perspectivePanelEmitWithNormals` + emitter when SSAO is enabled, inserts the SSAO passes between opaque geometry + and OIT/transparency, modifies the final composite. +- **`mesh/frontend.ts`**: `MeshShaderManager` adds `uViewNormalMatrix` uniform + and `vViewNormal` varying, calls `emit(color, pickId, viewNormal)`. +- **`perspective_view/panel.ts` (`OffscreenTextures`)**: NORMAL = 3 added as + 4th color attachment (RGBA8). +- **Settings panel**: SSAO checkbox and intensity/radius sliders appear in the + global Settings side panel via `ViewerSettingsPanel` in + `ui/viewer_settings.ts`, backed by trackable properties on `Viewer`. + +## Emitter variations + +Layer fragment shaders emitting into the perspective panel call one of two +overloads via the `renderContext` emitter: + +- `emit(color, pickId, viewNormal)` — opaque surfaces that should _receive_ AO. + `viewNormal` is in view space (right-handed, camera looking down -Z), need not + be unit length. Caller is responsible for the world→view normal transform. +- `emit(color, pickId)` — opaque skeletons, single-meshes, annotations, axis + lines, etc. Internally writes the zero-RGB sentinel `(0, 0, 0)` to NORMAL + with alpha = 1. The composite shader skips the AO multiply for those pixels. + +Highlighted (hovered) mesh segments emit a zero-RGB sentinel from +`MeshShaderManager`'s vertex shader, gated by a `uHighlighted` uniform owned by +the mesh layer. Any other layer that wants to exclude fragments from AO can do +the same: emit a zero-RGB view normal. The OIT emitter accepts the three-argument +signature for source compatibility but discards the normal: transparent surfaces +do not contribute to opaque AO. diff --git a/src/ssao/shaders.browser_test.ts b/src/ssao/shaders.browser_test.ts new file mode 100644 index 0000000000..e8fff4938f --- /dev/null +++ b/src/ssao/shaders.browser_test.ts @@ -0,0 +1,417 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { describe, it, expect } from "vitest"; +import { + defineBlurShader, + defineGTAOShader, + defineSSAOCompositeShader, +} from "#src/ssao/shaders.js"; +import { mat4 } from "#src/util/geom.js"; +import type { GL } from "#src/webgl/context.js"; +import { + FramebufferConfiguration, + OffscreenCopyHelper, + TextureBuffer, +} from "#src/webgl/offscreen.js"; +import { webglTest } from "#src/webgl/testing.js"; + +function makeTexture( + gl: GL, + internalFormat: number, + format: number, + type: number, + data: ArrayBufferView, + w = 1, + h = 1, +): WebGLTexture { + const tex = gl.createTexture()!; + gl.bindTexture(gl.TEXTURE_2D, tex); + gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1); + gl.texImage2D(gl.TEXTURE_2D, 0, internalFormat, w, h, 0, format, type, data); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.bindTexture(gl.TEXTURE_2D, null); + return tex; +} + +function makeRgba8Output(gl: GL): FramebufferConfiguration { + return new FramebufferConfiguration(gl, { + colorBuffers: [ + new TextureBuffer( + gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + ), + ], + }); +} + +function readRgba8(gl: GL): Uint8Array { + const out = new Uint8Array(4); + gl.readPixels( + 0, + 0, + 1, + 1, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + out, + ); + return out; +} + +// Runs the GTAO (SSAO) shader on a w × h depth grid with every pixel's normal facing +// the camera, and returns the RGBA bytes at (cx, cy). Used by tests that vary +// only the depth pattern. +function runGTAOAndReadCenter( + gl: GL, + w: number, + h: number, + cx: number, + cy: number, + depths: Float32Array, +): Uint8Array { + const helper = OffscreenCopyHelper.get(gl, defineGTAOShader, 2); + const depthTex = makeTexture( + gl, + WebGL2RenderingContext.R32F, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.FLOAT, + depths, + w, + h, + ); + // Every pixel: packed view-space normal (0, 0, 1), i.e., facing the camera. + const normals = new Uint8Array(w * h * 4); + for (let i = 0; i < w * h; i++) { + normals[i * 4] = 128; + normals[i * 4 + 1] = 128; + normals[i * 4 + 2] = 255; + normals[i * 4 + 3] = 255; + } + const normalTex = makeTexture( + gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + normals, + w, + h, + ); + const fbo = new FramebufferConfiguration(gl, { + colorBuffers: [ + new TextureBuffer( + gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + ), + ], + }); + try { + fbo.bind(w, h); + helper.shader.bind(); + const proj = mat4.create(); + // Make a camera with 90 degree vertical FOV, square aspect, [0.1, 10] depth range. + mat4.perspective(proj, Math.PI / 2, 1.0, 0.1, 10.0); + const invProj = mat4.create(); + mat4.invert(invProj, proj); + gl.uniformMatrix4fv(helper.shader.uniform("uProjection"), false, proj); + gl.uniformMatrix4fv( + helper.shader.uniform("uInvProjection"), + false, + invProj, + ); + // 0.4 > 1/uResolution.y (i.e., 0.125), so the sub-pixel exit is skipped; + // kernel reaches ~3 pixels from center. + gl.uniform1f(helper.shader.uniform("uRadius"), 0.4); + gl.uniform2f(helper.shader.uniform("uResolution"), w, h); + helper.draw(depthTex, normalTex); + const px = new Uint8Array(4); + gl.readPixels( + cx, + cy, + 1, + 1, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + px, + ); + return px; + } finally { + fbo.dispose(); + gl.deleteTexture(depthTex); + gl.deleteTexture(normalTex); + } +} + +describe("SSAO shaders", () => { + it("composite multiplies color by pow(ao, intensity)", () => { + webglTest((gl) => { + const helper = OffscreenCopyHelper.get(gl, defineSSAOCompositeShader, 3); + // Color (~1.0, ~0.502, ~0.251, 1.0) from bytes (255, 128, 64, 255). + const colorTex = makeTexture( + gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + new Uint8Array([255, 128, 64, 255]), + ); + // AO = 0.502 (byte 128). pow(0.502, 2.0) ≈ 0.252. + const aoTex = makeTexture( + gl, + WebGL2RenderingContext.R8, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.UNSIGNED_BYTE, + new Uint8Array([128]), + ); + // Non-sentinel normal (packed +Z, "facing camera"): bytes (128, 128, 255). + const normalTex = makeTexture( + gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + new Uint8Array([128, 128, 255, 255]), + ); + const fbo = makeRgba8Output(gl); + try { + fbo.bind(1, 1); + helper.shader.bind(); + gl.uniform1f(helper.shader.uniform("uIntensity"), 2.0); + helper.draw(colorTex, aoTex, normalTex); + const out = readRgba8(gl); + // Expected rgb * 0.252 ≈ (64, 32, 16); alpha unchanged. + expect(out[0]).toBeGreaterThanOrEqual(63); + expect(out[0]).toBeLessThanOrEqual(65); + expect(out[1]).toBeGreaterThanOrEqual(31); + expect(out[1]).toBeLessThanOrEqual(33); + expect(out[2]).toBeGreaterThanOrEqual(15); + expect(out[2]).toBeLessThanOrEqual(17); + expect(out[3]).toBe(255); + } finally { + fbo.dispose(); + gl.deleteTexture(colorTex); + gl.deleteTexture(aoTex); + gl.deleteTexture(normalTex); + } + }); + }); + + it("composite skips AO when NORMAL is the zero-RGB sentinel", () => { + webglTest((gl) => { + const helper = OffscreenCopyHelper.get(gl, defineSSAOCompositeShader, 3); + const colorTex = makeTexture( + gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + new Uint8Array([255, 128, 64, 255]), + ); + // AO = 0.502 — would yield color * 0.252 if the multiply ran. + const aoTex = makeTexture( + gl, + WebGL2RenderingContext.R8, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.UNSIGNED_BYTE, + new Uint8Array([128]), + ); + // Zero-RGB sentinel: composite should skip the AO multiply entirely. + const normalTex = makeTexture( + gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + new Uint8Array([0, 0, 0, 255]), + ); + const fbo = makeRgba8Output(gl); + try { + fbo.bind(1, 1); + helper.shader.bind(); + gl.uniform1f(helper.shader.uniform("uIntensity"), 2.0); + helper.draw(colorTex, aoTex, normalTex); + const out = readRgba8(gl); + // Color passes through unchanged. + expect(out[0]).toBe(255); + expect(out[1]).toBe(128); + expect(out[2]).toBe(64); + expect(out[3]).toBe(255); + } finally { + fbo.dispose(); + gl.deleteTexture(colorTex); + gl.deleteTexture(aoTex); + gl.deleteTexture(normalTex); + } + }); + }); + + it("SSAO no-AO sentinel: cleared depth (=0) returns ao=1.0", () => { + webglTest((gl) => { + const helper = OffscreenCopyHelper.get(gl, defineGTAOShader, 2); + // Cleared-background sentinel: shader bails before reading the normal. + const depthTex = makeTexture( + gl, + WebGL2RenderingContext.R32F, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.FLOAT, + new Float32Array([0.0]), + ); + const normalTex = makeTexture( + gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + new Uint8Array([0, 0, 0, 255]), + ); + const fbo = makeRgba8Output(gl); + try { + fbo.bind(1, 1); + helper.shader.bind(); + const id = mat4.create(); + gl.uniformMatrix4fv(helper.shader.uniform("uProjection"), false, id); + gl.uniformMatrix4fv(helper.shader.uniform("uInvProjection"), false, id); + gl.uniform1f(helper.shader.uniform("uRadius"), 0.05); + gl.uniform2f(helper.shader.uniform("uResolution"), 1.0, 1.0); + helper.draw(depthTex, normalTex); + const out = readRgba8(gl); + expect(out[0]).toBe(255); + expect(out[1]).toBe(255); + expect(out[2]).toBe(255); + } finally { + fbo.dispose(); + gl.deleteTexture(depthTex); + gl.deleteTexture(normalTex); + } + }); + }); + + it("SSAO no-AO sentinel: zero-RGB normal returns ao=1.0", () => { + webglTest((gl) => { + const helper = OffscreenCopyHelper.get(gl, defineGTAOShader, 2); + // Non-cleared depth so the shader proceeds past the depth check, then + // bails on dot(rawN, rawN) < SENTINEL_EPS. + const depthTex = makeTexture( + gl, + WebGL2RenderingContext.R32F, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.FLOAT, + new Float32Array([0.5]), + ); + const normalTex = makeTexture( + gl, + WebGL2RenderingContext.RGBA8, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.UNSIGNED_BYTE, + new Uint8Array([0, 0, 0, 255]), + ); + const fbo = makeRgba8Output(gl); + try { + fbo.bind(1, 1); + helper.shader.bind(); + const id = mat4.create(); + gl.uniformMatrix4fv(helper.shader.uniform("uProjection"), false, id); + gl.uniformMatrix4fv(helper.shader.uniform("uInvProjection"), false, id); + gl.uniform1f(helper.shader.uniform("uRadius"), 0.05); + gl.uniform2f(helper.shader.uniform("uResolution"), 1.0, 1.0); + helper.draw(depthTex, normalTex); + const out = readRgba8(gl); + expect(out[0]).toBe(255); + expect(out[1]).toBe(255); + expect(out[2]).toBe(255); + } finally { + fbo.dispose(); + gl.deleteTexture(depthTex); + gl.deleteTexture(normalTex); + } + }); + }); + + it("SSAO produces lowish < ao < highish when neighbors occlude the center", () => { + webglTest((gl) => { + const w = 8; + const h = 8; + const cx = 4; + const cy = 4; + // Center at depthVal 0.4 (further from camera); surrounding pixels at + // 0.7 (closer). The shader does fragZ = 1 - depthVal, so the center + // is "lower" and its raised neighbors occlude it. + const depths = new Float32Array(w * h); + for (let i = 0; i < w * h; i++) { + depths[i] = 0.7; + } + depths[cy * w + cx] = 0.4; + const px = runGTAOAndReadCenter(gl, w, h, cx, cy, depths); + // Loose bounds: tight enough to catch regressions where the algorithm + // pegs to either extreme (no AO or full black), wide enough to absorb + // expected variation in noise, falloff curve, or sample count. + expect(px[0]).toBeGreaterThan(64); + expect(px[0]).toBeLessThan(192); + }); + }); + + it("SSAO produces ao ≈ 1 on a flat surface (no occluders)", () => { + webglTest((gl) => { + const w = 8; + const h = 8; + const cx = 4; + const cy = 4; + // All pixels at the same depth: no occlusion possible. + const depths = new Float32Array(w * h).fill(0.5); + const px = runGTAOAndReadCenter(gl, w, h, cx, cy, depths); + // Every sample has sinH ≤ 0, total occlusion is 0, ao = 1.0. Allow + // tiny slack for floating-point rounding. + expect(px[0]).toBeGreaterThanOrEqual(250); + }); + }); + + it("blur is identity for constant input", () => { + webglTest((gl) => { + const helper = OffscreenCopyHelper.get(gl, defineBlurShader, 2); + // Constant AO = 0.698 (byte 178). + const aoTex = makeTexture( + gl, + WebGL2RenderingContext.R8, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.UNSIGNED_BYTE, + new Uint8Array([178]), + ); + const depthTex = makeTexture( + gl, + WebGL2RenderingContext.R32F, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.FLOAT, + new Float32Array([0.5]), + ); + const fbo = makeRgba8Output(gl); + try { + fbo.bind(1, 1); + helper.shader.bind(); + gl.uniform2f(helper.shader.uniform("uDirection"), 1.0, 0.0); + helper.draw(aoTex, depthTex); + const out = readRgba8(gl); + // All 5 taps hit the same texel at the same depth, so depthDiff=0 + // and every weight is 1; the bilateral average equals the input. + expect(out[0]).toBeGreaterThanOrEqual(177); + expect(out[0]).toBeLessThanOrEqual(179); + } finally { + fbo.dispose(); + gl.deleteTexture(aoTex); + gl.deleteTexture(depthTex); + } + }); + }); +}); diff --git a/src/ssao/shaders.ts b/src/ssao/shaders.ts new file mode 100644 index 0000000000..68e07e8bab --- /dev/null +++ b/src/ssao/shaders.ts @@ -0,0 +1,218 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ShaderBuilder } from "#src/webgl/shader.js"; + +const glsl_gtao = ` +// Number of directions in which to sample horizon angles. +#define NUM_DIRECTIONS 4 +// Number of steps along each direction at which to sample the horizon from the +// depth buffer. +#define NUM_STEPS 8 +#define PI 3.14159265 +// Cap the per-pixel kernel at this fraction of viewport height; avoids +// runaway sampling at extreme zoom-in. Sized for scenes of thin arbors. +#define MAX_KERNEL_FRACTION 0.6 +// Minimum view-space distance to count a horizon sample; avoids division by zero +// at coincident samples. +#define MIN_SAMPLE_DIST 0.0001 +// Squared length below which a packed normal is treated as the no-AO +// sentinel (zero-RGB plus rounding tolerance). Real packed unit normals +// have squared length >= 1/3, so 0.01 is safely below. +#define SENTINEL_EPS 0.01 +// Decorrelate the per-step noise from the per-direction noise by perturbing +// the hash input. +#define STEP_NOISE_SCALE 0.7 +#define STEP_NOISE_BIAS 0.3 + +vec3 viewPosFromDepth(vec2 uv, float fragZ, mat4 invProj) { + vec4 clip = vec4(uv * 2.0 - 1.0, fragZ * 2.0 - 1.0, 1.0); + vec4 view = invProj * clip; + return view.xyz / view.w; +} + +float gtaoHash(vec2 p) { + vec3 p3 = fract(vec3(p.xyx) * 0.1031); + p3 += dot(p3, p3.yzx + 33.33); + return fract((p3.x + p3.y) * p3.z); +} + +vec4 gtao() { + vec2 uv = vTexCoord; + float depthVal = getValue0().r; + if (depthVal == 0.0) { + return vec4(1.0); + } + + float fragZ = 1.0 - depthVal; + vec3 P = viewPosFromDepth(uv, fragZ, uInvProjection); + + // Zero RGB is the no-AO sentinel: cleared background pixels and + // highlighted objects (see emit shader) both land here. + vec3 rawN = getValue1().rgb; + if (dot(rawN, rawN) < SENTINEL_EPS) { + return vec4(1.0); + } + vec3 N = normalize(rawN * 2.0 - 1.0); + + // Here, uRadius acts as a fraction of viewport height, clamped at a + // reasonable maximum, to drive the marching distance. + float kernelRadius = uRadius; + kernelRadius = min(kernelRadius, MAX_KERNEL_FRACTION); + // Here, uRadius scales the view-space distance for per-sample falloff. + float wClip = uProjection[2][3] * P.z + uProjection[3][3]; + float falloffRadius = uRadius * 2.0 * wClip / uProjection[1][1]; + + // Sub-pixel kernel: nothing meaningful to sample. + if (kernelRadius < 1.0 / uResolution.y) { + return vec4(1.0); + } + + float noiseAngle = gtaoHash(gl_FragCoord.xy) * PI; + float stepNoise = gtaoHash(gl_FragCoord.xy * STEP_NOISE_SCALE + STEP_NOISE_BIAS); + + float totalOcclusion = 0.0; + + for (int d = 0; d < NUM_DIRECTIONS; d++) { + float phi = (float(d) + 0.5) / float(NUM_DIRECTIONS) * PI + noiseAngle; + // Correct for non-square viewports so azimuthal samples are uniform in + // world space rather than UV space. + vec2 dir2D = vec2(cos(phi), sin(phi)) * vec2(uResolution.y / uResolution.x, 1.0); + vec2 stepUV = dir2D * kernelRadius / float(NUM_STEPS); + + float maxSinH_pos = 0.0; + float maxSinH_neg = 0.0; + + for (int s = 1; s <= NUM_STEPS; s++) { + float t = float(s) + stepNoise * 0.5; + + vec2 uvP = uv + stepUV * t; + if (uvP.x > 0.0 && uvP.x < 1.0 && uvP.y > 0.0 && uvP.y < 1.0) { + float dv = texture(uSampler[0], uvP).r; + if (dv > 0.0) { + vec3 S = viewPosFromDepth(uvP, 1.0 - dv, uInvProjection); + vec3 delta = S - P; + float dist = length(delta); + if (dist > MIN_SAMPLE_DIST) { + float sinH = dot(delta / dist, N); + float falloff = clamp(1.0 - dist * dist / (falloffRadius * falloffRadius), 0.0, 1.0); + maxSinH_pos = max(maxSinH_pos, sinH * falloff); + } + } + } + + vec2 uvN = uv - stepUV * t; + if (uvN.x > 0.0 && uvN.x < 1.0 && uvN.y > 0.0 && uvN.y < 1.0) { + float dv = texture(uSampler[0], uvN).r; + if (dv > 0.0) { + vec3 S = viewPosFromDepth(uvN, 1.0 - dv, uInvProjection); + vec3 delta = S - P; + float dist = length(delta); + if (dist > MIN_SAMPLE_DIST) { + float sinH = dot(delta / dist, N); + float falloff = clamp(1.0 - dist * dist / (falloffRadius * falloffRadius), 0.0, 1.0); + maxSinH_neg = max(maxSinH_neg, sinH * falloff); + } + } + } + } + + totalOcclusion += (maxSinH_pos + maxSinH_neg) * 0.5; + } + + float ao = 1.0 - totalOcclusion / float(NUM_DIRECTIONS); + ao = clamp(ao, 0.0, 1.0); + return vec4(vec3(ao), 1.0); +} +`; + +export function defineGTAOShader(builder: ShaderBuilder) { + builder.addUniform("highp mat4", "uProjection"); + builder.addUniform("highp mat4", "uInvProjection"); + builder.addUniform("highp float", "uRadius"); + builder.addUniform("highp vec2", "uResolution"); + builder.addOutputBuffer("vec4", "v4f_fragColor", null); + builder.addFragmentCode(glsl_gtao); + builder.setFragmentMain(`v4f_fragColor = gtao();`); +} + +const glsl_blur = ` +// Bilateral falloff sharpness; tuned for normalized [0,1] depth so that +// samples across surface boundaries get rejected. +#define DEPTH_AWARE_FALLOFF 1000.0 + +vec4 blur() { + vec2 texelSize = 1.0 / vec2(textureSize(uSampler[0], 0)); + float centerDepth = getValue1().r; + + float result = 0.0; + float totalWeight = 0.0; + + for (int i = -2; i <= 2; i++) { + vec2 offset = vec2(float(i)) * texelSize * uDirection; + vec2 uv = vTexCoord + offset; + float sampleDepth = texture(uSampler[1], uv).r; + float depthDiff = abs(sampleDepth - centerDepth); + float w = exp(-depthDiff * DEPTH_AWARE_FALLOFF); + result += texture(uSampler[0], uv).r * w; + totalWeight += w; + } + + return vec4(vec3(result / totalWeight), 1.0); +} +`; + +export function defineBlurShader(builder: ShaderBuilder) { + builder.addUniform("highp vec2", "uDirection"); + builder.addOutputBuffer("vec4", "v4f_fragColor", null); + builder.addFragmentCode(glsl_blur); + builder.setFragmentMain(`v4f_fragColor = blur();`); +} + +const glsl_ssaoComposite = ` +// Squared length below which a packed normal is treated as the no-AO +// sentinel (zero-RGB plus rounding tolerance). Real packed unit normals +// have squared length >= 1/3, so 0.01 is safely below. +#define SENTINEL_EPS 0.01 + +// Set to 1 to visualize the (post-intensity) AO buffer directly instead of +// the color * AO composite. Useful for fine-tuning radius, intensity or +// blur falloff without the effect of mesh color or lighting. +#define DEBUG_SSAO 0 + +vec4 composite() { + vec4 color = getValue0(); + float ao = getValue1().r; + // Zero-RGB normal is the no-AO sentinel: cleared background, opaque + // annotations/skeletons (which write vec4(0)), and highlighted mesh + // segments. Skip the AO multiply so they render at the SSAO-off + // appearance. + vec3 normal = getValue2().rgb; + ao = dot(normal, normal) < SENTINEL_EPS ? 1.0 : pow(ao, uIntensity); + #if DEBUG_SSAO + return vec4(vec3(ao), 1.0); + #else + return vec4(color.rgb * ao, color.a); + #endif +} +`; + +export function defineSSAOCompositeShader(builder: ShaderBuilder) { + builder.addUniform("highp float", "uIntensity"); + builder.addOutputBuffer("vec4", "v4f_fragColor", null); + builder.addFragmentCode(glsl_ssaoComposite); + builder.setFragmentMain(`v4f_fragColor = composite();`); +} diff --git a/src/ssao/ssao_manager.ts b/src/ssao/ssao_manager.ts new file mode 100644 index 0000000000..eaade26746 --- /dev/null +++ b/src/ssao/ssao_manager.ts @@ -0,0 +1,149 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + defineGTAOShader, + defineBlurShader, + defineSSAOCompositeShader, +} from "#src/ssao/shaders.js"; +import { RefCounted } from "#src/util/disposable.js"; +import { mat4 } from "#src/util/geom.js"; +import type { GL } from "#src/webgl/context.js"; +import { + FramebufferConfiguration, + OffscreenCopyHelper, + TextureBuffer, +} from "#src/webgl/offscreen.js"; + +export class SSAOManager extends RefCounted { + private ssaoFboA: FramebufferConfiguration | undefined; + private ssaoFboB: FramebufferConfiguration | undefined; + private gtaoCopyHelper: OffscreenCopyHelper | undefined; + private blurCopyHelper: OffscreenCopyHelper | undefined; + private ssaoCompositeHelper: OffscreenCopyHelper | undefined; + private invProjectionMat = mat4.create(); + + constructor(private gl: GL) { + super(); + } + + private ensureResources() { + if (this.ssaoFboA !== undefined) return; + const { gl } = this; + this.ssaoFboA = this.registerDisposer( + new FramebufferConfiguration(gl, { + colorBuffers: [ + new TextureBuffer( + gl, + WebGL2RenderingContext.R8, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.UNSIGNED_BYTE, + ), + ], + }), + ); + this.ssaoFboB = this.registerDisposer( + new FramebufferConfiguration(gl, { + colorBuffers: [ + new TextureBuffer( + gl, + WebGL2RenderingContext.R8, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.UNSIGNED_BYTE, + ), + ], + }), + ); + this.gtaoCopyHelper = this.registerDisposer( + OffscreenCopyHelper.get(gl, defineGTAOShader, 2), + ); + this.blurCopyHelper = this.registerDisposer( + OffscreenCopyHelper.get(gl, defineBlurShader, 2), + ); + this.ssaoCompositeHelper = this.registerDisposer( + OffscreenCopyHelper.get(gl, defineSSAOCompositeShader, 3), + ); + } + + render( + width: number, + height: number, + depthTexture: WebGLTexture | null, + normalTexture: WebGLTexture | null, + projectionMat: mat4, + radius: number, + ) { + this.ensureResources(); + const { gl, invProjectionMat } = this; + // projectionMat is invertible for any non-degenerate viewport; the only + // singular case (near == far) would already have broken opaque rendering. + mat4.invert(invProjectionMat, projectionMat); + + // GTAO pass + this.ssaoFboA!.bind(width, height); + const gtaoShader = this.gtaoCopyHelper!.shader; + gtaoShader.bind(); + gl.uniformMatrix4fv( + gtaoShader.uniform("uProjection"), + false, + projectionMat, + ); + gl.uniformMatrix4fv( + gtaoShader.uniform("uInvProjection"), + false, + invProjectionMat, + ); + gl.uniform1f(gtaoShader.uniform("uRadius"), radius); + gl.uniform2f(gtaoShader.uniform("uResolution"), width, height); + this.gtaoCopyHelper!.draw(depthTexture, normalTexture); + + // Blur horizontal + this.ssaoFboB!.bind(width, height); + const blurShader = this.blurCopyHelper!.shader; + blurShader.bind(); + gl.uniform2f(blurShader.uniform("uDirection"), 1.0, 0.0); + this.blurCopyHelper!.draw( + this.ssaoFboA!.colorBuffers[0].texture, + depthTexture, + ); + + // Blur vertical + this.ssaoFboA!.bind(width, height); + blurShader.bind(); + gl.uniform2f(blurShader.uniform("uDirection"), 0.0, 1.0); + this.blurCopyHelper!.draw( + this.ssaoFboB!.colorBuffers[0].texture, + depthTexture, + ); + } + + drawComposite( + colorTexture: WebGLTexture | null, + normalTexture: WebGLTexture | null, + intensity: number, + ) { + this.ensureResources(); + const { gl } = this; + const shader = this.ssaoCompositeHelper!.shader; + shader.bind(); + gl.uniform1f(shader.uniform("uIntensity"), intensity); + this.ssaoCompositeHelper!.draw( + colorTexture, + this.ssaoFboA!.colorBuffers[0].texture, + normalTexture, + ); + } +} diff --git a/src/ssao/trackable_ssao_params.ts b/src/ssao/trackable_ssao_params.ts new file mode 100644 index 0000000000..fa40d3b22d --- /dev/null +++ b/src/ssao/trackable_ssao_params.ts @@ -0,0 +1,61 @@ +/** + * @license + * Copyright 2026 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TrackableBoolean } from "#src/trackable_boolean.js"; +import { TrackableValue } from "#src/trackable_value.js"; +import { verifyFiniteFloat } from "#src/util/json.js"; +import { CompoundTrackable } from "#src/util/trackable.js"; + +// Slider / clamp range for the SSAO sampling radius. +export const SSAO_RADIUS_RANGE = { min: 0.01, max: 3, step: 0.01 }; +// Slider / clamp range for the AO power exponent at composite time. +export const SSAO_INTENSITY_RANGE = { min: 0.5, max: 5.0, step: 0.1 }; + +// Clamps the parsed value to the given range; out-of-range URL state lands +// in-range rather than falling back to the default. +function clampToRange(range: { min: number; max: number }) { + return (obj: unknown) => + Math.min(range.max, Math.max(range.min, verifyFiniteFloat(obj))); +} + +export class TrackableSSAO extends CompoundTrackable { + enabled = new TrackableBoolean(false, false); + intensity = new TrackableValue( + 1.8, + clampToRange(SSAO_INTENSITY_RANGE), + 1.8, + ); + radius = new TrackableValue( + 0.2, + clampToRange(SSAO_RADIUS_RANGE), + 0.2, + ); + + constructor() { + super(); + this.add("enabled", this.enabled); + this.add("intensity", this.intensity); + this.add("radius", this.radius); + } + + toJSON(): any { + const result = super.toJSON(); + for (const key of Object.keys(result)) { + if (result[key] === undefined) delete result[key]; + } + return Object.keys(result).length === 0 ? undefined : result; + } +} diff --git a/src/ui/default_input_event_bindings.ts b/src/ui/default_input_event_bindings.ts index c041bc0335..070a119780 100644 --- a/src/ui/default_input_event_bindings.ts +++ b/src/ui/default_input_event_bindings.ts @@ -29,6 +29,7 @@ export function getDefaultGlobalBindings() { map.set("keyv", "toggle-default-annotations"); map.set("keya", "toggle-axis-lines"); map.set("keyo", "toggle-orthographic-projection"); + map.set("keyq", "toggle-ssao"); for (let i = 1; i <= 9; ++i) { map.set("digit" + i, "toggle-layer-" + i); diff --git a/src/ui/viewer_settings.ts b/src/ui/viewer_settings.ts index 913cb39749..1f9349d121 100644 --- a/src/ui/viewer_settings.ts +++ b/src/ui/viewer_settings.ts @@ -16,6 +16,10 @@ import "#src/ui/viewer_settings.css"; +import { + SSAO_INTENSITY_RANGE, + SSAO_RADIUS_RANGE, +} from "#src/ssao/trackable_ssao_params.js"; import { TrackableBooleanCheckbox } from "#src/trackable_boolean.js"; import type { TrackableValue, @@ -33,6 +37,7 @@ import { emptyToUndefined } from "#src/util/json.js"; import type { Viewer } from "#src/viewer.js"; import { ColorWidget } from "#src/widget/color.js"; import { NumberInputWidget } from "#src/widget/number_input_widget.js"; +import { RangeWidget } from "#src/widget/range.js"; import { TextInputWidget } from "#src/widget/text_input.js"; const DEFAULT_SETTINGS_PANEL_LOCATION: SidePanelLocation = { @@ -106,9 +111,11 @@ export class ViewerSettingsPanel extends SidePanel { const addCheckbox = ( label: string, value: WatchableValueInterface, + tooltip?: string, ) => { const labelElement = document.createElement("label"); labelElement.textContent = label; + if (tooltip !== undefined) labelElement.title = tooltip; const checkbox = this.registerDisposer( new TrackableBooleanCheckbox(value), ); @@ -144,5 +151,36 @@ export class ViewerSettingsPanel extends SidePanel { addColor("Cross-section background", viewer.crossSectionBackgroundColor); addColor("Projection background", viewer.perspectiveViewBackgroundColor); + + const addRange = ( + label: string, + value: WatchableValueInterface, + options: { min: number; max: number; step: number }, + tooltip?: string, + ) => { + const widget = this.registerDisposer( + new RangeWidget(value, { ...options, label }), + ); + if (tooltip !== undefined) widget.element.title = tooltip; + scroll.appendChild(widget.element); + }; + + addCheckbox( + "Enable SSAO (shadows)", + viewer.ssao.enabled, + "Screen-space ambient occlusion: more realistic shading/shadows on mesh surfaces.", + ); + addRange( + "SSAO intensity", + viewer.ssao.intensity, + SSAO_INTENSITY_RANGE, + "Higher values give darker shading/shadows.", + ); + addRange( + "SSAO radius", + viewer.ssao.radius, + SSAO_RADIUS_RANGE, + "Higher values give broader, softer shadows.", + ); } } diff --git a/src/viewer.ts b/src/viewer.ts index a985d0c2c3..167b0ece27 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -72,6 +72,7 @@ import { import { overlaysOpen } from "#src/overlay.js"; import { ScreenshotHandler } from "#src/python_integration/screenshots.js"; import { allRenderLayerRoles, RenderLayerRole } from "#src/renderlayer.js"; +import { TrackableSSAO } from "#src/ssao/trackable_ssao_params.js"; import { StatusMessage } from "#src/status.js"; import { ElementVisibilityFromTrackableBoolean, @@ -266,6 +267,7 @@ class TrackableViewerState extends CompoundTrackable { this.add("layers", viewer.layerSpecification); this.add("showAxisLines", viewer.showAxisLines); this.add("wireFrame", viewer.wireFrame); + this.add("ssao", viewer.ssao); this.add("enableAdaptiveDownsampling", viewer.enableAdaptiveDownsampling); this.add("showScaleBar", viewer.showScaleBar); this.add("showDefaultAnnotations", viewer.showDefaultAnnotations); @@ -440,6 +442,7 @@ export class Viewer extends RefCounted implements ViewerState { ); showAxisLines = new TrackableBoolean(true, true); wireFrame = new TrackableBoolean(false, false); + ssao = this.registerDisposer(new TrackableSSAO()); enableAdaptiveDownsampling = new TrackableBoolean(true, true); showScaleBar = new TrackableBoolean(true, true); showPerspectiveSliceViews = new TrackableBoolean(true, true); @@ -1163,6 +1166,7 @@ export class Viewer extends RefCounted implements ViewerState { this.showPerspectiveSliceViews.toggle(), ); this.bindAction("toggle-show-statistics", () => this.showStatistics()); + this.bindAction("toggle-ssao", () => this.ssao.enabled.toggle()); } toggleHelpPanel() { diff --git a/src/webgl/dynamic_shader.ts b/src/webgl/dynamic_shader.ts index e385a5220e..7ab6586b7b 100644 --- a/src/webgl/dynamic_shader.ts +++ b/src/webgl/dynamic_shader.ts @@ -230,6 +230,7 @@ export interface ParameterizedEmitterDependentShaderOptions< builder: ShaderBuilder, parameters: Parameters, extraParameters: ExtraParameters, + emitter?: ShaderModule, ) => void; } @@ -264,7 +265,12 @@ export function parameterizedEmitterDependentShaderGetter< extraParameters, ) => { builder.require(emitter); - return options.defineShader(builder, parameters, extraParameters); + return options.defineShader( + builder, + parameters, + extraParameters, + emitter, + ); }, }); } diff --git a/src/widget/range.ts b/src/widget/range.ts index 7558abc3e4..e721f208cb 100644 --- a/src/widget/range.ts +++ b/src/widget/range.ts @@ -24,6 +24,7 @@ export interface RangeWidgetOptions { min?: number; max?: number; step?: number; + label?: string; } export class RangeWidget extends RefCounted { @@ -33,11 +34,14 @@ export class RangeWidget extends RefCounted { constructor( public value: WatchableValueInterface, - { min = 0, max = 1, step = 0.01 }: RangeWidgetOptions = {}, + { min = 0, max = 1, step = 0.01, label }: RangeWidgetOptions = {}, ) { super(); const { element, inputElement, numericInputElement } = this; element.className = "range-slider"; + if (label !== undefined) { + element.appendChild(document.createTextNode(label)); + } const initInputElement = (el: HTMLInputElement) => { el.min = "" + min; el.max = "" + max;