diff --git a/.github/workflows/ffmpeg-compatibility.yml b/.github/workflows/ffmpeg-compatibility.yml index 4443fa0f3..078298474 100644 --- a/.github/workflows/ffmpeg-compatibility.yml +++ b/.github/workflows/ffmpeg-compatibility.yml @@ -3,10 +3,16 @@ name: FFmpeg compatibility on: + push: + branches: [vulkan-mvp] pull_request: paths: - 'src/FFmpeg*' + - 'src/Vulkan*' + - 'src/GpuDevices*' + - 'src/Qt/Vulkan*' - 'tests/FFmpeg*' + - 'tests/Vulkan*' - 'tests/CMakeLists.txt' - '.github/workflows/ffmpeg-compatibility.yml' workflow_dispatch: @@ -57,7 +63,7 @@ jobs: sudo apt-get update sudo apt-get install -y cmake ninja-build pkg-config nasm catch2 \ libasound2-dev qtbase5-dev libqt5svg5-dev libbabl-dev libjsoncpp-dev \ - libx264-dev libvpx-dev libvorbis-dev + libx264-dev libvpx-dev libvorbis-dev libvulkan-dev glslang-tools - name: Build isolated FFmpeg and audio dependencies run: | prefix="$RUNNER_TEMP/ffmpeg-dependencies" @@ -83,9 +89,11 @@ jobs: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH="$RUNNER_TEMP/ffmpeg-dependencies" \ -DENABLE_LIB_DOCS=OFF -DENABLE_PYTHON=OFF -DENABLE_OPENCV=OFF \ - -DENABLE_MAGICK=OFF -DUSE_QT6=OFF + -DENABLE_MAGICK=OFF -DUSE_QT6=OFF -DENABLE_GPU_FRAME=ON cmake --build build --parallel 2 --target \ - openshot-FFmpegCapabilities-test openshot-FFmpegWriter-test + openshot-FFmpegCapabilities-test openshot-FFmpegWriter-test \ + openshot-VulkanCompositor-test openshot-VulkanPresentation-test - name: Test capability validation and audio/video exports working-directory: build - run: ctest -L '^(FFmpegCapabilities|FFmpegWriter)$' --output-on-failure --timeout 180 -j2 + # Hardware cases skip without Vulkan support; the CPU fallback still runs. + run: ctest -L '^(FFmpegCapabilities|FFmpegWriter|VulkanCompositor|VulkanPresentation)$' --output-on-failure --timeout 180 -j2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 610e468cc..063f6aa92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,13 +24,17 @@ init-builders: linux-builder: stage: build-libopenshot variables: + OPENSHOT_QT_ROOT: /opt/openshot/qt-5.15.19 OPENCV_ROOT: /opt/opencv-4.13.0 OpenCV_DIR: /opt/opencv-4.13.0/lib/cmake/opencv4 + # Runner image provides FFmpeg 7 (with Vulkan) and newer Vulkan headers in /opt FFMPEG_ROOT: /opt/ffmpeg-7.1.3 + VULKAN_HEADERS_ROOT: /opt/vulkan-headers artifacts: expire_in: 6 months paths: - build/install-x64/* + - build/ffmpeg-hardware-check/* script: - "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=linux-builder" - if [ ! -f artifacts.zip ]; then @@ -43,13 +47,14 @@ linux-builder: - export PKG_CONFIG_PATH="$OPENCV_ROOT/lib/pkgconfig:$PKG_CONFIG_PATH" - export LD_LIBRARY_PATH="$OPENCV_ROOT/lib:$LD_LIBRARY_PATH" - export PATH="$OPENCV_ROOT/bin:$PATH" - - if [ ! -f "$FFMPEG_ROOT/env.sh" ]; then echo "ERROR runner is missing $FFMPEG_ROOT"; exit 1; fi + - if [ ! -f "$FFMPEG_ROOT/env.sh" ]; then echo "ERROR runner is missing $FFMPEG_ROOT (FFmpeg 7 with Vulkan)"; exit 1; fi - . "$FFMPEG_ROOT/env.sh" + - . "$OPENSHOT_QT_ROOT/env.sh" - export FFMPEGDIR="$FFMPEG_ROOT" - bash ci/linux/check-ffmpeg-hardware.sh "$FFMPEG_ROOT" "$CI_PROJECT_DIR/build/ffmpeg-hardware-check" - mkdir -p build; cd build; - pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0 || echo "WARNING PipeWire Wayland capture dependencies are missing; building without Wayland capture support" - - cmake -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON ../ + - cmake -U "Qt5*_DIR" -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_QT6=OFF -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$VULKAN_HEADERS_ROOT/include" -D"CMAKE_CXX_FLAGS=-I$VULKAN_HEADERS_ROOT/include" ../ - cmake --build . --parallel $(nproc) - ldd src/libopenshot.so | tee libopenshot-deps.log - if grep -qE "not found|libavcodec\.so\.58|libavutil\.so\.56" libopenshot-deps.log; then echo "ERROR unresolved libraries or system FFmpeg 4.2"; exit 1; fi @@ -89,7 +94,8 @@ mac-builder: - export PKG_CONFIG_PATH="$FFMPEGDIR/lib/pkgconfig:$PKG_CONFIG_PATH" - export DYLD_LIBRARY_PATH=$FFMPEGDIR/lib:$OPENCV_ROOT/lib:$DYLD_LIBRARY_PATH - mkdir -p build; cd build; - - cmake -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.12" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../ + - test -f "$FFMPEGDIR/lib/libMoltenVK.dylib" && test -x "$FFMPEGDIR/bin/glslangValidator" + - cmake -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$FFMPEGDIR/include" -D"Vulkan_LIBRARY:FILEPATH=$FFMPEGDIR/lib/libMoltenVK.dylib" -D"GLSLANG_VALIDATOR:FILEPATH=$FFMPEGDIR/bin/glslangValidator" -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.0.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.15" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../ - cmake --build . --parallel $(sysctl -n hw.ncpu) - make install - find "$CI_PROJECT_DIR/build/install-x64" -type f \( -name "*.dylib" -or -name "*.so" \) -exec sh -c 'for f do if ! otool -l "$f" | grep -q "$OPENCV_ROOT/lib"; then install_name_tool -add_rpath "$OPENCV_ROOT/lib" "$f" || true; fi; done' sh {} + @@ -125,7 +131,7 @@ windows-builder-x64: - 'if ($LASTEXITCODE -ne 0) { throw "Could not enumerate FFmpeg encoders" }; foreach ($codec in @("h264_nvenc", "hevc_nvenc", "h264_qsv", "hevc_qsv", "h264_amf", "hevc_amf")) { if ($gpuEncoders -notmatch "\b$codec\b") { throw "Windows FFmpeg is missing required hardware encoder $codec" } }' - ffmpeg -hide_banner -devices | findstr /I "gdigrab" - ffmpeg -hide_banner -devices | findstr /I "dshow" - - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" + - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -DENABLE_GPU_FRAME=ON - cmake --build build --parallel 4 - $testList = ctest --test-dir build --show-only=json-v1 | ConvertFrom-Json - if ($LASTEXITCODE -ne 0 -or @($testList.tests).Count -lt 400) { throw "Incomplete unit test discovery; check runtime DLL paths" } @@ -202,7 +208,7 @@ windows-builder-x86: - 'if ($LASTEXITCODE -ne 0) { throw "Could not enumerate FFmpeg encoders" }; foreach ($codec in @("h264_nvenc", "hevc_nvenc", "h264_qsv", "hevc_qsv", "h264_amf", "hevc_amf")) { if ($gpuEncoders -notmatch "\b$codec\b") { throw "Windows FFmpeg is missing required hardware encoder $codec" } }' - ffmpeg -hide_banner -devices | findstr /I "gdigrab" - ffmpeg -hide_banner -devices | findstr /I "dshow" - - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32 -msse2 -mfpmath=sse" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32 -msse2 -mfpmath=sse" + - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32 -msse2 -mfpmath=sse" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32 -msse2 -mfpmath=sse" -DENABLE_GPU_FRAME=ON - cmake --build build --parallel 4 - $testList = ctest --test-dir build --show-only=json-v1 | ConvertFrom-Json - if ($LASTEXITCODE -ne 0 -or @($testList.tests).Count -lt 400) { throw "Incomplete unit test discovery; check runtime DLL paths" } diff --git a/CMakeLists.txt b/CMakeLists.txt index 35e3a1bda..c3db62bdc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ For more information, please visit . set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") ################ PROJECT VERSION #################### -set(PROJECT_VERSION_FULL "1.1.1") +set(PROJECT_VERSION_FULL "1.2.0-dev1") set(PROJECT_SO_VERSION 33) # Remove the dash and anything following, to get the #.#.# version for project() @@ -75,6 +75,7 @@ option(ENABLE_OPENCV "Build with OpenCV algorithms (requires Boost, Protobuf 3)" option(ENABLE_WAYLAND_CAPTURE "Build Wayland screen capture support with xdg-desktop-portal and PipeWire (Linux only)" ON) option(USE_HW_ACCEL "Enable hardware-accelerated encoding-decoding with FFmpeg 3.4+" ON) option(ENABLE_VULKAN_BENCHMARK "Build the experimental Vulkan benchmark example" OFF) +option(ENABLE_GPU_FRAME "Build experimental deferred GPU frame storage" OFF) # Legacy commandline override if (DISABLE_TESTS) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index ef43d543f..662dae963 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -89,6 +89,14 @@ if (CMAKE_VERSION VERSION_GREATER 3.20) endif() ### Add the SWIG interface file (which defines all the SWIG methods) +# Older UseSWIG versions do not track transitive %include headers. Regenerate +# the wrapper when GPU diagnostics change, including in incremental builds. +set(SWIG_MODULE_pyopenshot_EXTRA_DEPS + "${PROJECT_SOURCE_DIR}/src/GpuDevices.h" + "${PROJECT_SOURCE_DIR}/src/GpuMetrics.h" + "${PROJECT_SOURCE_DIR}/src/Timeline.h" + "${PROJECT_SOURCE_DIR}/src/FFmpegWriter.h" + "${PROJECT_SOURCE_DIR}/src/Qt/VulkanOpenGLPresenter.h") swig_add_library(pyopenshot LANGUAGE python SOURCES openshot.i) if (POLICY CMP0078) diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index 6602b465c..99c6b3680 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -110,6 +110,13 @@ class QWidget; #include "QtTextReader.h" #include "KeyFrame.h" #include "RendererBase.h" +#if defined(OPENSHOT_ENABLE_GPU_FRAME) +#include "Qt/VulkanOpenGLPresenter.h" +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "GpuDevices.h" +#endif +#include "GpuMetrics.h" #include "Settings.h" #include "TimelineBase.h" #include "Timeline.h" @@ -520,6 +527,14 @@ static int openshot_swig_is_qwidget(PyObject *obj) { %include "KeyFrame.h" %include "AnimatedCurve.h" %include "RendererBase.h" +#if defined(OPENSHOT_ENABLE_GPU_FRAME) +%include "Qt/VulkanOpenGLPresenter.h" +namespace openshot { void ReleaseVulkanMemory(); uint64_t VulkanFrameReadbackCount(); uint64_t VulkanUploadFailureCount(); } +%include "GpuDevices.h" +%template(GpuDeviceVector) std::vector; +%template(StringVector) std::vector; +#endif +%include "GpuMetrics.h" %include "Settings.h" %include "TimelineBase.h" %include "Qt/VideoCacheThread.h" diff --git a/ci/linux/check-ffmpeg-hardware.sh b/ci/linux/check-ffmpeg-hardware.sh old mode 100644 new mode 100755 index 4c7275e21..45c3ade0f --- a/ci/linux/check-ffmpeg-hardware.sh +++ b/ci/linux/check-ffmpeg-hardware.sh @@ -11,7 +11,7 @@ export LD_LIBRARY_PATH="$root/lib:${LD_LIBRARY_PATH:-}" "$root/bin/ffmpeg" -hide_banner -version > "$log/version.txt" "$root/bin/ffmpeg" -hide_banner -encoders > "$log/encoders.txt" 2>&1 "$root/bin/ffmpeg" -hide_banner -hwaccels > "$log/hwaccels.txt" 2>&1 -for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf h264_vaapi hevc_vaapi; do +for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf h264_vaapi hevc_vaapi h264_vulkan hevc_vulkan; do grep -Eq "[[:space:]]$codec[[:space:]]" "$log/encoders.txt" || { echo "Missing required encoder: $codec"; exit 1; } done ldd "$root/lib/libavcodec.so.61" > "$log/avcodec-dependencies.txt" @@ -21,4 +21,4 @@ grep -q 'libvpl.so.2' "$log/avcodec-dependencies.txt" "$root/bin/ffmpeg" -v error -i "$log/h264-aac.mp4" -f null - "$root/bin/ffmpeg" -v error -f lavfi -i testsrc2=size=128x96:rate=4 -frames:v 4 -c:v libaom-av1 -cpu-used 8 -threads 2 -y "$log/av1.webm" "$root/bin/ffmpeg" -v error -c:v libdav1d -i "$log/av1.webm" -f null - -echo 'SUCCESS: NVENC, QSV, AMF and VA-API encoders compiled; dispatcher linked; H.264/AAC and AV1 software smoke tests passed' +echo 'SUCCESS: NVENC, QSV, AMF, VA-API and Vulkan encoders compiled; dispatcher linked; H.264/AAC and AV1 software smoke tests passed' diff --git a/ci/linux/setup-ffmpeg-vulkan.sh b/ci/linux/setup-ffmpeg-vulkan.sh new file mode 100755 index 000000000..fa56a020c --- /dev/null +++ b/ci/linux/setup-ffmpeg-vulkan.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +# Install FFmpeg with Vulkan (plus the codecs the focal AppImage ships) into +# /opt, without touching the system FFmpeg used by the CPU-only builds. +# Ubuntu 20.04 (focal). Run as a sudo-capable user. Safe to re-run. +# Not enabled: libwavpack (removed from FFmpeg 5+, native decoder used), +# chromaprint (focal's library links FFmpeg 4.2, loading two FFmpegs at once). +set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + +FFMPEG_VERSION=${FFMPEG_VERSION:-7.1.3} +VULKAN_HEADERS_TAG=${VULKAN_HEADERS_TAG:-v1.4.347} +AOM_TAG=${AOM_TAG:-v3.15.2} # focal has 1.0, FFmpeg 7 needs 2.0+ +DAV1D_TAG=${DAV1D_TAG:-1.4.3} # focal has no libdav1d-dev +NV_CODEC_TAG=${NV_CODEC_TAG:-n12.1.14.0} # NVIDIA driver 530+ for NVENC +PREFIX=${PREFIX:-/opt/ffmpeg-${FFMPEG_VERSION}} +VULKAN_PREFIX=${VULKAN_PREFIX:-/opt/vulkan-headers} +SRC=${SRC:-/opt/src} +JOBS=${JOBS:-$(nproc)} +SKIP_APT=${SKIP_APT:-0} + +export DEBIAN_FRONTEND=noninteractive +if [ "$SKIP_APT" != 1 ]; then +sudo apt-get update +sudo apt-get install -y --no-install-recommends \ + build-essential pkg-config nasm yasm git curl xz-utils cmake patch \ + glslang-tools libvulkan-dev \ + libx264-dev libx265-dev libvpx-dev meson ninja-build \ + libmp3lame-dev libopus-dev libvorbis-dev libtheora-dev libspeex-dev \ + libtwolame-dev libshine-dev libwavpack-dev libgsm1-dev libcodec2-dev \ + libxvidcore-dev libwebp-dev libopenjp2-7-dev libsnappy-dev libsoxr-dev \ + libass-dev libfreetype6-dev libfontconfig1-dev libfribidi-dev libxml2-dev \ + libbluray-dev libbs2b-dev libcaca-dev libcdio-paranoia-dev flite1-dev \ + libgme-dev libmysofa-dev libopenmpt-dev librsvg2-dev librubberband-dev \ + libssh-dev libgnutls28-dev libvidstab-dev libzmq3-dev libzvbi-dev \ + lv2-dev liblilv-dev ladspa-sdk frei0r-plugins-dev \ + libpulse-dev libjack-jackd2-dev libopenal-dev libsdl2-dev \ + ocl-icd-opencl-dev opencl-headers libgl-dev \ + libdc1394-22-dev libiec61883-dev libavc1394-dev libraw1394-dev \ + libdrm-dev libva-dev libvdpau-dev +fi + +sudo mkdir -p "$SRC" && sudo chown "$(id -u):$(id -g)" "$SRC" +cd "$SRC" + +# Vulkan headers: focal has 1.2.131, FFmpeg 7 needs 1.3.277+. Header-only. +if [ ! -f "$VULKAN_PREFIX/include/vulkan/vulkan_core.h" ]; then + rm -rf Vulkan-Headers + git clone --depth 1 --branch "$VULKAN_HEADERS_TAG" https://github.com/KhronosGroup/Vulkan-Headers.git + # Copy directly: its CMakeLists needs CMake 3.22, focal has 3.16 + sudo mkdir -p "$VULKAN_PREFIX" + sudo cp -r Vulkan-Headers/include "$VULKAN_PREFIX/" +fi + +# NVENC/CUDA headers (header-only, loaded at runtime from the driver) +if [ ! -f "$PREFIX/lib/pkgconfig/ffnvcodec.pc" ]; then + rm -rf nv-codec-headers + git clone --depth 1 --branch "$NV_CODEC_TAG" https://github.com/FFmpeg/nv-codec-headers.git + sudo make -C nv-codec-headers install PREFIX="$PREFIX" +fi + +# Intel oneVPL dispatcher (supports both oneVPL and legacy Media SDK runtimes). +# The dispatcher is bundled; GPU runtimes/drivers remain supplied by the host. +if [ ! -f "$PREFIX/lib/pkgconfig/vpl.pc" ]; then + archive=libvpl-2.10.2.tar.gz + if [ ! -f "$archive" ]; then + curl -fsSL --retry 2 https://github.com/intel/libvpl/archive/refs/tags/v2.10.2.tar.gz -o "$archive" + fi + printf '%s %s\n' ad956ea7ecf14614325f59dfb44cc5ba08e2fcac373342d61c7db152ac651253 "$archive" | sha256sum -c - + mkdir -p libvpl-2.10.2 + tar xzf "$archive" --strip-components=1 -C libvpl-2.10.2 + cmake -S libvpl-2.10.2 -B libvpl-2.10.2/build -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTS=OFF -DBUILD_TOOLS=OFF -DBUILD_EXAMPLES=OFF -DINSTALL_EXAMPLE_CODE=OFF \ + -DBUILD_DISPATCHER_ONEVPL_EXPERIMENTAL=OFF + cmake --build libvpl-2.10.2/build --parallel "$JOBS" + sudo cmake --install libvpl-2.10.2/build +fi + +# AMD AMF headers, pinned to the 1.4.35 release commit and verified individually. +# FFmpeg loads the AMF runtime from the installed AMD driver at export time. +amf="$PREFIX/include/AMF" +sudo mkdir -p "$amf" +while read -r digest header; do + if ! printf '%s %s\n' "$digest" "$amf/$header" | sha256sum -c - >/dev/null 2>&1; then + mkdir -p "amf-headers/$(dirname "$header")" + curl -fsSL --retry 2 "https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/AMF/ba07d1b3c7c7ee0d846125cffc1d6d78a020c45a/amf/public/include/$header" -o "amf-headers/$header" + printf '%s %s\n' "$digest" "amf-headers/$header" | sha256sum -c - + sudo mkdir -p "$amf/$(dirname "$header")" + sudo cp "amf-headers/$header" "$amf/$header" + fi +done <<'AMF_HEADERS' +70afd5279be90e656c2ec270229c5a702a11435a670052370e96b270e6b7640f components/AMFXInput.h +27770ae77d2f7a16eca9ab1f8b9ce87cf0c0350425b950bdd39189c2c91a519a components/Ambisonic2SRenderer.h +05a9ebc32217e69b6c4659f4744dbda9e5a2792f3e56bc1bcf61c15dcfd46cab components/AudioCapture.h +bbe91a1e744685a36269d3b6af9cd36d9716ee748c8d0ed95255a6a42b46892f components/Capture.h +a3a89490d44adaffc2fa2cccc14313f8f65e5af7a6431d89d85bd80e36c628a1 components/ChromaKey.h +2f6887d14516ebdcf034d59e39235f5d9974533ddb0b04d685c0c3b6d397d3d6 components/ColorSpace.h +f3f18680d55a65fdf5e0351d3fde1dcd9bdd2d6caec268c8c915c4227ffa1edc components/Component.h +2479eb2a6174a1b63552c509abac1b701eddcab5a5eac5a7f4d9f40274d588b4 components/ComponentCaps.h +7c22337ff313320bd7fe20ef0fb00756735012367a5daa7c621725030f578863 components/CursorCapture.h +693c8e491f90eaffd226c1f6b2da37d3a5ef1d3c610e554647f7d0b699832727 components/DisplayCapture.h +9dde7e850c8bdecda82b0cee1b331294f6cc91b32b1d1504344a023cd4dda8af components/FFMPEGAudioConverter.h +92f50d789bc18d5ebd143b2925e124594a89110eef7efaa8189eb063bf2f3aa8 components/FFMPEGAudioDecoder.h +2493067de9b3f540ffaae8a6466e57ef75af4772f037a81785b11ee9511029f5 components/FFMPEGAudioEncoder.h +87a9ae9480c3af6210b1282d1768e36673fcc1ef4970afa6edaa74481cf0831b components/FFMPEGComponents.h +5d64b7d2d50ca17ba601885dcdb51bb16ef4dccf88323437967e8524f26194bc components/FFMPEGEncoderAV1.h +3dae38db05bc55af525000c28559ff76625e0cddc2ad2c4c2af52eab69e915bc components/FFMPEGEncoderH264.h +3ad5cc5d3284b253a671499efa29c7a46b83a6e6373aad7142c7adcbfae31875 components/FFMPEGEncoderHEVC.h +be992e14086b04e2a7b5503471a6c262c58fda163c45ad6d00f8e425d9f4a820 components/FFMPEGFileDemuxer.h +3fefbe1aa7aaa4c000a97efb2e5ba26dc87ec15d11b07ceb0f43c768a1d589a2 components/FFMPEGFileMuxer.h +297b0ab2291baeb44f30f03ffbf791a5603289ab34f625d60a33150f2be1f7b6 components/FFMPEGVideoDecoder.h +ce5fb69c0546dd08dff0ee680972c5ec5a0b2d9b95e509338358eb2827f30221 components/FRC.h +b63e856b55db7a6853f444a6d4d32cfe006d9e0fb1d9be1b3079557534738591 components/HQScaler.h +c4e168d29e3f7d9318435958c3c784ad4c497159a617bd7a5598a2951b1b15f6 components/MediaSource.h +dae3564f20ef5db959d31dfc74c226453b039f4919ddb65027a183cdae95c171 components/PreAnalysis.h +5a01d71e128cf8bcaa4945fe76d9aad0156c409df3b25616a75645ef15972dc3 components/PreProcessing.h +550900e56c8254bf9c42d1b1d6538a390f7f7d3bf4826ee385b019585cb6cd22 components/SupportedCodecs.h +146e7cd35acc846f6b75b696bbe13a5d790f885bc23d9913e7de66f515da789b components/VQEnhancer.h +08f0162a631833f847d1b4b751e41e28b147a79ed19545b43631222d22e74507 components/VideoCapture.h +b0e173fbfe920a4923f461c1bffc73d1cbc06d54b676647dcd6ec300a0beee65 components/VideoConverter.h +588622f7d945c85c1ed334b497174fd46031e01664338bcf59c4b242920df886 components/VideoDecoderUVD.h +0b43ba9a0bb0d701c117e038d07270907d840ea2d2b13655fec71eda5e96d562 components/VideoEncoderAV1.h +b7827d2c3380a32cef350cf0f180802694f9f93db295f1c8e08238160d17c3d8 components/VideoEncoderHEVC.h +d106ce15028a9ea42e7299cff4b1a8697614a746b96740c9251f9b943bac2edc components/VideoEncoderVCE.h +9bc2d8477d67693b0cd4d433fbc6a5812277190f67be5f188f0206b463f256ec components/VideoStitch.h +3c6e6515a8fec111eb1847e349124a8c66c1e78be2d148bc0b8f85b9d9994e2d components/ZCamLiveStream.h +c9382c911fd3b5e12105c1d2a67eb11e62e9475068825222a285bb1314ac0ddd core/AudioBuffer.h +1ad18d3c813c3f81a836d393bbe86dfc0b7a1d7f04a77a272eb1461ce85ae3a4 core/Buffer.h +ef5721466fe48e1c038151018699519761e6e351e66e76a5de65386924653e20 core/Compute.h +50acf69fb670bc6cf093384d1b698bf9e44cda8dd49e02ef24eacfb86ad8ccae core/ComputeFactory.h +f9b4acb784038aad234ddec490e5f6ad6dacc4b14794eb84f3523d56dc3d6dd2 core/Context.h +76d16ba57d48ae9e457f3d904698a1e7c625007ac8887db9bcecdb90fd36d835 core/CurrentTime.h +f05771305598b534e1efa0dd28825267ea87418c477b198e373f6764d060c666 core/D3D12AMF.h +eaac4d80dd3215638c70f8a4ca01791d26d9df77586f665a772bb51907a8035b core/Data.h +56666589bf6ad20c6286a048da751c2d7722f70c73d962ee3c205a6d709b6aff core/Debug.h +a5e0fa9290cae767a4d135962ee5e30302e65e8d0ca5e016c620e72b759c86c6 core/Dump.h +9ba2ebe93af628bce621f29de493cddab87bf8a030eed504b25fcdc822c61b44 core/Factory.h +304f6d6077e9b0733859d44a5e43a343ae374c8fe72c7d79741c0aca89c21162 core/Interface.h +5ad5165e9de2afcfc2da828e8704c838b1f82e8ead7422fbcfc32c758672e1f3 core/Plane.h +59a4d11ecd150af25ee0b8d1d15f2efdfe5cf1060b8e8ab539f6a46d4bff8bc8 core/Platform.h +0059b787e78cd7c4e57212ccaab56ad7e17e9474a31337da0eac5b2d056a5dd8 core/PropertyStorage.h +871136e46586baa1fc77121935b503aaa16523d006c3e435ce50698605675fad core/PropertyStorageEx.h +a469ff33875484b5762199da757819056ff4a7535bfdcec1f4cf2ffc403b98f2 core/Result.h +7f0c99bbfe156169440031f75b0e37c28c435dd140ecb4e5089d0b1f01912c47 core/Surface.h +ae8c5fee671f02b90d3ccab312cfa6f74424351f070a7dd4ed7aba437fd55c30 core/Trace.h +72841e751595a2a8f62c8ac4a3e630b181411af8e0741d716bbe3379c9bf9a99 core/Variant.h +e26e25e658513f7120bb16d64d753d99205b456d83a1120a0a0d2ffdbf19f8c5 core/Version.h +385b8ad96b59e2d67a6f7a78c6aa660a2f2cdad1bf8c7cadcd83387a3ad4996c core/VulkanAMF.h +AMF_HEADERS + +# dav1d (fast AV1 decoder), built into the FFmpeg prefix +if [ ! -f "$PREFIX/lib/pkgconfig/dav1d.pc" ]; then + rm -rf dav1d + git clone --depth 1 --branch "$DAV1D_TAG" https://code.videolan.org/videolan/dav1d.git + meson setup dav1d/build dav1d --prefix="$PREFIX" --libdir=lib --buildtype=release \ + -Denable_tools=false -Denable_tests=false + ninja -C dav1d/build + sudo ninja -C dav1d/build install +fi + +# libaom (AV1 encoding), built into the FFmpeg prefix +if [ ! -f "$PREFIX/lib/pkgconfig/aom.pc" ]; then + rm -rf aom + git clone --depth 1 --branch "$AOM_TAG" https://aomedia.googlesource.com/aom + cmake -S aom -B aom/build-out -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=1 \ + -DENABLE_TESTS=0 -DENABLE_EXAMPLES=0 -DENABLE_DOCS=0 -DENABLE_TOOLS=0 + cmake --build aom/build-out --parallel "$JOBS" + sudo cmake --install aom/build-out +fi + +if [ ! -d "ffmpeg-$FFMPEG_VERSION" ]; then + # GitHub mirror: ffmpeg.org is not always reachable from the runners + archive="FFmpeg-n$FFMPEG_VERSION.tar.gz" + curl -fsSL --retry 2 "https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n$FFMPEG_VERSION.tar.gz" -o "$archive" + printf '%s %s\n' "${FFMPEG_SHA256:-e0b04c4b43d7e6d67cb6710334fb513adf13ac860532f30e1d0ac4c231f232fb}" "$archive" | sha256sum -c - + tar xzf "$archive" + mv "FFmpeg-n$FFMPEG_VERSION" "ffmpeg-$FFMPEG_VERSION" +fi +cd "ffmpeg-$FFMPEG_VERSION" +bash "$script_dir/../patch-ffmpeg-vulkan.sh" "$PWD" +export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig" +./configure --prefix="$PREFIX" --enable-gpl --enable-version3 \ + --enable-shared --disable-static --disable-doc \ + --extra-cflags="-I$VULKAN_PREFIX/include -I$PREFIX/include" \ + --extra-ldflags="-L$PREFIX/lib" \ + --enable-vulkan --enable-ffnvcodec --enable-nvenc --enable-nvdec --enable-cuvid \ + --enable-libvpl --enable-amf \ + --enable-vaapi --enable-vdpau --enable-libdrm --enable-opencl --enable-opengl \ + --enable-libx264 --enable-libx265 --enable-libvpx --enable-libaom --enable-libdav1d \ + --enable-libmp3lame --enable-libopus --enable-libvorbis --enable-libtheora \ + --enable-libspeex --enable-libtwolame --enable-libshine \ + --enable-libgsm --enable-libcodec2 --enable-libxvid --enable-libwebp \ + --enable-libopenjpeg --enable-libsnappy --enable-libsoxr \ + --enable-libass --enable-libfreetype --enable-libfontconfig --enable-libfribidi \ + --enable-libxml2 --enable-libbluray --enable-libbs2b --enable-libcaca \ + --enable-libcdio --enable-libflite --enable-libgme --enable-libmysofa \ + --enable-libopenmpt --enable-librsvg --enable-librubberband --enable-libssh \ + --enable-gnutls --enable-libvidstab --enable-libzmq --enable-libzvbi \ + --enable-lv2 --enable-ladspa --enable-frei0r \ + --enable-libpulse --enable-libjack --enable-openal --enable-sdl2 \ + --enable-libdc1394 --enable-libiec61883 +make -j"$JOBS" +sudo make install + +# Build env for GPU jobs: source this before cmake. +sudo tee "$PREFIX/env.sh" >/dev/null < "$PREFIX/compiled-encoders.txt" 2>&1 +for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf h264_vaapi hevc_vaapi h264_vulkan hevc_vulkan; do + grep -Eq "[[:space:]]$codec[[:space:]]" "$PREFIX/compiled-encoders.txt" || { echo "Required encoder missing: $codec"; exit 1; } +done +echo "Done: source $PREFIX/env.sh" diff --git a/ci/patch-ffmpeg-vulkan.sh b/ci/patch-ffmpeg-vulkan.sh new file mode 100644 index 000000000..55b3faaea --- /dev/null +++ b/ci/patch-ffmpeg-vulkan.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +# Apply the FFmpeg 7.1 Vulkan host-import and staging-copy fixes. +set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +source_dir=${1:?Specify the FFmpeg source directory} +fix="$script_dir/patches/ffmpeg-vulkan-readback.patch" +if patch --dry-run --reverse --silent -d "$source_dir" -p1 < "$fix" >/dev/null 2>&1; then + echo "FFmpeg Vulkan readback fixes already applied" +else + patch --forward -d "$source_dir" -p1 < "$fix" +fi diff --git a/ci/patches/ffmpeg-vulkan-readback.patch b/ci/patches/ffmpeg-vulkan-readback.patch new file mode 100644 index 000000000..2f4da0716 --- /dev/null +++ b/ci/patches/ffmpeg-vulkan-readback.patch @@ -0,0 +1,42 @@ +--- a/libavutil/hwcontext_vulkan.c ++++ b/libavutil/hwcontext_vulkan.c +@@ -3813,13 +3813,15 @@ + } + } + +- for (int i = 0; i < planes; i++) +- av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset, +- region[i].bufferRowLength, +- swf->data[i], +- swf->linesize[i], +- swf->linesize[i], +- region[i].imageExtent.height); ++ for (int i = 0; i < planes; i++) { ++ if (upload) ++ av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset, region[i].bufferRowLength, ++ swf->data[i], swf->linesize[i], swf->linesize[i], region[i].imageExtent.height); ++ else ++ av_image_copy_plane(swf->data[i], swf->linesize[i], ++ vkbuf->mapped_mem + region[i].bufferOffset, region[i].bufferRowLength, ++ swf->linesize[i], region[i].imageExtent.height); ++ } + + if (!(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) && upload) { + ret = vk->FlushMappedMemoryRanges(hwctx->act_dev, 1, +@@ -4079,7 +4081,15 @@ + return AVERROR(EINVAL); + + /* Setup buffers first */ +- if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY) { ++ /* AMD Linux host-pointer imports can fail submission with EFAULT and ++ * lose the device. Use owned staging memory for CPU transfers there. ++ * GPU-to-GPU decode/render/presentation/export paths are unchanged. */ ++ int use_host_import = !!(p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY); ++#ifdef __linux__ ++ if (p->props.properties.vendorID == 0x1002) ++ use_host_import = 0; ++#endif ++ if (use_host_import) { + err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload); + if (err >= 0) + host_mapped = 1; diff --git a/ci/patches/ffmpeg-vulkan-readback.patch.license b/ci/patches/ffmpeg-vulkan-readback.patch.license new file mode 100644 index 000000000..e8cd53c24 --- /dev/null +++ b/ci/patches/ffmpeg-vulkan-readback.patch.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +SPDX-License-Identifier: LGPL-3.0-or-later diff --git a/ci/windows/setup-ffmpeg-windows-final.sh b/ci/windows/setup-ffmpeg-windows-final.sh new file mode 100755 index 000000000..9ab7699a9 --- /dev/null +++ b/ci/windows/setup-ffmpeg-windows-final.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +# Run using C:\msys64\usr\bin\bash.exe. No pacman changes. +# Usage: bash setup-ffmpeg-windows.sh mingw64|mingw32 +set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +arch=${1:?Specify mingw64 or mingw32} +case "$arch" in mingw64) cpu=x86_64 ;; mingw32) cpu=x86 ;; *) exit 2 ;; esac +version=7.1.3 +root=/c/runner-setup +prefix=${PREFIX:-$root/ffmpeg-$version-$arch} +hw_prefix=${HW_PREFIX:-$root/hw-ffmpeg-$arch} +work=${BUILD_DIR:-$root/build-$version-$arch} +export PATH="$root/tools/bin:/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows" +export PKG_CONFIG_LIBDIR="/$arch/lib/pkgconfig:/$arch/share/pkgconfig" +export PKG_CONFIG_PATH="$hw_prefix/lib/pkgconfig:$prefix/lib/pkgconfig" +mkdir -p "$root/logs" "$prefix" "$work" +exec > >(tee -a "$root/logs/ffmpeg-$version-$arch.log") 2>&1 +trap 'echo "FAILED at line $LINENO"; exit 1' ERR +echo "Building $version $arch at $(date)" +gcc --version | head -1 +cd "$root" +if [ ! -f "FFmpeg-n$version.tar.gz" ]; then + curl --cacert /usr/ssl/cert.pem -fL --retry 2 "https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n$version.tar.gz" -o "FFmpeg-n$version.tar.gz" +fi +printf '%s %s\n' e0b04c4b43d7e6d67cb6710334fb513adf13ac860532f30e1d0ac4c231f232fb "FFmpeg-n$version.tar.gz" | sha256sum -c - +if [ ! -d "$root/FFmpeg-n$version" ]; then tar xzf "FFmpeg-n$version.tar.gz"; fi +bash "$script_dir/../patch-ffmpeg-vulkan.sh" "$root/FFmpeg-n$version" +cd "$work" +opts=(--prefix="$prefix" --target-os=mingw32 --arch="$cpu" + --enable-gpl --enable-version3 --enable-shared --disable-static --disable-doc + --enable-ffnvcodec --enable-nvenc --enable-nvdec --enable-cuvid --enable-amf --enable-libvpl + --extra-cflags="-I$hw_prefix/include" + --enable-x86asm --x86asmexe="$root/tools/bin/nasm.exe" --enable-libdav1d --extra-ldflags="-L$prefix/lib") +if [ "${WITH_VULKAN:-0}" = 1 ]; then + cflags="-I$hw_prefix/include -I$root/vulkan-$arch/include" + if [ "$arch" = mingw32 ]; then cflags="$cflags -msse2 -mfpmath=sse"; fi + opts+=(--enable-vulkan --extra-cflags="$cflags") +else + opts+=(--disable-vulkan) +fi +# Enable the existing codec libraries explicitly; configure reports incompatible ones. +for pair in libx264:x264 libx265:x265 libvpx:vpx libopus:opus libvorbis:vorbis libtheora:theoraenc libmp3lame:lame libwebp:libwebp libopenjpeg:libopenjp2 libass:libass libfreetype:freetype2 libfontconfig:fontconfig libfribidi:fribidi libsoxr:soxr; do + feature=${pair%%:*}; package=${pair#*:} + if pkg-config --exists "$package"; then opts+=("--enable-$feature"); fi +done +"$root/FFmpeg-n$version/configure" "${opts[@]}" +make -j"${JOBS:-6}" +make install +# The oneVPL dispatcher is an application dependency; vendor GPU drivers remain system supplied. +cp "$hw_prefix/bin/libvpl.dll" "$prefix/bin/" +mkdir -p "$prefix/share/vpl/licensing" +cp "$hw_prefix/share/vpl/licensing/"* "$prefix/share/vpl/licensing/" +export PATH="$prefix/bin:$hw_prefix/bin:$PATH" +"$prefix/bin/ffmpeg.exe" -version +"$prefix/bin/ffmpeg.exe" -hide_banner -f lavfi -i testsrc2=size=128x96:rate=5 -t 1 -c:v mpeg4 -y "$prefix/smoke-test.avi" +"$prefix/bin/ffprobe.exe" -v error -show_entries stream=codec_name,width,height "$prefix/smoke-test.avi" +cat > "$prefix/env.sh" < "$prefix/compiled-encoders.txt" 2>&1 +for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf; do + grep -Eq "[[:space:]]$codec[[:space:]]" "$prefix/compiled-encoders.txt" || { echo "Missing required encoder: $codec"; exit 1; } +done +cp config.h config_components.h "$prefix/" +echo "SUCCESS: $prefix includes NVIDIA NVENC, Intel QSV, and AMD AMF" diff --git a/ci/windows/setup-ffmpeg-windows.sh b/ci/windows/setup-ffmpeg-windows.sh new file mode 100755 index 000000000..c915b545b --- /dev/null +++ b/ci/windows/setup-ffmpeg-windows.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +set -euo pipefail +arch=${1:?Specify mingw64 or mingw32} +export PATH="/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows" +export WITH_VULKAN=${WITH_VULKAN:-1} +bash "$(dirname "$0")/setup-hw-ffmpeg-windows.sh" "$arch" +exec bash "$(dirname "$0")/setup-ffmpeg-windows-final.sh" "$@" diff --git a/ci/windows/setup-hw-ffmpeg-windows.sh b/ci/windows/setup-hw-ffmpeg-windows.sh new file mode 100755 index 000000000..37592c7b2 --- /dev/null +++ b/ci/windows/setup-hw-ffmpeg-windows.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +# Build pinned hardware encoder dependencies without changing the installed toolchain. +# Run with C:\msys64\usr\bin\bash.exe setup-hw-ffmpeg-windows.sh mingw64|mingw32. +set -euo pipefail +arch=${1:?Specify mingw64 or mingw32} +case "$arch" in mingw64|mingw32) ;; *) exit 2 ;; esac +root=/c/runner-setup +prefix=${HW_PREFIX:-$root/hw-ffmpeg-$arch} +export PATH="/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows" +mkdir -p "$prefix" "$root/logs" +exec > >(tee -a "$root/logs/hw-ffmpeg-$arch.log") 2>&1 +trap 'echo "FAILED at line $LINENO"' ERR +fetch() { + local name=$1 url=$2 digest=$3 + if [ ! -f "$root/$name" ]; then curl --cacert /usr/ssl/cert.pem -fL --retry 2 "$url" -o "$root/$name.part"; mv "$root/$name.part" "$root/$name"; fi + printf '%s %s\n' "$digest" "$root/$name" | sha256sum -c - +} +fetch nv-codec-headers-n12.1.14.0.tar.gz https://github.com/FFmpeg/nv-codec-headers/archive/refs/tags/n12.1.14.0.tar.gz 2fefaa227d2a3b4170797796425a59d1dd2ed5fd231db9b4244468ba327acd0b +fetch libvpl-2.10.2.tar.gz https://github.com/intel/libvpl/archive/refs/tags/v2.10.2.tar.gz ad956ea7ecf14614325f59dfb44cc5ba08e2fcac373342d61c7db152ac651253 +for name in nv-codec-headers-n12.1.14.0 libvpl-2.10.2; do + source="$root/hw-sources/$name" + if [ ! -d "$source" ]; then mkdir -p "$source"; tar xzf "$root/$name.tar.gz" --strip-components=1 -C "$source"; fi +done +make -C "$root/hw-sources/nv-codec-headers-n12.1.14.0" install PREFIX="$prefix" +cmake -G 'MinGW Makefiles' -S "$root/hw-sources/libvpl-2.10.2" -B "$root/hw-vpl-build-$arch" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(cygpath -m "$prefix")" \ + -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DBUILD_SHARED_LIBS=ON -DBUILD_TESTS=OFF \ + -DBUILD_TOOLS=OFF -DBUILD_EXAMPLES=OFF -DINSTALL_EXAMPLE_CODE=OFF \ + -DBUILD_DISPATCHER_ONEVPL_EXPERIMENTAL=OFF +cmake --build "$root/hw-vpl-build-$arch" --parallel "${JOBS:-6}" +cmake --install "$root/hw-vpl-build-$arch" +# oneVPL installs Win32 DLLs under bin/x86; applications need the matching +# dispatcher directly beside FFmpeg, just like the x64 build. +if [ "$arch" = mingw32 ] && [ -f "$prefix/bin/x86/libvpl.dll" ]; then + cp "$prefix/bin/x86/libvpl.dll" "$prefix/bin/libvpl.dll" +fi +test -f "$prefix/bin/libvpl.dll" +# AMD AMF 1.4.35 headers, pinned to the release commit. No SDK binaries needed. +amf="$prefix/include/AMF" +mkdir -p "$amf" +while read -r digest header; do + mkdir -p "$amf/$(dirname "$header")" + if ! printf '%s %s\n' "$digest" "$amf/$header" | sha256sum -c - >/dev/null 2>&1; then + curl --cacert /usr/ssl/cert.pem -fL --retry 2 \ + "https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/AMF/ba07d1b3c7c7ee0d846125cffc1d6d78a020c45a/amf/public/include/$header" -o "$amf/$header" + fi + printf '%s %s\n' "$digest" "$amf/$header" | sha256sum -c - +done <<'AMF_HEADERS' +70afd5279be90e656c2ec270229c5a702a11435a670052370e96b270e6b7640f components/AMFXInput.h +27770ae77d2f7a16eca9ab1f8b9ce87cf0c0350425b950bdd39189c2c91a519a components/Ambisonic2SRenderer.h +05a9ebc32217e69b6c4659f4744dbda9e5a2792f3e56bc1bcf61c15dcfd46cab components/AudioCapture.h +bbe91a1e744685a36269d3b6af9cd36d9716ee748c8d0ed95255a6a42b46892f components/Capture.h +a3a89490d44adaffc2fa2cccc14313f8f65e5af7a6431d89d85bd80e36c628a1 components/ChromaKey.h +2f6887d14516ebdcf034d59e39235f5d9974533ddb0b04d685c0c3b6d397d3d6 components/ColorSpace.h +f3f18680d55a65fdf5e0351d3fde1dcd9bdd2d6caec268c8c915c4227ffa1edc components/Component.h +2479eb2a6174a1b63552c509abac1b701eddcab5a5eac5a7f4d9f40274d588b4 components/ComponentCaps.h +7c22337ff313320bd7fe20ef0fb00756735012367a5daa7c621725030f578863 components/CursorCapture.h +693c8e491f90eaffd226c1f6b2da37d3a5ef1d3c610e554647f7d0b699832727 components/DisplayCapture.h +9dde7e850c8bdecda82b0cee1b331294f6cc91b32b1d1504344a023cd4dda8af components/FFMPEGAudioConverter.h +92f50d789bc18d5ebd143b2925e124594a89110eef7efaa8189eb063bf2f3aa8 components/FFMPEGAudioDecoder.h +2493067de9b3f540ffaae8a6466e57ef75af4772f037a81785b11ee9511029f5 components/FFMPEGAudioEncoder.h +87a9ae9480c3af6210b1282d1768e36673fcc1ef4970afa6edaa74481cf0831b components/FFMPEGComponents.h +5d64b7d2d50ca17ba601885dcdb51bb16ef4dccf88323437967e8524f26194bc components/FFMPEGEncoderAV1.h +3dae38db05bc55af525000c28559ff76625e0cddc2ad2c4c2af52eab69e915bc components/FFMPEGEncoderH264.h +3ad5cc5d3284b253a671499efa29c7a46b83a6e6373aad7142c7adcbfae31875 components/FFMPEGEncoderHEVC.h +be992e14086b04e2a7b5503471a6c262c58fda163c45ad6d00f8e425d9f4a820 components/FFMPEGFileDemuxer.h +3fefbe1aa7aaa4c000a97efb2e5ba26dc87ec15d11b07ceb0f43c768a1d589a2 components/FFMPEGFileMuxer.h +297b0ab2291baeb44f30f03ffbf791a5603289ab34f625d60a33150f2be1f7b6 components/FFMPEGVideoDecoder.h +ce5fb69c0546dd08dff0ee680972c5ec5a0b2d9b95e509338358eb2827f30221 components/FRC.h +b63e856b55db7a6853f444a6d4d32cfe006d9e0fb1d9be1b3079557534738591 components/HQScaler.h +c4e168d29e3f7d9318435958c3c784ad4c497159a617bd7a5598a2951b1b15f6 components/MediaSource.h +dae3564f20ef5db959d31dfc74c226453b039f4919ddb65027a183cdae95c171 components/PreAnalysis.h +5a01d71e128cf8bcaa4945fe76d9aad0156c409df3b25616a75645ef15972dc3 components/PreProcessing.h +550900e56c8254bf9c42d1b1d6538a390f7f7d3bf4826ee385b019585cb6cd22 components/SupportedCodecs.h +146e7cd35acc846f6b75b696bbe13a5d790f885bc23d9913e7de66f515da789b components/VQEnhancer.h +08f0162a631833f847d1b4b751e41e28b147a79ed19545b43631222d22e74507 components/VideoCapture.h +b0e173fbfe920a4923f461c1bffc73d1cbc06d54b676647dcd6ec300a0beee65 components/VideoConverter.h +588622f7d945c85c1ed334b497174fd46031e01664338bcf59c4b242920df886 components/VideoDecoderUVD.h +0b43ba9a0bb0d701c117e038d07270907d840ea2d2b13655fec71eda5e96d562 components/VideoEncoderAV1.h +b7827d2c3380a32cef350cf0f180802694f9f93db295f1c8e08238160d17c3d8 components/VideoEncoderHEVC.h +d106ce15028a9ea42e7299cff4b1a8697614a746b96740c9251f9b943bac2edc components/VideoEncoderVCE.h +9bc2d8477d67693b0cd4d433fbc6a5812277190f67be5f188f0206b463f256ec components/VideoStitch.h +3c6e6515a8fec111eb1847e349124a8c66c1e78be2d148bc0b8f85b9d9994e2d components/ZCamLiveStream.h +c9382c911fd3b5e12105c1d2a67eb11e62e9475068825222a285bb1314ac0ddd core/AudioBuffer.h +1ad18d3c813c3f81a836d393bbe86dfc0b7a1d7f04a77a272eb1461ce85ae3a4 core/Buffer.h +ef5721466fe48e1c038151018699519761e6e351e66e76a5de65386924653e20 core/Compute.h +50acf69fb670bc6cf093384d1b698bf9e44cda8dd49e02ef24eacfb86ad8ccae core/ComputeFactory.h +f9b4acb784038aad234ddec490e5f6ad6dacc4b14794eb84f3523d56dc3d6dd2 core/Context.h +76d16ba57d48ae9e457f3d904698a1e7c625007ac8887db9bcecdb90fd36d835 core/CurrentTime.h +f05771305598b534e1efa0dd28825267ea87418c477b198e373f6764d060c666 core/D3D12AMF.h +eaac4d80dd3215638c70f8a4ca01791d26d9df77586f665a772bb51907a8035b core/Data.h +56666589bf6ad20c6286a048da751c2d7722f70c73d962ee3c205a6d709b6aff core/Debug.h +a5e0fa9290cae767a4d135962ee5e30302e65e8d0ca5e016c620e72b759c86c6 core/Dump.h +9ba2ebe93af628bce621f29de493cddab87bf8a030eed504b25fcdc822c61b44 core/Factory.h +304f6d6077e9b0733859d44a5e43a343ae374c8fe72c7d79741c0aca89c21162 core/Interface.h +5ad5165e9de2afcfc2da828e8704c838b1f82e8ead7422fbcfc32c758672e1f3 core/Plane.h +59a4d11ecd150af25ee0b8d1d15f2efdfe5cf1060b8e8ab539f6a46d4bff8bc8 core/Platform.h +0059b787e78cd7c4e57212ccaab56ad7e17e9474a31337da0eac5b2d056a5dd8 core/PropertyStorage.h +871136e46586baa1fc77121935b503aaa16523d006c3e435ce50698605675fad core/PropertyStorageEx.h +a469ff33875484b5762199da757819056ff4a7535bfdcec1f4cf2ffc403b98f2 core/Result.h +7f0c99bbfe156169440031f75b0e37c28c435dd140ecb4e5089d0b1f01912c47 core/Surface.h +ae8c5fee671f02b90d3ccab312cfa6f74424351f070a7dd4ed7aba437fd55c30 core/Trace.h +72841e751595a2a8f62c8ac4a3e630b181411af8e0741d716bbe3379c9bf9a99 core/Variant.h +e26e25e658513f7120bb16d64d753d99205b456d83a1120a0a0d2ffdbf19f8c5 core/Version.h +385b8ad96b59e2d67a6f7a78c6aa660a2f2cdad1bf8c7cadcd83387a3ad4996c core/VulkanAMF.h +AMF_HEADERS +export PKG_CONFIG_PATH="$prefix/lib/pkgconfig" +pkg-config --modversion ffnvcodec vpl +echo "SUCCESS: hardware encoder dependencies installed in $prefix" diff --git a/ci/windows/smoke-ffmpeg-windows.sh b/ci/windows/smoke-ffmpeg-windows.sh new file mode 100755 index 000000000..a98357e1c --- /dev/null +++ b/ci/windows/smoke-ffmpeg-windows.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +set -euo pipefail +arch=${1:?mingw64 or mingw32} +prefix=${PREFIX:-/c/runner-setup/ffmpeg-7.1.3-$arch} +export PATH="$prefix/bin:/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows" +exec > >(tee "/c/runner-setup/logs/ffmpeg-smoke-$arch.log") 2>&1 +"$prefix/bin/ffmpeg.exe" -version +"$prefix/bin/ffmpeg.exe" -hide_banner -hwaccels | grep vulkan +"$prefix/bin/ffmpeg.exe" -hide_banner -encoders > "$prefix/compiled-encoders.txt" 2>&1 +for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf; do + grep -Eq "[[:space:]]$codec[[:space:]]" "$prefix/compiled-encoders.txt" +done +"$prefix/bin/ffmpeg.exe" -hide_banner -devices > "$prefix/compiled-devices.txt" 2>&1 +grep -q gdigrab "$prefix/compiled-devices.txt" +grep -q dshow "$prefix/compiled-devices.txt" +"$prefix/bin/ffmpeg.exe" -hide_banner -f lavfi -i testsrc2=size=320x240:rate=24 -f lavfi -i sine=frequency=440:sample_rate=48000 -t 2 -c:v libx264 -preset fast -pix_fmt yuv420p -c:a aac -y "$prefix/h264-aac-test.mp4" +"$prefix/bin/ffmpeg.exe" -v error -i "$prefix/h264-aac-test.mp4" -f null - +"$prefix/bin/ffprobe.exe" -v error -show_entries stream=codec_name,width,height,sample_rate "$prefix/h264-aac-test.mp4" +"$prefix/bin/ffmpeg.exe" -v error -i /c/runner-setup/libopenshot-vulkan-mvp/examples/test_video_sync.mp4 -frames:v 3 -f null - +echo "SUCCESS: vendor encoders, screen/camera capture, Vulkan compiled in; H.264/AAC encode and decode $arch" diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 12e05cb9f..6a92d7dc6 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -64,6 +64,10 @@ target_compile_definitions(openshot-example PRIVATE target_link_libraries(openshot-example openshot) add_executable(openshot-html-example ExampleHtml.cpp) +add_executable(openshot-timeline-benchmark TimelineBenchmark.cpp) +target_link_libraries(openshot-timeline-benchmark openshot) +add_executable(openshot-export-benchmark TimelineExportBenchmark.cpp) +target_link_libraries(openshot-export-benchmark openshot) target_link_libraries(openshot-html-example openshot Qt${QT_VERSION_MAJOR}::Gui) if(ENABLE_VULKAN_BENCHMARK) diff --git a/examples/TimelineBenchmark.cpp b/examples/TimelineBenchmark.cpp new file mode 100644 index 000000000..b53324b37 --- /dev/null +++ b/examples/TimelineBenchmark.cpp @@ -0,0 +1,59 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "Clip.h" +#include "Timeline.h" +#include +#include +#include +#include +#include + +// Same workload for CPU and experimental GPU builds. Readback is optional so +// processing and the existing QImage presentation boundary can be distinguished. +int main(int argc, char** argv) { + if (argc < 2 || argc > 6) { + std::cerr << "Usage: openshot-timeline-benchmark VIDEO [FRAMES=120] [LAYERS=1] [READBACK=0] [DUMP.png]\n"; + return 2; + } + try { + const int count = argc > 2 ? std::stoi(argv[2]) : 120; + const int layers = argc > 3 ? std::stoi(argv[3]) : 1; + const bool readback = argc > 4 && std::stoi(argv[4]) != 0; + if (count < 1 || layers < 1 || layers > 16) throw std::invalid_argument("Invalid frame/layer count"); + std::vector> clips; + openshot::Timeline timeline(3840, 2160, openshot::Fraction(24, 1), 48000, 2, openshot::LAYOUT_STEREO); + timeline.SetMaxSize(854, 480); + for (int i = 0; i < layers; ++i) { + auto clip = std::make_unique(argv[1]); + clip->Layer(i); + if (i) { + clip->scale_x = openshot::Keyframe(0.7); + clip->scale_y = openshot::Keyframe(0.7); + clip->rotation = openshot::Keyframe(12.0 * i); + clip->alpha = openshot::Keyframe(0.6); + } + timeline.AddClip(clip.get()); + clips.push_back(std::move(clip)); + } + timeline.Open(); + int deferred = 0; + std::shared_ptr last; + const auto start = std::chrono::steady_clock::now(); + for (int i = 1; i <= count; ++i) { + last = timeline.GetFrame(i); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + deferred += bool(last->GetDeferredImage()); +#endif + if (readback) last->GetImage(); + } + const double seconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::cout << "frames=" << count << " layers=" << layers << " readback=" << readback + << " deferred_frames=" << deferred << " seconds=" << seconds + << " fps=" << count / seconds << std::endl; + if (argc > 5 && !last->GetImage()->save(argv[5])) throw std::runtime_error("Dump failed"); + timeline.Close(); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/examples/TimelineExportBenchmark.cpp b/examples/TimelineExportBenchmark.cpp new file mode 100644 index 000000000..3739de5ba --- /dev/null +++ b/examples/TimelineExportBenchmark.cpp @@ -0,0 +1,75 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "Clip.h" +#include "Timeline.h" +#include "FFmpegWriter.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanFrameImage.h" +#endif +#include +#include +#include +#include + +// Identical full-resolution rendering + NVENC workload for CPU/GPU builds. +// Optional audio input exercises timeline mixing and muxing. Output is replaced. +int main(int argc,char** argv) { + if(argc<3 || argc>7){ + std::cerr<<"Usage: openshot-export-benchmark VIDEO OUTPUT.mp4 [FRAMES=120] [LAYERS=2] [AUDIO|-] [CODEC=h264_nvenc]\n"; + return 2; + } + try { + const int count=argc>3?std::stoi(argv[3]):120; + const int layers=argc>4?std::stoi(argv[4]):2; + if(count<1 || layers<1 || layers>16) throw std::invalid_argument("Invalid frame/layer count"); + std::vector> clips; + openshot::Timeline timeline(3840,2160,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.SetMaxSize(3840,2160); + for(int index=0;index(argv[1]); + clip->Layer(index); + if(index){ + clip->scale_x=openshot::Keyframe(0.7); + clip->scale_y=openshot::Keyframe(0.65); + clip->shear_x=openshot::Keyframe(0.1); + clip->rotation=openshot::Keyframe(12.0*index); + clip->alpha=openshot::Keyframe(0.6); + } + timeline.AddClip(clip.get());clips.push_back(std::move(clip)); + } + if(argc>5 && std::string(argv[5])!="-"){ + auto audio=std::make_unique(argv[5]);audio->Layer(-1); + // Do not render cover art embedded in an audio file. + audio->has_video=openshot::Keyframe(0.0); + timeline.AddClip(audio.get());clips.push_back(std::move(audio)); + } + timeline.Open(); + openshot::FFmpegWriter writer(argv[2]); + writer.SetVideoOptions(true,argc>6?argv[6]:"h264_nvenc",openshot::Fraction(24,1),3840,2160,openshot::Fraction(1,1),false,false,40000000); + writer.SetAudioOptions(true,"aac",48000,2,openshot::LAYOUT_STEREO,192000); + writer.Open(); + int deferred=0; + std::shared_ptr last; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const auto readbacks=openshot::VulkanFrameReadbackCount(); +#endif + const auto start=std::chrono::steady_clock::now(); + for(int n=1;n<=count;++n){ + last=timeline.GetFrame(n); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + deferred+=bool(last->GetDeferredImage()); +#endif + writer.WriteFrame(last); + } + writer.Close(); + const double seconds=std::chrono::duration(std::chrono::steady_clock::now()-start).count(); + std::cout<<"frames="<] [--overlay ] [--frames ]\n" << " [--mode ] [--vaapi-device ] [--row ]\n" << " [--dump-dir ] [--dump-every ]\n" + << " [--cpu-only | --vulkan-only] (select compositor)\n" << "Rows: CPU->CPU, CPU->Vk, VAAPI->CPU, VAAPI->Vk, CUDA->CPU, CUDA->Vk, Vulkan->CPU, VulkanDirect->Vk\n"; std::exit(0); } else { @@ -2312,7 +2313,9 @@ int main(int argc, char* argv[]) { const QImage overlay_image = LoadOverlayImage(options); std::vector results; auto row_enabled = [&](const std::string& name) { - return options.row_name.empty() || options.row_name == name; + const bool vulkan_composite = name.size() >= 4 && name.compare(name.size() - 4, 4, "->Vk") == 0; + return (vulkan_composite ? options.run_vulkan : options.run_cpu) && + (options.row_name.empty() || options.row_name == name); }; auto run_row = [&](BenchmarkResult (*fn)(const BenchmarkOptions&, const QImage&), const std::string& fallback_name) { @@ -2393,7 +2396,7 @@ int main(int argc, char* argv[]) { add_unavailable_row("VulkanDirect->Vk", "Vulkan runtime unavailable"); } if (results.empty()) - throw std::runtime_error("Unknown row name: " + options.row_name); + throw std::runtime_error("No rows selected: check --row and compositor filters"); for (const auto& result : results) { result.stats.Print(result.name + " / " + result.decode_backend + " / " + result.composite_backend); @@ -2406,6 +2409,10 @@ int main(int argc, char* argv[]) { std::cout << "\nMatrix Summary\n"; PrintResultTable(results); + const bool processed_frames = std::any_of(results.begin(), results.end(), + [](const BenchmarkResult& result) { return !result.skipped && result.stats.frames > 0; }); + if (!processed_frames) + throw std::runtime_error("No benchmark row successfully processed frames"); return 0; } catch (const std::exception& exc) { std::cerr << "Benchmark failed: " << exc.what() << "\n"; diff --git a/examples/build-render-backend.sh b/examples/build-render-backend.sh new file mode 100644 index 000000000..4e58dec3a --- /dev/null +++ b/examples/build-render-backend.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Build isolated CPU/Vulkan experiments without replacing installed libraries. +set -euo pipefail + +backend=${OPENSHOT_RENDER_BACKEND:-cpu} +case "$backend" in + cpu) vulkan=OFF; target=openshot-benchmark ;; + vulkan) vulkan=ON; target=openshot-vulkan-benchmark ;; + *) echo 'OPENSHOT_RENDER_BACKEND must be cpu or vulkan' >&2; exit 2 ;; +esac + +source_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +build_dir=${OPENSHOT_BUILD_DIR:-"$source_dir/build-mvp-$backend"} + +# Additional arguments are CMake dependency/toolchain settings. Backend and +# Release settings below take precedence so the two experiments stay comparable. +cmake -S "$source_dir" -B "$build_dir" "$@" \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_VULKAN_BENCHMARK="$vulkan" \ + -DENABLE_GPU_FRAME="$vulkan" \ + -DENABLE_PYTHON=ON \ + -DBUILD_TESTING=ON +cmake --build "$build_dir" --target "$target" openshot-timeline-benchmark pyopenshot --parallel "${OPENSHOT_BUILD_JOBS:-4}" + +if [[ "$backend" == vulkan ]]; then + echo "Built experimental Vulkan benchmark: $build_dir/examples/$target" + echo 'Experimental GPU library enabled; verify GPU residency and fallbacks with the timeline benchmark.' +else + echo "Built CPU library and benchmark: $build_dir/tests/$target" +fi diff --git a/examples/color-chart-hlg.mp4 b/examples/color-chart-hlg.mp4 new file mode 100644 index 000000000..cd8183092 Binary files /dev/null and b/examples/color-chart-hlg.mp4 differ diff --git a/examples/color-chart-pq.mp4 b/examples/color-chart-pq.mp4 new file mode 100644 index 000000000..ae79235d4 Binary files /dev/null and b/examples/color-chart-pq.mp4 differ diff --git a/examples/color-chart-sdr-bt709.mp4 b/examples/color-chart-sdr-bt709.mp4 new file mode 100644 index 000000000..2dfff15f9 Binary files /dev/null and b/examples/color-chart-sdr-bt709.mp4 differ diff --git a/examples/run-timeline-benchmarks.py b/examples/run-timeline-benchmarks.py new file mode 100644 index 000000000..a3fffc862 --- /dev/null +++ b/examples/run-timeline-benchmarks.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +""" +Run comparable CPU/GPU timeline workloads sequentially and retain raw evidence. + +Example (inherit the Vulkan ICD and FFmpeg library environment): + python3 examples/run-timeline-benchmarks.py --cpu-build build --vulkan-build build-vk \ + --video /path/to/4k.mp4 --output /tmp/timeline-results.json +""" +import argparse +import datetime +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shutil +import statistics +# Local developer tools use explicit argument lists, never a shell. +import subprocess # nosec B404 +import sys + +RESULT = re.compile( + r"frames=(\d+) layers=(\d+) readback=(\d+) deferred_frames=(\d+) " + r"seconds=([\d.eE+-]+) fps=([\d.eE+-]+)" +) + + +def capture(command, timeout=20): + try: + # Developer-selected local tools run with argument lists, never a shell. + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, # nosec B603 + stderr=subprocess.STDOUT, timeout=timeout) + return {"command": command, "returncode": result.returncode, "output": result.stdout} + except (OSError, subprocess.TimeoutExpired) as error: + return {"command": command, "error": str(error)} + + +def validate_result(row, frames, layers, readback): + """Reject incomplete runs and unexpected CPU fallback before recording FPS.""" + match = RESULT.search(row.get("output", "")) + if row.get("returncode") == 0 and match: + got_frames, got_layers, got_readback, deferred, seconds, fps = match.groups() + row.update(frames=int(got_frames), deferred_frames=int(deferred), + seconds=float(seconds), fps=float(fps)) + row["valid"] = (int(got_frames) == frames and int(got_layers) == layers + and int(got_readback) == readback and float(seconds) > 0 + and (row["backend"] != "vulkan" or int(deferred) == frames)) + if not row["valid"]: + row["error"] = "Incomplete workload or Vulkan CPU fallback detected" + else: + row["valid"] = False + + +def verify_image(row, image_path, cpu_images, layers, readback): + """Compare a GPU result with the preceding CPU trial of the same workload.""" + from PIL import Image, ImageChops, ImageStat + backend = row["backend"] + try: + with Image.open(image_path) as loaded: + pixels = loaded.convert("RGB") + means = ImageStat.Stat(pixels).mean + row["image"] = {"path": str(image_path), "size": pixels.size, "mean_rgb": means} + if backend == "cpu": + cpu_images[(layers, readback)] = pixels + else: + reference = cpu_images[(layers, readback)] + if pixels.size != reference.size: + raise ValueError("GPU image dimensions differ from CPU") + row["image"]["cpu_mae_rgb"] = ImageStat.Stat(ImageChops.difference(pixels, reference)).mean + reference_means = ImageStat.Stat(reference).mean + if sum(reference_means) > 3 and sum(means) < 0.1 * sum(reference_means): + raise ValueError("GPU image is black/near-black while matching CPU image is not") + except (OSError, ValueError) as error: + row["valid"] = False + row["error"] = str(error) + + +def summarize(backend, layers, readback, successes): + """Keep failed trials out of the aggregate measurements.""" + return {"backend": backend, "layers": layers, "readback": readback, + "valid_runs": len(successes), + "median_fps": statistics.median(successes) if successes else None, + "min_fps": min(successes) if successes else None, + "max_fps": max(successes) if successes else None} + + +def measure(args, binaries, video, report, output): + """Run sequential trials, retaining CPU images for the GPU comparisons.""" + def save(): + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, indent=2) + "\n") + + if args.verify_images: + image_dir = output.parent / (output.stem + "-frames") + image_dir.mkdir(parents=True, exist_ok=True) + cpu_images = {} + failed = False + for layers in (1, 2): + for readback in (0, 1): + for backend, binary in binaries.items(): + successes = [] + for trial in range(1, args.runs + 1): + command = [str(binary), str(video), str(args.frames), str(layers), str(readback)] + image_path = None + if args.verify_images: + image_path = image_dir / f"{backend}-layers{layers}-readback{readback}-run{trial}.png" + command.append(str(image_path)) + evidence = capture(command, args.timeout) + row = {"backend": backend, "layers": layers, "readback": readback, + "trial": trial, **evidence} + validate_result(row, args.frames, layers, readback) + if row["valid"] and image_path: + verify_image(row, image_path, cpu_images, layers, readback) + failed |= not row["valid"] + if row["valid"]: + successes.append(row["fps"]) + report["measurements"].append(row) + print(f"{backend} layers={layers} readback={readback} trial={trial}: " + f"{row.get('fps', 'no result')} FPS valid={row['valid']}", flush=True) + save() + report["summary"].append(summarize(backend, layers, readback, successes)) + return failed + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cpu-build", required=True, type=Path) + parser.add_argument("--vulkan-build", required=True, type=Path) + parser.add_argument("--video", required=True, type=Path) + parser.add_argument("--frames", default=120, type=int) + parser.add_argument("--runs", default=3, type=int) + parser.add_argument("--timeout", default=180, type=int) + parser.add_argument("--output", default=Path("timeline-results.json"), type=Path) + parser.add_argument("--note", default="", help="Record background services or other measurement conditions") + parser.add_argument("--verify-images", action="store_true", help="Save final-frame PNGs and compare CPU/GPU pixels (requires Pillow)") + args = parser.parse_args() + if min(args.frames, args.runs, args.timeout) < 1: + parser.error("frames, runs and timeout must be positive") + video = args.video.resolve(strict=True) + binaries = {name: (directory / "examples/openshot-timeline-benchmark").resolve(strict=True) + for name, directory in (("cpu", args.cpu_build), ("vulkan", args.vulkan_build))} + output = args.output.resolve() + if output == video or output in binaries.values(): + parser.error("output must not overwrite an input file") + repo = Path(__file__).resolve().parent.parent + report = { + "started_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "machine": {"platform": platform.platform(), "processor": platform.processor(), + "logical_cpus": os.cpu_count()}, + "video": {"path": str(video), "bytes": video.stat().st_size, + "mtime_ns": video.stat().st_mtime_ns}, + "frames": args.frames, "runs": args.runs, "measurement_note": args.note, + "environment": {name: os.environ[name] for name in ( + "VK_ICD_FILENAMES", "VK_DRIVER_FILES", "LD_LIBRARY_PATH", "QT_QPA_PLATFORM", + "OMP_NUM_THREADS") if name in os.environ}, + "revision": capture(["git", "-C", str(repo), "rev-parse", "HEAD"]), + "working_tree": capture(["git", "-C", str(repo), "status", "--short"]), + "binaries": {}, "measurements": [], "summary": [], + "notes": "Each trial is a fresh process; includes initialization. Run without other GPU/CPU workloads. " + "Readback=0 excludes presentation; readback=1 includes QImage materialization, not Qt display. " + "Compare library versions below before attributing differences to the backend." + } + for name, binary in binaries.items(): + report["binaries"][name] = {"path": str(binary), + "sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "libraries": capture(["ldd", str(binary)])} + for name, command in (("gpu", ["nvidia-smi", "--query-gpu=name,driver_version,memory.total,memory.used,utilization.gpu", "--format=csv,noheader"]), + ("source", ["ffprobe", "-v", "error", "-show_streams", "-show_format", "-of", "json", str(video)])): + if shutil.which(command[0]): + report[name] = capture(command) + + if shutil.which("nvidia-smi"): + report["gpu_processes"] = capture(["nvidia-smi", "--query-compute-apps=pid,process_name,used_gpu_memory", "--format=csv,noheader"]) + + failed = measure(args, binaries, video, report, output) + report["finished_utc"] = datetime.datetime.now(datetime.timezone.utc).isoformat() + report["valid"] = not failed + output.write_text(json.dumps(report, indent=2) + "\n") + print(f"Results: {output}") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/AudioWaveformer.cpp b/src/AudioWaveformer.cpp index 3c0a5eb10..90437b6f2 100644 --- a/src/AudioWaveformer.cpp +++ b/src/AudioWaveformer.cpp @@ -31,6 +31,36 @@ using namespace std; using namespace openshot; +namespace { +// Waveform extraction reads every frame exactly once, front to back, so a large +// reader cache only costs memory (and, because CacheMemory sizes are rescanned on +// every insert, time). Keep readers owned by the waveformer nearly cache-free. +constexpr int64_t WAVEFORM_READER_CACHE_BYTES = 2 * 1024 * 1024; + +void LimitWaveformReaderCache(ReaderBase* reader) +{ + if (!reader) { + return; + } + if (CacheBase* cache = reader->GetCache()) { + cache->SetMaxBytes(WAVEFORM_READER_CACHE_BYTES); + } +} + +// Linearly interpolate a keyframe at a fractional (1-based) frame position +double KeyframeValueAt(const Keyframe& keyframe, double frame) +{ + const double floor_frame = std::floor(frame); + const int64_t index = static_cast(floor_frame); + const double value = keyframe.GetValue(index); + const double fraction = frame - floor_frame; + if (fraction <= 0.0) { + return value; + } + return value + (keyframe.GetValue(index + 1) - value) * fraction; +} +} + // Default constructor AudioWaveformer::AudioWaveformer(ReaderBase* new_reader) : reader(new_reader), @@ -81,9 +111,20 @@ AudioWaveformData AudioWaveformer::ExtractSamples(int channel, int num_per_secon return base; } +// Extract the mixed envelope plus one envelope per channel, in a single decode pass +AudioWaveformData AudioWaveformer::ExtractChannelSamples(int num_per_second, bool normalize) { + AudioWaveformData data; + if (!reader) { + return data; + } + ReaderBase* source = ResolveWaveformReader(); + return ExtractSamplesFromReader(source, -1, num_per_second, normalize, true); +} + AudioWaveformData AudioWaveformer::ExtractSamples(const std::string& path, int channel, int num_per_second, bool normalize) { FFmpegReader temp_reader(path); temp_reader.Open(); + LimitWaveformReaderCache(&temp_reader); // Disable video for speed bool has_video = temp_reader.info.has_video; temp_reader.info.has_video = false; @@ -102,6 +143,7 @@ AudioWaveformData AudioWaveformer::ExtractSamples(const std::string& path, bool normalize) { FFmpegReader temp_reader(path); temp_reader.Open(); + LimitWaveformReaderCache(&temp_reader); bool has_video = temp_reader.info.has_video; temp_reader.info.has_video = false; Fraction source_fps = temp_reader.info.fps; @@ -121,96 +163,121 @@ AudioWaveformData AudioWaveformer::ApplyKeyframes(const AudioWaveformData& base, int channel, bool normalize) { AudioWaveformData data; - if (num_per_second <= 0) { + if (source_fps.ToDouble() <= 0.0) { return data; } + if (channel != -1 && (channel < 0 || channel >= source_channels)) { + return data; + } + + data = ApplyKeyframes(base.max_samples, base.rms_samples, time_keyframe, volume_keyframe, project_fps, num_per_second); + + if (normalize) { + float samples_max = 0.0f; + for (auto v : data.max_samples) { + samples_max = std::max(samples_max, std::abs(v)); + } + if (samples_max > 0.0f) { + data.scale(static_cast(data.max_samples.size()), 1.0f / samples_max); + } + } - double project_fps_value = project_fps.ToDouble(); - double source_fps_value = source_fps.ToDouble(); - if (project_fps_value <= 0.0 || source_fps_value <= 0.0) { + return data; +} + +AudioWaveformData AudioWaveformer::ApplyKeyframes(const std::vector& max_samples, + const std::vector& rms_samples, + const Keyframe* time_keyframe, + const Keyframe* volume_keyframe, + const Fraction& project_fps, + int num_per_second) { + AudioWaveformData data; + if (num_per_second <= 0) { return data; } - if (channel != -1 && (channel < 0 || channel >= source_channels)) { + const double project_fps_value = project_fps.ToDouble(); + if (project_fps_value <= 0.0) { return data; } - size_t base_total = base.max_samples.size(); + const size_t base_total = max_samples.size(); if (base_total == 0) { return data; } + const size_t rms_total = std::min(rms_samples.size(), base_total); - // Determine output duration from time curve (if any). Time curves are in project-frame domain. - int64_t output_frames = 0; - if (time_keyframe && time_keyframe->GetCount() > 0) { - output_frames = time_keyframe->GetLength(); + // A single point cannot remap time, and an empty volume curve has no gain to apply + if (time_keyframe && time_keyframe->GetCount() < 2) { + time_keyframe = nullptr; } - if (output_frames <= 0) { - // Default to source duration derived from base waveform length - double source_duration = static_cast(base_total) / static_cast(num_per_second); - output_frames = static_cast(std::llround(source_duration * project_fps_value)); + if (volume_keyframe && volume_keyframe->GetCount() < 1) { + volume_keyframe = nullptr; } - double output_duration_seconds = static_cast(output_frames) / project_fps_value; - int total_samples = static_cast(std::ceil(output_duration_seconds * num_per_second)); + // Output duration follows the time curve (project frames), otherwise the source envelope + int64_t total_samples = static_cast(base_total); + if (time_keyframe) { + const int64_t output_frames = time_keyframe->GetLength(); + if (output_frames <= 0) { + return data; + } + const double output_duration_seconds = static_cast(output_frames) / project_fps_value; + total_samples = static_cast(std::ceil(output_duration_seconds * num_per_second)); + } if (total_samples <= 0) { return data; } - data.resize(total_samples); - data.zero(total_samples); + data.max_samples.assign(static_cast(total_samples), 0.0f); + data.rms_samples.assign(static_cast(total_samples), 0.0f); + + const double frames_per_sample = project_fps_value / static_cast(num_per_second); + const double samples_per_frame = static_cast(num_per_second) / project_fps_value; - for (int i = 0; i < total_samples; ++i) { - double out_time = static_cast(i) / static_cast(num_per_second); - // Time keyframes are defined in project-frame domain; evaluate using project frames - double project_frame = out_time * project_fps_value; - double mapped_project_frame = time_keyframe ? time_keyframe->GetValue(project_frame) : project_frame; - // Convert mapped project frame to seconds (project FPS), then to waveform index - double source_time = mapped_project_frame / project_fps_value; - double source_index = source_time * static_cast(num_per_second); + for (int64_t i = 0; i < total_samples; ++i) { + // 1-based project frame at this output sample (keyframes use 1-based frame numbers) + const double project_frame = static_cast(i) * frames_per_sample + 1.0; + + double source_index = static_cast(i); + if (time_keyframe) { + const double mapped_frame = KeyframeValueAt(*time_keyframe, project_frame); + source_index = (mapped_frame - 1.0) * samples_per_frame; + } // Sample base waveform (nearest with simple linear blend) - int idx0 = static_cast(std::floor(source_index)); - int idx1 = idx0 + 1; - double frac = source_index - static_cast(idx0); + const int64_t idx0 = static_cast(std::floor(source_index)); + const int64_t idx1 = idx0 + 1; + const double frac = source_index - static_cast(idx0); float max_sample = 0.0f; float rms_sample = 0.0f; - if (idx0 >= 0 && idx0 < static_cast(base_total)) { - max_sample = base.max_samples[idx0]; - rms_sample = base.rms_samples[idx0]; + if (idx0 >= 0 && idx0 < static_cast(base_total)) { + max_sample = max_samples[idx0]; + if (idx0 < static_cast(rms_total)) { + rms_sample = rms_samples[idx0]; + } } - if (idx1 >= 0 && idx1 < static_cast(base_total)) { - max_sample = static_cast((1.0 - frac) * max_sample + frac * base.max_samples[idx1]); - rms_sample = static_cast((1.0 - frac) * rms_sample + frac * base.rms_samples[idx1]); + if (frac > 0.0 && idx1 >= 0 && idx1 < static_cast(base_total)) { + max_sample = static_cast((1.0 - frac) * max_sample + frac * max_samples[idx1]); + if (idx1 < static_cast(rms_total)) { + rms_sample = static_cast((1.0 - frac) * rms_sample + frac * rms_samples[idx1]); + } } double gain = 1.0; if (volume_keyframe) { - double project_frame = out_time * project_fps_value; - gain = volume_keyframe->GetValue(project_frame); + gain = std::abs(KeyframeValueAt(*volume_keyframe, project_frame)); } - max_sample = static_cast(max_sample * gain); - rms_sample = static_cast(rms_sample * gain); - - data.max_samples[i] = max_sample; - data.rms_samples[i] = rms_sample; - } - if (normalize) { - float samples_max = 0.0f; - for (auto v : data.max_samples) { - samples_max = std::max(samples_max, std::abs(v)); - } - if (samples_max > 0.0f) { - data.scale(total_samples, 1.0f / samples_max); - } + data.max_samples[i] = static_cast(max_sample * gain); + data.rms_samples[i] = static_cast(rms_sample * gain); } return data; } -AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_reader, int channel, int num_per_second, bool normalize) { +AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_reader, int channel, int num_per_second, bool normalize, bool per_channel) { AudioWaveformData data; if (!source_reader || num_per_second <= 0) { @@ -220,6 +287,9 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r // Open reader (if needed) if (!source_reader->IsOpen()) { source_reader->Open(); + if (source_reader == detached_reader.get()) { + LimitWaveformReaderCache(source_reader); + } } const auto retry_delay = std::chrono::milliseconds(100); @@ -245,14 +315,10 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r } }; - int sample_rate = source_reader->info.sample_rate; + int64_t sample_rate = source_reader->info.sample_rate; if (sample_rate <= 0) { sample_rate = num_per_second; } - int sample_divisor = sample_rate / num_per_second; - if (sample_divisor <= 0) { - sample_divisor = 1; - } // Determine length of video frames (for waveform) int64_t reader_video_length = source_reader->info.video_length; @@ -276,16 +342,21 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r return data; } - const bool known_duration = reader_duration > 0.0f && reader_video_length > 0; - int total_samples = known_duration ? static_cast(std::ceil(reader_duration * num_per_second)) : 0; - if (source_reader->info.channels == 0) { + const int reader_channels = source_reader->info.channels; + if (reader_channels <= 0) { return data; } - if (channel != -1 && (channel < 0 || channel >= source_reader->info.channels)) { + if (channel != -1 && (channel < 0 || channel >= reader_channels)) { return data; } + // Per-channel envelopes describe every source channel, so they need all channels + per_channel = per_channel && channel == -1; + + const bool known_duration = reader_duration > 0.0f && reader_video_length > 0; + const int total_samples = known_duration ? static_cast(std::ceil(reader_duration * num_per_second)) : 0; + // Resize and clear audio buffers when duration is known. Some audio-only // formats, especially raw FLAC, may not expose a container duration even // though frames are readable, so those are appended dynamically below. @@ -296,104 +367,146 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r data.max_samples.reserve(num_per_second); data.rms_samples.reserve(num_per_second); } + if (per_channel) { + data.channel_max_samples.assign(reader_channels, std::vector(known_duration ? total_samples : 0, 0.0f)); + data.channel_rms_samples.assign(reader_channels, std::vector(known_duration ? total_samples : 0, 0.0f)); + } + + // Channels which contribute to the requested envelope + std::vector selected_channels; + for (int channel_index = 0; channel_index < reader_channels; channel_index++) { + if (channel == -1 || channel == channel_index) { + selected_channels.push_back(channel_index); + } + } + + // Per-channel accumulators for the current chunk + std::vector chunk_max(reader_channels, 0.0f); + std::vector chunk_squared_sum(reader_channels, 0.0); + std::vector channel_samples(reader_channels, nullptr); + + // Chunk k covers source samples [boundary(k - 1), boundary(k)). Boundaries are rounded + // from the exact rational position, so rates like 44100 / 200 = 220.5 never drift. + auto chunk_boundary = [&](int64_t chunk_index) -> int64_t { + return ((chunk_index + 1) * sample_rate + num_per_second / 2) / num_per_second; + }; int extracted_index = 0; - int sample_index = 0; + int64_t samples_read = 0; + int64_t chunk_sample_count = 0; + int64_t next_boundary = std::max(1, chunk_boundary(0)); float samples_max = 0.0f; - float chunk_max = 0.0f; - double chunk_squared_sum = 0.0; - int channel_count = (channel == -1) ? source_reader->info.channels : 1; - std::vector channels(source_reader->info.channels, nullptr); const double frame_rate = fps_value > 0.0 ? fps_value : 30.0; const int64_t max_unknown_frames = static_cast(std::ceil(frame_rate * 60.0 * 60.0 * 12.0)); int empty_audio_frames = 0; - auto append_sample = [&](float max_sample, float rms_sample) { - if (known_duration) { - if (extracted_index >= total_samples) { - return; - } - data.max_samples[extracted_index] = max_sample; - data.rms_samples[extracted_index] = rms_sample; - } else { - data.max_samples.push_back(max_sample); - data.rms_samples.push_back(rms_sample); - } - samples_max = std::max(samples_max, max_sample); - extracted_index++; + auto has_room = [&]() { + return !known_duration || extracted_index < total_samples; }; - try { - for (int64_t f = 1; - (known_duration ? f <= reader_video_length && extracted_index < total_samples : f <= max_unknown_frames); - f++) { - std::shared_ptr frame = get_frame_with_retry(f); - if (!known_duration && frame && frame->number < f) { - break; - } - int sample_count = frame->GetAudioSamplesCount(); - if (sample_count <= 0) { - if (!known_duration && extracted_index > 0 && ++empty_audio_frames >= 3) { - break; - } - continue; + // Store the current chunk as one envelope sample, then reset the accumulators + auto emit_chunk = [&]() { + if (chunk_sample_count <= 0) { + return; + } + if (has_room()) { + float mixed_max = 0.0f; + double mixed_squared_sum = 0.0; + for (int channel_index : selected_channels) { + mixed_max = std::max(mixed_max, chunk_max[channel_index]); + mixed_squared_sum += chunk_squared_sum[channel_index]; } - empty_audio_frames = 0; - - for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) { - if (channel == channel_index || channel == -1) { - channels[channel_index] = frame->GetAudioSamples(channel_index); - } + const double mixed_count = static_cast(chunk_sample_count) * static_cast(selected_channels.size()); + const float mixed_rms = static_cast(std::sqrt(mixed_squared_sum / mixed_count)); + + if (known_duration) { + data.max_samples[extracted_index] = mixed_max; + data.rms_samples[extracted_index] = mixed_rms; + } else { + data.max_samples.push_back(mixed_max); + data.rms_samples.push_back(mixed_rms); } - - for (int s = 0; s < sample_count; s++) { - for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) { - if (channel == channel_index || channel == -1) { - float *samples = channels[channel_index]; - if (!samples) { - continue; - } - float abs_sample = std::abs(samples[s]); - chunk_squared_sum += static_cast(samples[s]) * static_cast(samples[s]); - chunk_max = std::max(chunk_max, abs_sample); + if (per_channel) { + for (int channel_index = 0; channel_index < reader_channels; channel_index++) { + const float channel_rms = static_cast(std::sqrt( + chunk_squared_sum[channel_index] / static_cast(chunk_sample_count))); + if (known_duration) { + data.channel_max_samples[channel_index][extracted_index] = chunk_max[channel_index]; + data.channel_rms_samples[channel_index][extracted_index] = channel_rms; + } else { + data.channel_max_samples[channel_index].push_back(chunk_max[channel_index]); + data.channel_rms_samples[channel_index].push_back(channel_rms); } } + } + samples_max = std::max(samples_max, mixed_max); + extracted_index++; + } - sample_index += 1; + std::fill(chunk_max.begin(), chunk_max.end(), 0.0f); + std::fill(chunk_squared_sum.begin(), chunk_squared_sum.end(), 0.0); + chunk_sample_count = 0; + next_boundary = std::max(samples_read + 1, chunk_boundary(extracted_index)); + }; - if (sample_index % sample_divisor == 0) { - float avg_squared_sum = 0.0f; - if (channel_count > 0) { - avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_divisor * channel_count)); - } + for (int64_t f = 1; + (known_duration ? f <= reader_video_length && extracted_index < total_samples : f <= max_unknown_frames); + f++) { + std::shared_ptr frame = get_frame_with_retry(f); + if (!frame) { + break; + } + if (!known_duration && frame->number < f) { + break; + } + const int sample_count = frame->GetAudioSamplesCount(); + if (sample_count <= 0) { + if (!known_duration && extracted_index > 0 && ++empty_audio_frames >= 3) { + break; + } + continue; + } + empty_audio_frames = 0; - if (!known_duration || extracted_index < total_samples) { - append_sample(chunk_max, std::sqrt(avg_squared_sum)); - } + const int frame_channels = frame->GetAudioChannelsCount(); + for (int channel_index : selected_channels) { + channel_samples[channel_index] = channel_index < frame_channels ? frame->GetAudioSamples(channel_index) : nullptr; + } - sample_index = 0; - chunk_max = 0.0f; - chunk_squared_sum = 0.0; + int64_t offset = 0; + while (offset < sample_count) { + const int64_t span = std::min(sample_count - offset, next_boundary - samples_read); + for (int channel_index : selected_channels) { + const float* samples = channel_samples[channel_index]; + if (!samples) { + continue; + } + float span_max = chunk_max[channel_index]; + double span_squared_sum = chunk_squared_sum[channel_index]; + for (int64_t s = offset; s < offset + span; s++) { + const float value = samples[s]; + span_max = std::max(span_max, std::abs(value)); + span_squared_sum += static_cast(value) * static_cast(value); + } + chunk_max[channel_index] = span_max; + chunk_squared_sum[channel_index] = span_squared_sum; + } + offset += span; + samples_read += span; + chunk_sample_count += span; - if (known_duration && extracted_index >= total_samples) { - break; - } + if (samples_read >= next_boundary) { + emit_chunk(); + if (!has_room()) { + break; } } } - } catch (...) { - throw; } - if (sample_index > 0 && (!known_duration || extracted_index < total_samples)) { - float avg_squared_sum = 0.0f; - if (channel_count > 0) { - avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_index * channel_count)); - } - - append_sample(chunk_max, std::sqrt(avg_squared_sum)); - } + // Keep the trailing partial chunk (end of the media) + emit_chunk(); if (normalize && samples_max > 0.0f) { float scale = 1.0f / samples_max; diff --git a/src/AudioWaveformer.h b/src/AudioWaveformer.h index 73ed9566f..73c591fa0 100644 --- a/src/AudioWaveformer.h +++ b/src/AudioWaveformer.h @@ -35,6 +35,10 @@ namespace openshot { std::vector max_samples; std::vector rms_samples; + /// Optional per-channel envelopes, filled by ExtractChannelSamples() + std::vector> channel_max_samples; + std::vector> channel_rms_samples; + /// Resize both datasets void resize(int total_samples) { max_samples.resize(total_samples); @@ -47,20 +51,30 @@ namespace openshot { std::fill(rms_samples.begin(), rms_samples.end(), 0); } - /// Scale # of values by some factor + /// Scale # of values by some factor (including any per-channel envelopes) void scale(int total_samples, float factor) { for (auto s = 0; s < total_samples; s++) { max_samples[s] *= factor; rms_samples[s] *= factor; } + for (auto& channel : channel_max_samples) { + for (auto& value : channel) value *= factor; + } + for (auto& channel : channel_rms_samples) { + for (auto& value : channel) value *= factor; + } } - /// Clear and free memory of both datasets + /// Clear and free memory of all datasets void clear() { max_samples.clear(); max_samples.shrink_to_fit(); rms_samples.clear(); rms_samples.shrink_to_fit(); + channel_max_samples.clear(); + channel_max_samples.shrink_to_fit(); + channel_rms_samples.clear(); + channel_rms_samples.shrink_to_fit(); } /// Return a vector of vectors (containing both datasets) @@ -70,6 +84,22 @@ namespace openshot { output.push_back(rms_samples); return output; } + + /// Number of per-channel envelopes available + int channel_count() const { + return static_cast(channel_max_samples.size()); + } + + /// Return the max and RMS envelopes of one channel (empty when unavailable) + std::vector> channel_vectors(int channel) const { + std::vector> output(2); + if (channel >= 0 && channel < channel_count() + && channel < static_cast(channel_rms_samples.size())) { + output[0] = channel_max_samples[channel]; + output[1] = channel_rms_samples[channel]; + } + return output; + } }; /** @@ -100,6 +130,16 @@ namespace openshot { /// @brief Extract audio samples from a media file path (audio-only fast path) AudioWaveformData ExtractSamples(const std::string& path, int channel, int num_per_second, bool normalize); + /// @brief Extract the mixed envelope and every channel's envelope in a single decode pass + /// + /// max_samples / rms_samples hold the all-channel envelope (the same values as + /// ExtractSamples(-1, ...)), and channel_max_samples / channel_rms_samples hold one + /// envelope per source channel (the same values as ExtractSamples(channel, ...)). + /// Clip keyframes are NOT applied, since per-channel envelopes describe the source media. + /// @param num_per_second How many samples per second to return + /// @param normalize Scale all envelopes by the largest mixed peak, so it becomes 1.0 + AudioWaveformData ExtractChannelSamples(int num_per_second, bool normalize); + /// @brief Apply time and volume keyframes to an existing waveform data set AudioWaveformData ApplyKeyframes(const AudioWaveformData& base, const openshot::Keyframe* time_keyframe, @@ -111,6 +151,17 @@ namespace openshot { int channel, bool normalize); + /// @brief Apply time and volume keyframes to existing max / RMS envelopes + /// + /// Keyframes are evaluated in the project-frame domain (1-based frames). Pass nullptr for + /// either keyframe to skip it. Time curves with fewer than 2 points are ignored. + AudioWaveformData ApplyKeyframes(const std::vector& max_samples, + const std::vector& rms_samples, + const openshot::Keyframe* time_keyframe, + const openshot::Keyframe* volume_keyframe, + const openshot::Fraction& project_fps, + int num_per_second); + /// @brief Convenience: extract then apply keyframes in one step from a file path AudioWaveformData ExtractSamples(const std::string& path, const openshot::Keyframe* time_keyframe, @@ -124,7 +175,7 @@ namespace openshot { ~AudioWaveformer(); private: - AudioWaveformData ExtractSamplesFromReader(openshot::ReaderBase* source_reader, int channel, int num_per_second, bool normalize); + AudioWaveformData ExtractSamplesFromReader(openshot::ReaderBase* source_reader, int channel, int num_per_second, bool normalize, bool per_channel = false); openshot::ReaderBase* ResolveSourceReader(openshot::ReaderBase* source_reader); openshot::Fraction ResolveSourceFPS(openshot::ReaderBase* source_reader); openshot::ReaderBase* ResolveWaveformReader(); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 87867c6aa..eb3db458a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -60,6 +60,7 @@ set(OPENSHOT_SOURCES ChunkReader.cpp ChunkWriter.cpp Color.cpp + ColorManagement.cpp Clip.cpp ClipBase.cpp Coordinate.cpp @@ -185,6 +186,9 @@ set(CMAKE_MACOSX_RPATH 0) ############### CREATE LIBRARY ################# # Create shared openshot library add_library(openshot SHARED) +if(ENABLE_GPU_FRAME) + target_compile_definitions(openshot PUBLIC OPENSHOT_ENABLE_GPU_FRAME=1) +endif() target_sources(openshot PRIVATE ${OPENSHOT_SOURCES} @@ -237,7 +241,7 @@ add_feature_info("Wayland screen capture" OPENSHOT_WAYLAND_CAPTURE "Use the opti # Find JUCE-based openshot Audio libraries if(NOT TARGET OpenShot::Audio) # Only load if necessary (not for integrated builds) - find_package(OpenShotAudio 1.0.2 REQUIRED) + find_package(OpenShotAudio 1.1.0 REQUIRED) endif() target_link_libraries(openshot PUBLIC OpenShot::Audio) @@ -433,6 +437,8 @@ message(STATUS "Found Qt${QT_VERSION_MAJOR} ${Qt${QT_VERSION_MAJOR}Core_VERSION_ set(QT_VERSION_STR ${Qt${QT_VERSION_MAJOR}Core_VERSION_STRING} CACHE STRING "Qt version linked with" FORCE) mark_as_advanced(QT_VERSION_STR) +target_sources(openshot PRIVATE GpuMetrics.cpp) + ################### FFMPEG ##################### # Find FFmpeg libraries (used for video encoding / decoding) find_package(FFmpeg REQUIRED @@ -441,6 +447,54 @@ find_package(FFmpeg REQUIRED ) set(all_comps avcodec avdevice avformat avutil swscale) +if(ENABLE_GPU_FRAME) + find_package(FFmpeg REQUIRED COMPONENTS avfilter) + list(APPEND all_comps avfilter) + target_sources(openshot PRIVATE VulkanFrameImage.cpp) + target_sources(openshot PRIVATE VulkanCompositor.cpp) + target_sources(openshot PRIVATE GpuDevices.cpp VulkanLoader.cpp) + if(APPLE) + enable_language(OBJCXX) + target_sources(openshot PRIVATE MacVulkanInterop.mm MacVulkanInterop.h Qt/VulkanOpenGLPresenterMac.mm Qt/VulkanOpenGLPresenter.h) + set_property(TARGET openshot PROPERTY OBJCXX_STANDARD 17) + target_link_libraries(openshot PRIVATE "-framework Foundation" "-framework CoreFoundation" objc "-framework Metal" "-framework CoreVideo" "-framework IOSurface" "-framework OpenGL") + else() + target_sources(openshot PRIVATE Qt/VulkanOpenGLPresenter.cpp Qt/VulkanOpenGLPresenter.h) + endif() + find_package(Vulkan REQUIRED) + find_program(GLSLANG_VALIDATOR glslangValidator) + if(NOT GLSLANG_VALIDATOR) + message(FATAL_ERROR "ENABLE_GPU_FRAME requires glslangValidator") + endif() + # Embed SPIR-V in the library so installed builds need no shader files. + set(VULKAN_SHADER_HEADERS) + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/vulkan") + foreach(shader affine encode_nv12) + set(shader_header "${CMAKE_CURRENT_BINARY_DIR}/vulkan/${shader}.comp.h") + add_custom_command(OUTPUT "${shader_header}" + COMMAND "${GLSLANG_VALIDATOR}" -V -S comp --vn "openshot_${shader}_spv" + -o "${shader_header}" "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/${shader}.comp" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/${shader}.comp" VERBATIM) + list(APPEND VULKAN_SHADER_HEADERS "${shader_header}") + endforeach() + add_custom_target(openshot-vulkan-shaders DEPENDS ${VULKAN_SHADER_HEADERS}) + add_dependencies(openshot openshot-vulkan-shaders) + target_include_directories(openshot PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/vulkan") + # Vulkan is loaded at runtime (VulkanLoader.cpp), so OpenShot starts without it. + # macOS links the MoltenVK it bundles; nothing to load there. + target_compile_definitions(openshot PRIVATE VK_NO_PROTOTYPES) + if(APPLE) + target_link_libraries(openshot PRIVATE Vulkan::Vulkan) + else() + target_include_directories(openshot PRIVATE ${Vulkan_INCLUDE_DIRS}) + target_link_libraries(openshot PRIVATE ${CMAKE_DL_LIBS}) + endif() + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + pkg_check_modules(VULKAN_FRAME_PC QUIET vulkan) + target_include_directories(openshot PRIVATE ${VULKAN_FRAME_PC_INCLUDE_DIRS}) + endif() +endif() set(version_comps avcodec avformat avutil) # Pick a resampler. Prefer swresample if possible diff --git a/src/CacheMemory.cpp b/src/CacheMemory.cpp index a89c9f7ee..1b7758e1f 100644 --- a/src/CacheMemory.cpp +++ b/src/CacheMemory.cpp @@ -255,7 +255,19 @@ void CacheMemory::CleanUp() // Create a scoped lock, to protect the cache from multiple threads const std::lock_guard lock(*cacheMutex); - while (GetBytes() > max_bytes && frame_numbers.size() > 20) + size_t retained_frames = 20; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // A CPU-oriented twenty-frame floor can pin gigabytes of GPU memory + // across reader, mapper and timeline caches. Honor byte budgets for + // caches holding hardware surfaces while retaining the newest frame. + for (const auto& entry : frames) { + if (entry.second->GetDeferredImage()) { + retained_frames = 1; + break; + } + } +#endif + while (GetBytes() > max_bytes && frame_numbers.size() > retained_frames) { // Get the oldest frame number. int64_t frame_to_remove = frame_numbers.back(); diff --git a/src/Clip.cpp b/src/Clip.cpp index 0fc043d87..712b050b5 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -10,6 +10,7 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" #include #include "Clip.h" @@ -17,6 +18,10 @@ #include "Exceptions.h" #include "FFmpegReader.h" #include "FrameMapper.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#endif #include "QtImageReader.h" #include "QtTextReader.h" #include "ChunkReader.h" @@ -26,6 +31,7 @@ #include "effects/AudioVisualization.h" #include +#include #include #include #include @@ -503,12 +509,27 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun // Get time mapped frame object (used to increase speed, change direction, etc...) apply_timemapping(frame); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Audio-only/disabled-video layers have nothing to paint in the basic + // pipeline. Keep their audio, without manufacturing a CPU canvas. + if (options && (!reader->info.has_video || has_video.GetInt(frame->number) == 0) && + !Waveform() && effects.empty() && + (!timeline || static_cast(timeline)->Effects().empty())) + return frame; +#endif + // Apply waveform image (if any) apply_waveform(frame, timeline_size); // Apply effects BEFORE applying keyframes (if any local or global effects are used) apply_effects(frame, timeline_frame_number, options, true); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Timeline layers only need the composited result, so skip the separate transform pass + if (options && background_frame && apply_vulkan_layer(frame, background_frame, timeline_size)) + return frame; +#endif + // Apply keyframe / transforms to current clip image apply_keyframes(frame, timeline_size); @@ -773,6 +794,7 @@ std::shared_ptr Clip::GetOrCreateFrame(int64_t number, bool enable_time) "number", number, "clip_frame_number", clip_frame_number); // Attempt to get a frame (but this could fail if a reader has just been closed) + if (reader->info.has_single_image) gpu_metrics::Record(gpu_metrics::ImageSource); auto reader_frame = reader->GetFrame(clip_frame_number); if (reader_frame) { // Override frame # (due to time-mapping might change it) @@ -1352,6 +1374,37 @@ void Clip::RemoveEffect(EffectBase* effect) void Clip::apply_background(std::shared_ptr frame, std::shared_ptr background_frame, bool update_frame_image) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Audio-only frames contribute audio in Timeline::add_layer, but have no + // image to composite. Do not materialize the GPU canvas for an empty layer. + if (!frame->has_image_data) { + if (update_frame_image) { + if (auto background = background_frame->GetDeferredImage()) frame->AddDeferredImage(background); + else frame->AddImage(background_frame->GetImage()); + } + return; + } + if (composite == COMPOSITE_SOURCE_OVER) { + auto source = frame->GetDeferredImage(); + auto background = background_frame->GetDeferredImage(); + QColor clear; + if (std::dynamic_pointer_cast(source) && + (std::dynamic_pointer_cast(background) || background_frame->GetBlankColor(clear))) { + try { + auto result = RenderVulkanImage(source, background, background_frame->GetWidth(), + background_frame->GetHeight(), QTransform(), 1.0, clear); + gpu_metrics::Record(gpu_metrics::RenderGpu); + background_frame->AddDeferredImage(result); + if (update_frame_image) frame->AddDeferredImage(result); + return; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan composite CPU fallback: ") + error.what()); + } + } + } +#endif + gpu_metrics::Record(gpu_metrics::RenderCpu); + gpu_metrics::Reason gpu_reason("Composite"); // Add background canvas std::shared_ptr background_canvas = background_frame->GetImage(); QPainter painter(background_canvas.get()); @@ -1374,9 +1427,13 @@ void Clip::apply_effects(std::shared_ptr frame, int64_t timeline_frame_nu { // Apply the effect to this frame if (effect->info.apply_before_clip && before_keyframes) { + gpu_metrics::Reason gpu_reason("Effect"); effect->ProcessFrame(frame, frame->number); + if (effect->info.has_video) gpu_metrics::RecordRender(*frame); } else if (!effect->info.apply_before_clip && !before_keyframes) { + gpu_metrics::Reason gpu_reason("Effect"); effect->ProcessFrame(frame, frame->number); + if (effect->info.has_video) gpu_metrics::RecordRender(*frame); } } @@ -1394,6 +1451,58 @@ bool Clip::isNear(double a, double b) return fabs(a - b) < 0.000001; } +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +std::shared_ptr VulkanSource(const std::shared_ptr& frame) { + if (!VulkanRenderingEnabled()) return {}; + auto source = std::dynamic_pointer_cast(frame->GetDeferredImage()); + if (source || !frame->has_image_data) return source; + try { + auto uploaded = UploadVulkanImage(*frame->GetImage()); + frame->AddDeferredImage(uploaded); + return std::dynamic_pointer_cast(uploaded); + } catch (const std::exception& error) { + // Log the first failure and then every 100th, so repeats stay visible + // without flooding the log; the total is in VulkanUploadFailureCount(). + RecordVulkanUploadFailure(); + const auto failures = VulkanUploadFailureCount(); + if (failures == 1 || failures % 100 == 0) + Logger::Instance()->Log("GPU image upload unavailable; using CPU (" + std::to_string(failures) + + " failures): " + error.what(), Logger::LevelWarning); + return {}; + } +} +} +bool Clip::apply_vulkan_layer(std::shared_ptr frame, std::shared_ptr background_frame, QSize timeline_size) +{ + if (!frame->has_image_data || composite != COMPOSITE_SOURCE_OVER || + corner_radius.GetValue(frame->number) != 0.0 || display != FRAME_DISPLAY_NONE) + return false; + // Effects applied after keyframes (and timeline transitions/masks) need the transformed layer + for (auto effect : effects) + if (!effect->info.apply_before_clip) + return false; + if (timeline && !static_cast(timeline)->Effects().empty()) + return false; + + auto source = VulkanSource(frame); + auto background = background_frame->GetDeferredImage(); + QColor clear; + if (!source || !(std::dynamic_pointer_cast(background) || background_frame->GetBlankColor(clear))) + return false; + try { + background_frame->AddDeferredImage(RenderVulkanImage(source, background, timeline_size.width(), + timeline_size.height(), get_transform(frame, timeline_size.width(), timeline_size.height()), + alpha.GetValue(frame->number), clear)); + gpu_metrics::Record(gpu_metrics::RenderGpu); + return true; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan layer CPU fallback: ") + error.what()); + return false; + } +} +#endif + // Apply keyframes to the source frame (if any) void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { // Skip out if video was disabled or only an audio frame (no visualisation in use) @@ -1402,6 +1511,25 @@ void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { return; } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (composite == COMPOSITE_SOURCE_OVER && + corner_radius.GetValue(frame->number) == 0.0 && display == FRAME_DISPLAY_NONE) { + if (auto source = VulkanSource(frame)) { + try { + frame->AddDeferredImage(RenderVulkanImage(source, nullptr, timeline_size.width(), + timeline_size.height(), get_transform(frame, timeline_size.width(), timeline_size.height()), + alpha.GetValue(frame->number))); + gpu_metrics::Record(gpu_metrics::RenderGpu); + return; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan transform CPU fallback: ") + error.what()); + } + } + } +#endif + + gpu_metrics::Record(gpu_metrics::RenderCpu); + gpu_metrics::Reason gpu_reason("Transform"); // Get image from clip, and create transparent background image std::shared_ptr source_image = frame->GetImage(); std::shared_ptr background_canvas = std::make_shared(timeline_size.width(), @@ -1557,8 +1685,12 @@ QSize Clip::scale_size(QSize source_size, ScaleType source_scale, int target_wid // Get QTransform from keyframes QTransform Clip::get_transform(std::shared_ptr frame, int width, int height) { - // Get image from clip - std::shared_ptr source_image = frame->GetImage(); + // Transform geometry does not require CPU pixels for a deferred surface. +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const QSize source_dimensions(frame->GetWidth(), frame->GetHeight()); +#else + const QSize source_dimensions = frame->GetImage()->size(); +#endif const double margin_value = std::max(0.0, std::min(0.5, margin.GetValue(frame->number))); const double margin_pixels = margin_value * std::min(width, height); @@ -1568,7 +1700,7 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig const double layout_height = std::max(1.0, height - (margin_pixels * 2.0)); /* RESIZE SOURCE IMAGE - based on scale type */ - QSize source_size = scale_size(source_image->size(), scale, layout_width, layout_height); + QSize source_size = scale_size(source_dimensions, scale, layout_width, layout_height); // Text rasterizes extra detail for a scaled-up SCALE_NONE clip; draw it at its layout size if (scale == SCALE_NONE) { @@ -1758,8 +1890,8 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig transform.translate(-origin_x_offset,-origin_y_offset); } // SCALE CLIP (if needed) - float source_width_scale = (float(source_size.width()) / float(source_image->width())) * sx; - float source_height_scale = (float(source_size.height()) / float(source_image->height())) * sy; + float source_width_scale = (float(source_size.width()) / float(source_dimensions.width())) * sx; + float source_height_scale = (float(source_size.height()) / float(source_dimensions.height())) * sy; if (!isNear(source_width_scale, 1.0) || !isNear(source_height_scale, 1.0)) { transform.scale(source_width_scale, source_height_scale); } diff --git a/src/Clip.h b/src/Clip.h index 072cb0c4e..c1150c3a7 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -145,6 +145,12 @@ namespace openshot { /// Apply keyframes to an openshot::Frame and use an existing background frame (if any) void apply_keyframes(std::shared_ptr frame, QSize timeline_size); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + /// Transform and composite onto the background in one Vulkan pass. Returns false + /// when an effect needs the transformed layer by itself, or the pass is unavailable. + bool apply_vulkan_layer(std::shared_ptr frame, std::shared_ptr background_frame, QSize timeline_size); +#endif + /// Apply waveform image to an openshot::Frame and use an existing background frame (if any) void apply_waveform(std::shared_ptr frame, QSize timeline_size); diff --git a/src/ColorManagement.cpp b/src/ColorManagement.cpp new file mode 100644 index 000000000..cfb441627 --- /dev/null +++ b/src/ColorManagement.cpp @@ -0,0 +1,255 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "ColorManagement.h" + +#include +#include +#include + +extern "C" { +#include +#include +#include +} + +namespace openshot { +namespace color { +namespace { + +// BT.2100 PQ (SMPTE ST 2084) +constexpr double kPqM1 = 2610.0 / 16384.0, kPqM2 = 2523.0 / 4096.0 * 128.0; +constexpr double kPqC1 = 3424.0 / 4096.0, kPqC2 = 2413.0 / 4096.0 * 32.0, kPqC3 = 2392.0 / 4096.0 * 32.0; +// BT.2100 HLG +constexpr double kHlgA = 0.17883277, kHlgB = 0.28466892, kHlgC = 0.55991073; +constexpr double kHlgGamma = 1.2; // system gamma at the nominal 1000 nit peak +// SDR display EOTF (BT.1886), as BT.2408 uses to place SDR within HDR. +constexpr double kSdrGamma = 2.4; + +// Linear BT.2020 RGB to linear BT.709 RGB (BT.2087). +constexpr float kBt2020To709[9] = { + 1.6604910f, -0.5876411f, -0.0728499f, + -0.1245505f, 1.1328999f, -0.0083494f, + -0.0181508f, -0.1005789f, 1.1187297f}; +constexpr float kLuma2020[3] = {0.2627f, 0.6780f, 0.0593f}; +constexpr float kLuma709[3] = {0.2126f, 0.7152f, 0.0722f}; + +// Lookups stay small enough for L1 cache. 12-bit signal indices exceed the +// 10-bit sources' precision (and PQ's 12-bit visibility threshold). +constexpr int kSignalBits = 12; +constexpr int kSignalSize = 1 << kSignalBits; +// Encode lookup indexed by the float's exponent and top 10 mantissa bits, +// covering linear 2^-24 (below half an 8-bit code) to 1.0. +constexpr uint32_t kEncodeShift = 13; +constexpr uint32_t kEncodeBase = (127u - 24u) << (23 - kEncodeShift); +constexpr int kEncodeSize = int((127u << (23 - kEncodeShift)) - kEncodeBase) + 1; + +// Tone scale lookup indexed by the brightest channel's exponent and top 7 +// mantissa bits (under 0.4% apart), from the knee to 64x reference white. +constexpr uint32_t kToneShift = 16; +constexpr uint32_t kToneBase = (127u - 1u) << (23 - kToneShift); +constexpr int kToneSize = int(((127u + 6u) << (23 - kToneShift)) - kToneBase); + +inline uint32_t FloatBits(float value) { uint32_t bits; std::memcpy(&bits, &value, sizeof(bits)); return bits; } +inline int ToneIndex(float brightest) { + const int index = int(FloatBits(brightest) >> kToneShift) - int(kToneBase); + return std::clamp(index, 0, kToneSize - 1); +} +inline int EncodeIndex(float linear) { + const int index = int(FloatBits(linear) >> kEncodeShift) - int(kEncodeBase); + return std::clamp(index, 0, kEncodeSize - 1); +} + +double PqToNits(double e) { + const double p = std::pow(std::clamp(e, 0.0, 1.0), 1.0 / kPqM2); + return 10000.0 * std::pow(std::max(p - kPqC1, 0.0) / (kPqC2 - kPqC3 * p), 1.0 / kPqM1); +} +double HlgToScene(double e) { + e = std::clamp(e, 0.0, 1.0); + return e <= 0.5 ? e * e / 3.0 : (std::exp((e - kHlgC) / kHlgA) + kHlgB) / 12.0; +} +bool IsBt2020Matrix(AVColorSpace matrix) { + return matrix == AVCOL_SPC_BT2020_NCL || matrix == AVCOL_SPC_BT2020_CL; +} +} // namespace + +AVColorSpace DecodeMatrix(AVColorSpace tagged, int width, int height) { + switch (tagged) { + case AVCOL_SPC_BT709: + case AVCOL_SPC_BT470BG: + case AVCOL_SPC_SMPTE170M: + case AVCOL_SPC_SMPTE240M: + case AVCOL_SPC_FCC: + case AVCOL_SPC_BT2020_NCL: + case AVCOL_SPC_BT2020_CL: + return tagged; + default: + // Untagged files keep BT.601 at every size: earlier OpenShot exports + // and older FFmpeg encodes used it without tagging, and + // cameras and phones tag their files. + (void) width; (void) height; + return AVCOL_SPC_SMPTE170M; + } +} + +AVColorSpace EncodeMatrix(int width, int height) { + // Tagged; this is also what players assume for untagged files. + return (width >= 1280 || height > 576) ? AVCOL_SPC_BT709 : AVCOL_SPC_SMPTE170M; +} + +int SwsColorspace(AVColorSpace matrix) { + switch (matrix) { + case AVCOL_SPC_BT709: return SWS_CS_ITU709; + case AVCOL_SPC_FCC: return SWS_CS_FCC; + case AVCOL_SPC_SMPTE240M: return SWS_CS_SMPTE240M; + case AVCOL_SPC_BT2020_NCL: + case AVCOL_SPC_BT2020_CL: return SWS_CS_BT2020; + default: return SWS_CS_ITU601; + } +} + +float SourceColor::RelativePeak() const { + if (transfer == Transfer::Sdr) return 1.0f; + const double nits = transfer == Transfer::HLG ? kDefaultPeakNits : peak_nits; + return float(std::clamp(nits / kReferenceWhiteNits, 1.0, 10000.0 / kReferenceWhiteNits)); +} + +double FramePeakNits(const AVFrame* frame) { + if (!frame) return 0.0; + if (const AVFrameSideData* light = av_frame_get_side_data(frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL)) { + const auto* cll = reinterpret_cast(light->data); + if (cll->MaxCLL > 0) return double(cll->MaxCLL); + } + if (const AVFrameSideData* display = av_frame_get_side_data(frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA)) { + const auto* mastering = reinterpret_cast(display->data); + if (mastering->has_luminance && mastering->max_luminance.den > 0) { + const double nits = av_q2d(mastering->max_luminance); + if (nits > 0.0) return nits; + } + } + return 0.0; +} + +SourceColor DescribeFrame(const AVFrame* frame, int width, int height, double peak_nits) { + SourceColor result; + const AVColorSpace tagged = frame ? frame->colorspace : AVCOL_SPC_UNSPECIFIED; + result.matrix = DecodeMatrix(tagged, width, height); + result.full_range = frame && frame->color_range == AVCOL_RANGE_JPEG; + if (frame) { + if (frame->color_trc == AVCOL_TRC_SMPTE2084) result.transfer = Transfer::PQ; + else if (frame->color_trc == AVCOL_TRC_ARIB_STD_B67) result.transfer = Transfer::HLG; + // Unspecified primaries follow a BT.2020 matrix, as players do. + result.bt2020 = frame->color_primaries == AVCOL_PRI_BT2020 || + (frame->color_primaries == AVCOL_PRI_UNSPECIFIED && IsBt2020Matrix(tagged)); + } + if (peak_nits > 0.0) result.peak_nits = peak_nits; + return result; +} + +float ToneMap(float value, float relative_peak) { + if (value <= kToneKnee || relative_peak <= 1.0f) return std::min(value, 1.0f); + // Linear below the knee; above it an extended-Reinhard shoulder with unit + // slope at the knee that reaches 1.0 exactly at the content peak. + const float span = 1.0f - kToneKnee; + const float t = (value - kToneKnee) / span; + const float w = std::max((relative_peak - kToneKnee) / span, 1.0f); + const float shoulder = t * (1.0f + t / (w * w)) / (1.0f + t); + return std::min(kToneKnee + span * shoulder, 1.0f); +} + +SdrConverter::SdrConverter(const SourceColor& source_color) + : source(source_color), peak(source_color.RelativePeak()), linearize(kSignalSize), encode(kEncodeSize) { + for (int i = 0; i < kSignalSize; ++i) { + // Bucket centre of the 16-bit codes sharing this index. + const double e = (i + 0.5) / kSignalSize; + double value; + switch (source.transfer) { + case Transfer::PQ: value = PqToNits(e) / kReferenceWhiteNits; break; + case Transfer::HLG: value = HlgToScene(e); break; + default: value = std::pow(e, kSdrGamma); break; + } + linearize[i] = float(value); + } + if (source.transfer == Transfer::HLG) { + // OOTF: display = peak * Ys^(gamma-1) * scene, relative to reference white. + hlg_gain.resize(kSignalSize + 1); + for (int i = 0; i <= kSignalSize; ++i) + hlg_gain[i] = float(kDefaultPeakNits / kReferenceWhiteNits * std::pow(double(i) / kSignalSize, kHlgGamma - 1.0)); + } + tone_scale.resize(kToneSize); + for (int i = 0; i < kToneSize; ++i) { + const uint32_t bits = ((uint32_t(i) + kToneBase) << kToneShift) | (1u << (kToneShift - 1)); + float brightest; + std::memcpy(&brightest, &bits, sizeof(brightest)); + tone_scale[i] = brightest > kToneKnee ? ToneMap(brightest, peak) / brightest : 1.0f; + } + for (int i = 0; i < kEncodeSize; ++i) { + // Middle of the floats sharing this index. + const uint32_t bits = ((uint32_t(i) + kEncodeBase) << kEncodeShift) | (1u << (kEncodeShift - 1)); + float linear; + std::memcpy(&linear, &bits, sizeof(linear)); + encode[i] = uint8_t(std::lround(255.0 * std::pow(std::min(double(linear), 1.0), 1.0 / kSdrGamma))); + } +} + +void SdrConverter::ToSdr(float rgb[3]) const { + if (source.bt2020) { + const float r = rgb[0], g = rgb[1], b = rgb[2]; + rgb[0] = kBt2020To709[0] * r + kBt2020To709[1] * g + kBt2020To709[2] * b; + rgb[1] = kBt2020To709[3] * r + kBt2020To709[4] * g + kBt2020To709[5] * b; + rgb[2] = kBt2020To709[6] * r + kBt2020To709[7] * g + kBt2020To709[8] * b; + } + // Out-of-gamut colors clip per channel. + for (int c = 0; c < 3; ++c) rgb[c] = std::max(rgb[c], 0.0f); + const float brightest = std::max(rgb[0], std::max(rgb[1], rgb[2])); + if (brightest > kToneKnee) { + // Scale all channels by the max-channel curve, preserving hue. + const float scale = ToneMap(brightest, peak) / brightest; + for (int c = 0; c < 3; ++c) rgb[c] *= scale; + } + for (int c = 0; c < 3; ++c) rgb[c] = std::min(rgb[c], 1.0f); +} + +template +void SdrConverter::ConvertRows(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes, + int width, int height) const { + const float* luma = Gamut ? kLuma2020 : kLuma709; + constexpr int shift = 16 - kSignalBits; + #pragma omp parallel for if(int64_t(width) * height >= 65536) schedule(static) + for (int y = 0; y < height; ++y) { + const uint16_t* in = reinterpret_cast(reinterpret_cast(src) + int64_t(y) * src_stride_bytes); + uint8_t* out = dst + int64_t(y) * dst_stride_bytes; + for (int x = 0; x < width; ++x, in += 4, out += 4) { + float r = linearize[in[0] >> shift], g = linearize[in[1] >> shift], b = linearize[in[2] >> shift]; + if (Hlg) { + const float scene_luma = luma[0] * r + luma[1] * g + luma[2] * b; + const float gain = hlg_gain[std::min(kSignalSize, int(scene_luma * kSignalSize + 0.5f))]; + r *= gain; g *= gain; b *= gain; + } + if (Gamut) { + const float r2 = kBt2020To709[0] * r + kBt2020To709[1] * g + kBt2020To709[2] * b; + const float g2 = kBt2020To709[3] * r + kBt2020To709[4] * g + kBt2020To709[5] * b; + const float b2 = kBt2020To709[6] * r + kBt2020To709[7] * g + kBt2020To709[8] * b; + r = r2; g = g2; b = b2; + } + // Clip out-of-gamut colors, then scale by the max-channel tone curve. + r = std::max(r, 0.0f); g = std::max(g, 0.0f); b = std::max(b, 0.0f); + const float scale = tone_scale[ToneIndex(std::max(r, std::max(g, b)))]; + out[0] = encode[EncodeIndex(r * scale)]; + out[1] = encode[EncodeIndex(g * scale)]; + out[2] = encode[EncodeIndex(b * scale)]; + out[3] = uint8_t(in[3] >> 8); + } + } +} + +void SdrConverter::Convert(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes, + int width, int height) const { + const bool hlg = source.transfer == Transfer::HLG; + if (hlg && source.bt2020) ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height); + else if (hlg) ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height); + else if (source.bt2020) ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height); + else ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height); +} + +} // namespace color +} // namespace openshot diff --git a/src/ColorManagement.h b/src/ColorManagement.h new file mode 100644 index 000000000..dcd0a63f2 --- /dev/null +++ b/src/ColorManagement.h @@ -0,0 +1,87 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_COLOR_MANAGEMENT_H +#define OPENSHOT_COLOR_MANAGEMENT_H + +#include +#include + +extern "C" { +#include +} +struct AVFrame; + +namespace openshot { +namespace color { + +/// SDR graphics/reference white in HDR signals (ITU-R BT.2408). HDR sources +/// map this luminance to SDR 100%. +constexpr double kReferenceWhiteNits = 203.0; +/// Nominal HLG display peak (BT.2100) and the PQ peak assumed without metadata. +constexpr double kDefaultPeakNits = 1000.0; +/// Linear SDR level (1.0 = reference white) above which highlights compress. +/// Must match TONE_KNEE in src/vulkan/affine.comp. +constexpr float kToneKnee = 0.8f; + +enum class Transfer : int { Sdr = 0, PQ = 1, HLG = 2 }; + +/// The YUV matrix to decode with: the tagged matrix, or BT.601 when untagged +/// (how OpenShot exports before 2026 and older FFmpeg encodes were written). +AVColorSpace DecodeMatrix(AVColorSpace tagged, int width, int height); +/// The YUV matrix OpenShot encodes with (and tags) for an output size: +/// BT.709 for HD and larger, BT.601 for SD. +AVColorSpace EncodeMatrix(int width, int height); +/// The swscale coefficient table index (SWS_CS_*) for a matrix. +int SwsColorspace(AVColorSpace matrix); + +/// How a decoded source must be converted into OpenShot's SDR BT.709 RGBA. +struct SourceColor { + AVColorSpace matrix = AVCOL_SPC_BT709; + bool full_range = false; + Transfer transfer = Transfer::Sdr; + bool bt2020 = false; ///< BT.2020 primaries: convert gamut to BT.709 + double peak_nits = kDefaultPeakNits; ///< HDR content peak, for tone mapping + + /// True when transfer or gamut conversion is required (HDR or wide gamut). + bool NeedsConversion() const { return transfer != Transfer::Sdr || bt2020; } + /// Peak in units of reference white (>= 1); 1 means no tone mapping. + float RelativePeak() const; +}; + +/// Describe a frame's color. peak_nits is the content peak from stream +/// metadata (0 = unknown); it is only used for PQ. +SourceColor DescribeFrame(const AVFrame* frame, int width, int height, double peak_nits = 0.0); +/// Content peak from a frame's HDR10 static metadata (MaxCLL, then mastering +/// display maximum), or 0 when absent. +double FramePeakNits(const AVFrame* frame); + +/// CPU conversion of nonlinear R'G'B'A (16-bit, native endian, full range, in +/// the source's primaries and transfer) into SDR BT.709 RGBA8. Lookup tables +/// are built once per SourceColor; Convert is thread-safe. +class SdrConverter { +public: + explicit SdrConverter(const SourceColor& source); + const SourceColor& Source() const { return source; } + void Convert(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes, + int width, int height) const; + /// Convert one linear-light RGB value (1.0 = reference white, source + /// primaries) to SDR display RGB in [0,1]. Exposed for tests. + void ToSdr(float rgb[3]) const; +private: + SourceColor source; + float peak; + std::vector linearize; ///< 12-bit signal -> linear (relative to reference white) + std::vector hlg_gain; ///< HLG OOTF gain by scene luminance + std::vector encode; ///< linear (by float exponent/mantissa) -> BT.1886 8-bit code + std::vector tone_scale; ///< brightest channel -> tone curve scale + template + void ConvertRows(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes, + int width, int height) const; +}; + +/// Tone-map one linear SDR-relative value (max channel) to [0,1]. +float ToneMap(float value, float relative_peak); + +} // namespace color +} // namespace openshot +#endif diff --git a/src/CrashHandler.cpp b/src/CrashHandler.cpp index 153e8ab66..d009d0529 100644 --- a/src/CrashHandler.cpp +++ b/src/CrashHandler.cpp @@ -55,6 +55,23 @@ CrashHandler *CrashHandler::Instance() } #ifdef __MINGW32__ +// GpuDebug: with OPENSHOT_GPU_DEBUG set, save a small crash dump (thread +// stacks only) to %USERPROFILE%\.openshot_qt for debugging test builds. +static void writeMiniDump() +{ + const char* profile = getenv("USERPROFILE"); + if (!getenv("OPENSHOT_GPU_DEBUG") || !profile) + return; + char path[MAX_PATH]; + snprintf(path, sizeof(path), "%s\\.openshot_qt\\openshot-crash-%lu.dmp", profile, GetCurrentProcessId()); + HANDLE file = CreateFileA(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) + return; + if (MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), file, MiniDumpNormal, NULL, NULL, NULL)) + fprintf(stderr, "Crash dump saved: %s\n", path); + CloseHandle(file); +} + // Windows exception handler void CrashHandler::abortHandler(int signum) { @@ -76,6 +93,7 @@ void CrashHandler::abortHandler(int signum) // Dump a stack trace. printStackTrace(stderr, 63); + writeMiniDump(); // Quit exit( signum ); diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index 415e1c9af..863d97a7e 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -26,6 +26,10 @@ #include "FFmpegReader.h" #include "QtImageReader.h" #include "Logger.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME + #include "VulkanCompositor.h" + #include "VulkanFrameImage.h" +#endif #include #include #include @@ -502,6 +506,11 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ return empty_mask; } + return ResolveReaderMaskImage(target_image->width(),target_image->height(),frame_number); +} + +std::shared_ptr EffectBase::ResolveReaderMaskImage(int width,int height,int64_t frame_number) { + if (!mask_reader) return {}; @@ -515,8 +524,8 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ if (mask_reader->info.has_single_image && cached_single_mask_image && - cached_single_mask_width == target_image->width() && - cached_single_mask_height == target_image->height()) { + cached_single_mask_width == width && + cached_single_mask_height == height) { source_mask = cached_single_mask_image; used_cached_scaled = true; } @@ -541,16 +550,54 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ auto scaled_mask = std::make_shared( source_mask->scaled( - target_image->width(), target_image->height(), + width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); if (mask_reader->info.has_single_image) { cached_single_mask_image = scaled_mask; - cached_single_mask_width = target_image->width(); - cached_single_mask_height = target_image->height(); + cached_single_mask_width = width; + cached_single_mask_height = height; } return scaled_mask; } +std::shared_ptr EffectBase::ResolveReaderMaskSurface(int width, int height, int64_t frame_number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (!mask_reader) + return {}; + std::shared_ptr surface; + bool still = true; + #pragma omp critical (open_effect_mask_reader) + { + try { + if (!mask_reader->IsOpen()) + mask_reader->Open(); + // Still images are scaled and uploaded once, then reused. + still = mask_reader->info.has_single_image; + if (!still) { + auto source_frame = mask_reader->GetFrame(MapMaskFrameNumber(frame_number)); + if (source_frame) + surface = source_frame->GetDeferredImage(); + } + } catch (const std::exception& e) { + Logger::Instance()->Log( + std::string("EffectBase::ResolveReaderMaskSurface unable to read mask frame: ") + e.what()); + } + } + if (surface) { + // One GPU pass converts decoder formats to RGBA and scales to the target. + const QTransform scale = QTransform::fromScale(double(width) / surface->GetWidth(), + double(height) / surface->GetHeight()); + return RenderVulkanImage(surface, nullptr, width, height, scale, 1.0, Qt::transparent, true); + } + auto image = ResolveReaderMaskImage(width, height, frame_number); + if (image && !image->isNull()) + return UploadVulkanImage(*image); +#else + (void) width; (void) height; (void) frame_number; +#endif + return {}; +} + std::shared_ptr EffectBase::TrackedObjectMask(std::shared_ptr target_image, int64_t frame_number) const { #ifdef USE_OPENCV if (!target_image || target_image->isNull() || trackedObjects.empty()) @@ -657,29 +704,76 @@ std::shared_ptr EffectBase::ProcessFrame(std::shared_ptrGetImage(); - if (!pre_image || pre_image->isNull()) - return GetFrame(frame, frame_number); + const auto blend_cpu = [&](std::shared_ptr original_image, std::shared_ptr output_frame) { + if (!output_frame) + return output_frame; + auto effected_image = output_frame->GetImage(); + if (!effected_image || effected_image->isNull() || + effected_image->size() != original_image->size()) + return output_frame; - const auto original_image = std::make_shared(pre_image->copy()); - auto output_frame = GetFrame(frame, frame_number); - if (!output_frame) - return output_frame; - auto effected_image = output_frame->GetImage(); - if (!effected_image || effected_image->isNull() || - effected_image->size() != original_image->size()) - return output_frame; + auto mask_image = GetMaskImage(effected_image, frame_number); + if (!mask_image || mask_image->isNull()) + return output_frame; - auto mask_image = GetMaskImage(effected_image, frame_number); - if (!mask_image || mask_image->isNull()) + if (UseCustomMaskBlend(frame_number)) + ApplyCustomMaskBlend(original_image, effected_image, mask_image, frame_number); + else + BlendWithMask(original_image, effected_image, mask_image); return output_frame; + }; + +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // GPU frames keep their input surface, and the mask blend stays on the GPU. + std::shared_ptr original_surface; + const bool drive = UseCustomMaskBlend(frame_number) && MaskDrivesAmount(frame_number); + if (VulkanRenderingEnabled() && RendersOnVulkan() && (!UseCustomMaskBlend(frame_number) || drive)) { + original_surface = frame->GetDeferredImage(); + // Clip images are uploaded as the effect would; the effect reuses this upload. + if (!original_surface && ParentClip() && frame->has_image_data) { + try { + original_surface = UploadVulkanImage(*frame->GetImage()); + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan effect mask upload failed: ") + error.what()); + } + } + } + if (original_surface) { + auto output_frame = GetFrame(frame, frame_number); + if (!output_frame) + return output_frame; + const int width = original_surface->GetWidth(), height = original_surface->GetHeight(); + if (auto effected = output_frame->GetDeferredImage()) { + // A resized result has no matching mask area, as on the CPU. + if (effected->GetWidth() != width || effected->GetHeight() != height) + return output_frame; + try { + // Generated masks only need the target size, not its pixels. + std::shared_ptr mask; + if (ResolveMaskSourceEffect()) { + auto generated = GetMaskImage(std::make_shared(width, height, QImage::Format_RGBA8888_Premultiplied), frame_number); + if (generated && !generated->isNull()) + mask = UploadVulkanImage(*generated); + } else + mask = ResolveReaderMaskSurface(width, height, frame_number); + if (!mask) + return output_frame; + output_frame->AddDeferredImage(BlendMaskVulkanImage(original_surface, effected, mask, mask_invert, drive)); + return output_frame; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan effect mask CPU fallback: ") + error.what()); + } + } + return blend_cpu(std::make_shared(original_surface->Materialize()), output_frame); + } +#endif - if (UseCustomMaskBlend(frame_number)) - ApplyCustomMaskBlend(original_image, effected_image, mask_image, frame_number); - else - BlendWithMask(original_image, effected_image, mask_image); + auto pre_image = frame->GetImage(); + if (!pre_image || pre_image->isNull()) + return GetFrame(frame, frame_number); - return output_frame; + const auto original_image = std::make_shared(pre_image->copy()); + return blend_cpu(original_image, GetFrame(frame, frame_number)); } /// Parent clip object of this reader (which can be unparented and NULL) diff --git a/src/EffectBase.h b/src/EffectBase.h index 9da9c7149..143c417c6 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -25,6 +25,7 @@ namespace openshot { + class FrameImage; // GPU surface; only produced in Vulkan builds class ReaderBase; /** @@ -93,6 +94,13 @@ namespace openshot return GetMaskImage(target_image, frame_number); } + /// Resolve reader artwork without requiring CPU pixels from the target frame. + std::shared_ptr ResolveReaderMaskImage(int width, int height, int64_t frame_number); + + /// Reader mask as a GPU surface at the target size. Video masks decoded on + /// the GPU stay there; other masks are read on the CPU and uploaded. + std::shared_ptr ResolveReaderMaskSurface(int width, int height, int64_t frame_number); + /// Optional override for effects that need custom mask behavior. virtual bool UseCustomMaskBlend(int64_t frame_number) const { return false; } @@ -103,6 +111,13 @@ namespace openshot /// Optional override for effects that apply mask processing inside GetFrame(). virtual bool HandlesMaskInternally() const { return false; } + /// Effects with a Vulkan path keep masked GPU frames on the GPU. + virtual bool RendersOnVulkan() const { return false; } + + /// True when the custom mask blend is the squared "drive amount" blend, + /// which the GPU applies too. + virtual bool MaskDrivesAmount(int64_t frame_number) const { return false; } + public: /// Parent effect (which properties will set this effect properties) EffectBase* parentEffect; diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 842bf69fb..55786c205 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -13,6 +13,7 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" #include // for std::this_thread::sleep_for #include // for std::chrono::milliseconds #include @@ -24,10 +25,19 @@ #include "FFmpegUtilities.h" #include "FFmpegColorRange.h" +#include "ColorManagement.h" + +extern "C" { +#include +} #include "PreviewSize.h" #include "effects/CropHelpers.h" #include "FFmpegReader.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "GpuDevices.h" +#include "VulkanFrameImage.h" +#endif #include "Exceptions.h" #include "MemoryTrim.h" #include "Timeline.h" @@ -142,6 +152,20 @@ bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64 // Get hardware pix format static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts) { +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__) + if (ctx->hw_device_ctx && reinterpret_cast(ctx->hw_device_ctx->data)->type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX) { + for (auto format = pix_fmts; *format != AV_PIX_FMT_NONE; ++format) + if (*format == AV_PIX_FMT_VIDEOTOOLBOX) return *format; + return avcodec_default_get_format(ctx, pix_fmts); + } +#endif +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (ctx->hw_device_ctx && reinterpret_cast(ctx->hw_device_ctx->data)->type == AV_HWDEVICE_TYPE_VULKAN) { + for (auto format = pix_fmts; *format != AV_PIX_FMT_NONE; ++format) + if (*format == AV_PIX_FMT_VULKAN) return *format; + return avcodec_default_get_format(ctx, pix_fmts); + } +#endif const enum AVPixelFormat *p; // Prefer only the format matching the selected hardware decoder @@ -247,6 +271,12 @@ void FFmpegReader::Open() { pFormatCtx = NULL; { hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // GPU builds decode through Vulkan unless the CPU backend is selected + // at runtime. Decode failures reopen this reader in software. + hw_de_on = !force_sw_decode && VulkanRenderingEnabled(); + vulkan_processor.reset(); +#endif hw_decode_failed = false; hw_decode_error_count = 0; hw_decode_succeeded = false; @@ -300,6 +330,17 @@ void FFmpegReader::Open() { // Get codec and codec context from stream const AVCodec *pCodec = avcodec_find_decoder(codecId); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Full FFmpeg builds prefer libdav1d for AV1, but that decoder cannot + // produce Vulkan frames. Keep it as the default for software retries. + if (codecId == AV_CODEC_ID_AV1 && hw_de_on && VulkanAv1DecodeSupported()) { + const AVCodec* native_av1 = avcodec_find_decoder_by_name("av1"); + if (native_av1) + pCodec = native_av1; + } +#endif + if (!pCodec) + throw InvalidCodec("A valid video codec could not be found for this file.", path); AVDictionary *opts = NULL; int retry_decode_open = 2; // If hw accel is selected but hardware cannot handle repeat with software decoding @@ -309,6 +350,21 @@ void FFmpegReader::Open() { if (hw_de_on && (retry_decode_open==2)) { // Up to here no decision is made if hardware or software decode hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + hw_de_supported = 0; + for (int index = 0; const AVCodecHWConfig* config = avcodec_get_hw_config(pCodec, index); ++index) { + if (config->device_type == +#ifdef __APPLE__ + AV_HWDEVICE_TYPE_VIDEOTOOLBOX && +#else + AV_HWDEVICE_TYPE_VULKAN && +#endif + (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) { + hw_de_supported = 1; + break; + } + } +#endif } #endif retry_decode_open = 0; @@ -316,14 +372,15 @@ void FFmpegReader::Open() { // Set number of threads equal to number of processors (not to exceed 16) pCodecCtx->thread_count = std::min(FF_VIDEO_NUM_PROCESSORS, 16); - if (pCodec == NULL) { - throw InvalidCodec("A valid video codec could not be found for this file.", path); - } - // Init options av_dict_set(&opts, "strict", "experimental", 0); #if USE_HW_ACCEL if (hw_de_on && hw_de_supported) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Vulkan decode surfaces must not be held by frame-thread workers + // while the consumer waits on their timeline semaphores. + pCodecCtx->thread_type = FF_THREAD_SLICE; +#endif // Open Hardware Acceleration int i_decoder_hw = 0; char adapter[256]; @@ -413,9 +470,21 @@ void FFmpegReader::Open() { Logger::Instance()->AppendDebugMethod("Decode Device not present using default"); } +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#ifdef __APPLE__ + hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; +#else + hw_de_av_device_type = AV_HWDEVICE_TYPE_VULKAN; +#endif + adapter_ptr = nullptr; +#endif hw_device_ctx = NULL; // Here the first hardware initialisations are made +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && !defined(__APPLE__) + if (CreateSharedVulkanDevice(&hw_device_ctx) >= 0) { +#else if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) { +#endif const char* hw_name = av_hwdevice_get_type_name(hw_de_av_device_type); std::string hw_msg = "HW decode active: "; hw_msg += (hw_name ? hw_name : "unknown"); @@ -453,7 +522,12 @@ void FFmpegReader::Open() { } else { Logger::Instance()->Log("HW decode active: no (falling back to software)"); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + hw_de_supported = 0; + pCodecCtx->get_format = avcodec_default_get_format; +#else throw InvalidCodec("Hardware device create failed.", path); +#endif } } #endif // USE_HW_ACCEL @@ -485,6 +559,9 @@ void FFmpegReader::Open() { ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config; constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig); #endif // ENABLE_VAAPI +#ifdef OPENSHOT_ENABLE_GPU_FRAME + constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx, nullptr); +#endif if (constraints) { if (pCodecCtx->coded_width < constraints->min_width || pCodecCtx->coded_height < constraints->min_height || @@ -656,10 +733,26 @@ void FFmpegReader::Open() { } // Process video stream side data (rotation, spherical metadata, etc) + hdr_peak_nits = 0.0; + sdr_converter.reset(); for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) { AVStream* st = pFormatCtx->streams[i]; if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { size_t side_data_size = 0; + if (static_cast(i) == videoStream) { + // HDR10 static metadata sets the tone mapping peak (MaxCLL, then mastering peak). + const uint8_t *light = ffmpeg_stream_get_side_data( + st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL, &side_data_size); + if (light && side_data_size >= sizeof(AVContentLightMetadata)) + hdr_peak_nits = reinterpret_cast(light)->MaxCLL; + const uint8_t *mastering = ffmpeg_stream_get_side_data( + st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, &side_data_size); + if (hdr_peak_nits <= 0.0 && mastering && side_data_size >= sizeof(AVMasteringDisplayMetadata)) { + const auto *display = reinterpret_cast(mastering); + if (display->has_luminance && display->max_luminance.den > 0) + hdr_peak_nits = av_q2d(display->max_luminance); + } + } const uint8_t *displaymatrix = ffmpeg_stream_get_side_data( st, AV_PKT_DATA_DISPLAYMATRIX, &side_data_size); if (displaymatrix && @@ -703,6 +796,24 @@ void FFmpegReader::Open() { const int final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 2); working_cache.SetMaxBytesFromInfo(working_cache_frames, info.width, info.height, info.sample_rate, info.channels); final_cache.SetMaxBytesFromInfo(final_cache_frames, info.width, info.height, info.sample_rate, info.channels); + if (!info.has_video && info.has_audio && info.sample_rate > 0 && info.channels > 0 && info.fps.ToDouble() > 0.0) { + // Audio-only frames carry no image, so a width x height budget is never + // reached. The cache would then keep every frame of the file, and since + // CacheMemory rescans its frames on each insert, long reads turn quadratic. + const int64_t samples_per_frame = static_cast(std::ceil(info.sample_rate / info.fps.ToDouble())); + const int64_t audio_frame_bytes = samples_per_frame * info.channels * static_cast(sizeof(float)); + final_cache.SetMaxBytes(final_cache_frames * audio_frame_bytes); + } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (hw_de_on && hw_de_supported) { + // Full-resolution hardware images consume device memory, not RAM. + // Bound each reader cache independently of FPS/thread-count heuristics. + constexpr int64_t gpu_cache_bytes = 256LL * 1024 * 1024; + // Pending video/audio frames cannot be evicted before they are paired. + // Keep the existing working-cache budget; bound finalized images below. + final_cache.SetMaxBytes(std::min(final_cache.GetMaxBytes(), gpu_cache_bytes)); + } +#endif // Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp. // This method allows us to shift timestamps to ensure at least 1 stream is starting at zero. @@ -733,6 +844,9 @@ void FFmpegReader::Close() { // Mark as "closed" is_open = false; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + vulkan_processor.reset(); +#endif // Keep track of most recent packet AVPacket *recent_packet = packet; @@ -1252,6 +1366,14 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { // Prevent async calls to the remainder of this code const std::lock_guard lock(getFrameMutex); +#if USE_HW_ACCEL && defined(OPENSHOT_ENABLE_GPU_FRAME) + // A lost GPU device cannot decode; reopen this reader on the CPU. + if (hw_de_on && !force_sw_decode && VulkanDeviceLost()) { + hw_decode_failed = true; + ReopenWithoutHardwareDecode(requested_frame); + } +#endif + // Check the cache a 2nd time (due to the potential previous lock) frame = final_cache.GetFrame(requested_frame); if (frame) { @@ -1460,7 +1582,11 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { // Copy and return the largest processed frame (assuming it was the last in the video file) std::shared_ptr f = CreateFrame(largest_frame_processed); if (frame->has_image_data) { - f->AddImage(std::make_shared(frame->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*frame->GetImage())); } // Use solid color (if no image data found) @@ -1478,10 +1604,18 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { std::shared_ptr f = CreateFrame(largest_frame_processed); if (last_final_video_frame && last_final_video_frame->has_image_data && last_final_video_frame->number <= requested_frame) { - f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*last_final_video_frame->GetImage())); } else if (last_video_frame && last_video_frame->has_image_data && last_video_frame->number <= requested_frame) { - f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*last_video_frame->GetImage())); } else { f->AddColor(info.width, info.height, "#000"); } @@ -1565,8 +1699,16 @@ bool FFmpegReader::GetAVFrame() { #if USE_HW_ACCEL // Get the format from the variables set in get_hw_dec_format +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__) + hw_de_av_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX; + hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; +#elif defined(OPENSHOT_ENABLE_GPU_FRAME) + hw_de_av_pix_fmt = AV_PIX_FMT_VULKAN; + hw_de_av_device_type = AV_HWDEVICE_TYPE_VULKAN; +#else hw_de_av_pix_fmt = hw_de_av_pix_fmt_global; hw_de_av_device_type = hw_de_av_device_type_global; +#endif #endif // USE_HW_ACCEL if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) { Logger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts); @@ -1634,6 +1776,15 @@ bool FFmpegReader::GetAVFrame() { #if USE_HW_ACCEL if (hw_de_on && hw_de_supported) { int err; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if ((next_frame2->format == AV_PIX_FMT_VULKAN +#ifdef __APPLE__ + || next_frame2->format == AV_PIX_FMT_VIDEOTOOLBOX +#endif + )) { + decoded_frame = next_frame2; + } else +#endif if (next_frame2->format == hw_de_av_pix_fmt) { if ((err = av_hwframe_transfer_data(next_frame, next_frame2, 0)) < 0) { Logger::Instance()->AppendDebugMethod( @@ -1672,7 +1823,11 @@ bool FFmpegReader::GetAVFrame() { decoded_frame = next_frame2; } - if (!decoded_frame->data[0]) { + if (!decoded_frame->data[0] +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__) + && !(decoded_frame->format == AV_PIX_FMT_VIDEOTOOLBOX && decoded_frame->data[3]) +#endif + ) { Logger::Instance()->AppendDebugMethod( "FFmpegReader::GetAVFrame (Decoded frame missing image data)", "format", decoded_frame->format, @@ -1688,10 +1843,38 @@ bool FFmpegReader::GetAVFrame() { hw_decode_error_count = 0; #if USE_HW_ACCEL if (hw_de_on && hw_de_supported && !force_sw_decode) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const bool previously_decoded_on_gpu = hw_decode_succeeded; + hw_decode_succeeded = hw_decode_succeeded || (next_frame2->format == AV_PIX_FMT_VULKAN +#ifdef __APPLE__ + || next_frame2->format == AV_PIX_FMT_VIDEOTOOLBOX +#endif + ); + if (!previously_decoded_on_gpu && hw_decode_succeeded) + Logger::Instance()->Log("GPU decode confirmed: " + info.vcodec + "; " + + (next_frame2->format == AV_PIX_FMT_VULKAN ? "Vulkan" : "VideoToolbox") + "; " + path, + Logger::LevelInfo); +#else hw_decode_succeeded = true; +#endif } #endif + const auto* metrics_format = av_pix_fmt_desc_get(static_cast(next_frame2->format)); + if (!info.has_single_image) + gpu_metrics::Record(metrics_format && (metrics_format->flags & AV_PIX_FMT_FLAG_HWACCEL) + ? gpu_metrics::DecodeGpu : gpu_metrics::DecodeCpu); packet_status.video_decoded++; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Cover art and still images are decoded on the CPU by design. + if (packet_status.video_decoded == 1 && !hw_decode_succeeded && !info.has_single_image) + Logger::Instance()->Log("GPU decode not observed: CPU video frame received; codec=" + + info.vcodec + "; " + path, Logger::LevelInfo); +#endif + + // HDR10 metadata may only arrive with frames. Keep the first value so + // tone mapping does not change from frame to frame. + if (hdr_peak_nits <= 0.0) + hdr_peak_nits = color::FramePeakNits(decoded_frame); // Allocate image (align 32 for simd) AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format); @@ -1699,6 +1882,16 @@ bool FFmpegReader::GetAVFrame() { decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format); const int decoded_width = decoded_frame->width > 0 ? decoded_frame->width : info.width; const int decoded_height = decoded_frame->height > 0 ? decoded_frame->height : info.height; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (decoded_pix_fmt == AV_PIX_FMT_VULKAN +#ifdef __APPLE__ + || decoded_pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX +#endif + ) { + if (av_frame_ref(pFrame, decoded_frame) < 0) + throw OutOfMemory("Failed to retain Vulkan frame", path); + } else { +#endif if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, decoded_width, decoded_height) <= 0) { throw OutOfMemory("Failed to allocate image buffer", path); } @@ -1712,6 +1905,9 @@ bool FFmpegReader::GetAVFrame() { pFrame->color_primaries = decoded_frame->color_primaries; pFrame->color_trc = decoded_frame->color_trc; pFrame->chroma_location = decoded_frame->chroma_location; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + } +#endif // Get display PTS from video frame, often different than packet->pts. // Sending packets to the decoder (i.e. packet->pts) is async, @@ -2018,6 +2214,39 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { } } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (pFrame && (pFrame->format == AV_PIX_FMT_VULKAN +#ifdef __APPLE__ + || pFrame->format == AV_PIX_FMT_VIDEOTOOLBOX +#endif + )) { + try { + if (!vulkan_processor) vulkan_processor = std::make_shared(); + auto f = CreateFrame(current_frame); + f->AddDeferredImage(vulkan_processor->Process(pFrame, width, height, + color::DescribeFrame(pFrame, src_width, src_height, hdr_peak_nits), + openshot::Settings::Instance()->HIGH_QUALITY_SCALING)); + gpu_metrics::Record(gpu_metrics::RenderGpu); + if (ApplyOrientationMetadata() && std::fabs(source_rotation) >= 0.0001) + Logger::Instance()->AppendDebugMethod("Vulkan reader: source orientation requires scaled CPU readback"); + ApplyFrameOrientation(f); + if (first_video_frame == 0 && !is_seeking) first_video_frame = current_frame; + working_cache.Add(f); + last_video_frame = f; + RemoveAVFrame(pFrame); + pFrame = nullptr; + video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds; + return; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan processing failed; retrying software decode: ") + error.what(), Logger::LevelWarning); + hw_decode_failed = true; + RemoveAVFrame(pFrame); + pFrame = nullptr; + return; + } + } +#endif + // Determine required buffer size and allocate buffer const int bytes_per_pixel = 4; int raw_buffer_size = (width * height * bytes_per_pixel) + 128; @@ -2027,21 +2256,36 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { int buffer_size = ((raw_buffer_size + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT; buffer = (unsigned char*) aligned_malloc(buffer_size, ALIGNMENT); - // Copy picture data from one AVFrame (or AVPicture) to another one. - AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height); + // HDR (PQ/HLG) and BT.2020 sources keep 16 bits through scaling, then + // convert to SDR BT.709. Other sources convert straight to RGBA8. + const color::SourceColor source_color = color::DescribeFrame(pFrame, src_width, src_height, hdr_peak_nits); + const bool convert_color = source_color.NeedsConversion(); + const AVPixelFormat rgb_pix_fmt = convert_color ? AV_PIX_FMT_RGBA64 : PIX_FMT_RGBA; + if (convert_color) { + hdr_buffer.resize(size_t(width) * height * 4); + av_image_fill_arrays(pFrameRGB->data, pFrameRGB->linesize, + reinterpret_cast(hdr_buffer.data()), AV_PIX_FMT_RGBA64, width, height, 1); + const auto* previous = sdr_converter ? &sdr_converter->Source() : nullptr; + if (!previous || previous->transfer != source_color.transfer || previous->bt2020 != source_color.bt2020 || + previous->RelativePeak() != source_color.RelativePeak()) + sdr_converter = std::make_shared(source_color); + } else { + // Copy picture data from one AVFrame (or AVPicture) to another one. + AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height); + } int scale_mode = SWS_FAST_BILINEAR; if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) { scale_mode = SWS_BICUBIC; } - img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL); + img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, rgb_pix_fmt, scale_mode, NULL, NULL, NULL); if (!img_convert_ctx) throw OutOfMemory("Failed to initialize sws context", path); - const int *src_coeff = sws_getCoefficients(SWS_CS_DEFAULT); - const int *dst_coeff = sws_getCoefficients(SWS_CS_DEFAULT); + // Decode with the source's tagged YUV matrix (BT.601 when untagged). + const int *coefficients = sws_getCoefficients(color::SwsColorspace(source_color.matrix)); const int dst_full_range = 1; // RGB outputs are full-range - sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0, - dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16); + sws_setColorspaceDetails(img_convert_ctx, coefficients, src_full_range ? 1 : 0, + coefficients, dst_full_range, 0, 1 << 16, 1 << 16); if (!pFrame || !pFrame->data[0] || pFrame->linesize[0] <= 0) { #if USE_HW_ACCEL @@ -2064,6 +2308,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { } // Resize / Convert to RGB + gpu_metrics::Record(gpu_metrics::RenderCpu); const int scaled_lines = sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0, original_height, pFrameRGB->data, pFrameRGB->linesize); if (scaled_lines <= 0) { @@ -2087,6 +2332,9 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { return; } + if (convert_color) + sdr_converter->Convert(hdr_buffer.data(), pFrameRGB->linesize[0], buffer, width * bytes_per_pixel, width, height); + // Create or get the existing frame object std::shared_ptr f = CreateFrame(current_frame); @@ -2821,7 +3069,11 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { // visual order, especially when decode/prefetch is out-of-order. std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); if (previous_frame_instance && previous_frame_instance->has_image_data) { - f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*previous_frame_instance->GetImage())); } // Fall back to last finalized timeline image (survives cache churn). @@ -2829,7 +3081,11 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { && last_final_video_frame && last_final_video_frame->has_image_data && last_final_video_frame->number <= f->number) { - f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*last_final_video_frame->GetImage())); } // Fall back to the last decoded image only when it is not from the future. @@ -2837,7 +3093,11 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { && last_video_frame && last_video_frame->has_image_data && last_video_frame->number <= f->number) { - f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*last_video_frame->GetImage())); } // Last-resort fallback if no prior image is available. @@ -2850,8 +3110,10 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { } } + // Every frame waiting for its audio keeps a decoded image, so keep this margin + // small: 0.25 sec covers partial and delayed audio packets. double audio_pts_diff = audio_pts_seconds - frame_pts_seconds; - if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 1.0) + if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 0.25) || (recent_pts_diff > 1.5) || packet_status.audio_eof || packet_status.end_of_file) { // Audio stream is past this frame (so it must be done) @@ -2886,21 +3148,35 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (info.has_single_image) { // For still-image video (including attached cover art), reuse the most // recent image so playback does not stall waiting for video EOF. + // QImage copies share pixels until written, and keep the cache key + // that lets the GPU reuse one upload instead of one per frame. std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); if (previous_frame_instance && previous_frame_instance->has_image_data) { - f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*previous_frame_instance->GetImage())); } if (!f->has_image_data && last_final_video_frame && last_final_video_frame->has_image_data && last_final_video_frame->number <= f->number) { - f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*last_final_video_frame->GetImage())); } if (!f->has_image_data && last_video_frame && last_video_frame->has_image_data && last_video_frame->number <= f->number) { - f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*last_video_frame->GetImage())); } } @@ -2910,19 +3186,31 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (!f->has_image_data && is_video_ready && is_audio_ready) { std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); if (previous_frame_instance && previous_frame_instance->has_image_data) { - f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*previous_frame_instance->GetImage())); } if (!f->has_image_data && last_final_video_frame && last_final_video_frame->has_image_data && last_final_video_frame->number <= f->number) { - f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*last_final_video_frame->GetImage())); } if (!f->has_image_data && last_video_frame && last_video_frame->has_image_data && last_video_frame->number <= f->number) { - f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif + f->AddImage(std::make_shared(*last_video_frame->GetImage())); } } @@ -3049,6 +3337,13 @@ void FFmpegReader::CheckFPS() { void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) { // Remove pFrame (if exists) if (remove_frame) { + // Referenced hardware frames release their image through AVBufferRef. +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (remove_frame->hw_frames_ctx) { + av_frame_free(&remove_frame); + return; + } +#endif // Free memory av_freep(&remove_frame->data[0]); #ifndef WIN32 diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index c8f1abfce..b7b918e06 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "AudioLocation.h" #include "CacheMemory.h" #include "Clip.h" @@ -36,6 +37,9 @@ namespace openshot { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + class VulkanFrameProcessor; +#endif /** * @brief This struct holds the packet counts and end-of-file detection for an openshot::FFmpegReader. * @@ -100,9 +104,15 @@ namespace openshot { * r.Close(); * @endcode */ + namespace color { class SdrConverter; } + class FFmpegReader : public ReaderBase { private: std::string path; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + std::shared_ptr vulkan_processor; + int hw_de_on = 0; // Per-reader state for concurrent Vulkan/CPU readers. +#endif AVFormatContext *pFormatCtx; int videoStream, audioStream; @@ -166,6 +176,9 @@ namespace openshot { // Cached conversion contexts and frames for performance SwsContext *img_convert_ctx = nullptr; ///< Cached video scaler context + double hdr_peak_nits = 0.0; ///< HDR10 content peak from metadata (0 = unknown) + std::shared_ptr sdr_converter; ///< HDR/wide gamut to SDR (CPU path) + std::vector hdr_buffer; ///< 16-bit RGBA scratch for SDR conversion SWRCONTEXT *avr_ctx = nullptr; ///< Cached audio resample context AVFrame *pFrameRGB_cached = nullptr; ///< Temporary frame used for video conversion diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 96e7dbbec..e0b065b7b 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -13,6 +13,7 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" #include #include #include @@ -27,26 +28,46 @@ #include "FFmpegUtilities.h" #include "FFmpegWriter.h" +#include "FFmpegColorRange.h" +#include "ColorManagement.h" #include "Exceptions.h" #include "Frame.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "GpuDevices.h" +#include "VulkanFrameImage.h" +#include "VulkanCompositor.h" +#include "VulkanEncodeConverter.h" +#ifdef __APPLE__ +#include "MacVulkanInterop.h" +#endif +#endif #include "OpenMPUtilities.h" #include "Settings.h" #include "Logger.h" using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +bool IsVulkanEncoder(const std::string& name) { + return name == "h264_vulkan" || name == "hevc_vulkan" || name == "av1_vulkan"; +} +std::string SoftwareEncoder(const std::string& name) { + const std::vector candidates = name == "h264_vulkan" ? std::vector{"libx264", "libopenh264"} + : name == "hevc_vulkan" ? std::vector{"libx265"} + : std::vector{"libsvtav1", "libaom-av1", "librav1e"}; + for (const auto& candidate : candidates) + if (avcodec_find_encoder_by_name(candidate.c_str())) return candidate; + return {}; +} +} +#endif + // Multiplexer parameters temporary storage AVDictionary *mux_dict = NULL; #if USE_HW_ACCEL -int hw_en_on = 1; // Is set in UI -int hw_en_supported = 0; // Is set by FFmpegWriter -AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE; -AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI; -static AVBufferRef *hw_device_ctx = NULL; -AVFrame *hw_frame = NULL; - -static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height) +static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height, AVPixelFormat format) { AVBufferRef *hw_frames_ref; AVHWFramesContext *frames_ctx = NULL; @@ -57,7 +78,7 @@ static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int6 return -1; } frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data); - frames_ctx->format = hw_en_av_pix_fmt; + frames_ctx->format = format; frames_ctx->sw_format = AV_PIX_FMT_NV12; frames_ctx->width = width; frames_ctx->height = height; @@ -101,14 +122,44 @@ void FFmpegWriter::Open() { if (!is_open) { // Open the writer is_open = true; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + gpu_export_failed = false; + gpu_export_logged = false; + // FFmpeg binds a GPU frame to the first CUDA context it is sent to. Start + // each export with fresh pools, so no frame is tied to an old encoder. + ReleaseVulkanMemory(); +#endif // Prepare streams (if needed) if (!prepare_streams) PrepareStreams(); // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers - if (info.has_video && video_st) + if (info.has_video && video_st) { +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL + try { + open_video(oc, video_st); + } catch (const std::exception& error) { + // Choose the replacement before the muxer header is written. + // Switching encoders after packets are emitted is unsafe. + const auto fallback = IsVulkanEncoder(info.vcodec) ? SoftwareEncoder(info.vcodec) : std::string(); + if (fallback.empty() || write_header) throw; + Logger::Instance()->Log("Vulkan encoder CPU fallback: " + std::string(error.what()) + "; using " + fallback, Logger::LevelWarning); + av_buffer_unref(&hw_device_ctx); + SetVideoOptions(true, fallback, info.fps, info.width, info.height, info.pixel_ratio, + info.interlaced_frame, info.top_field_first, info.video_bit_rate); + add_video_stream(video_st); + open_video(oc, video_st); + } +#else open_video(oc, video_st); +#endif +#ifdef OPENSHOT_ENABLE_GPU_FRAME + Logger::Instance()->Log("GPU export setup: encoder=" + info.vcodec + + "; hardware encoding configured=" + (hw_en_on && hw_en_supported ? "yes" : "no") + + "; frame interop is checked when frames arrive", Logger::LevelInfo); +#endif + } if (info.has_audio && audio_st) open_audio(oc, audio_st); @@ -167,11 +218,31 @@ void FFmpegWriter::initialize_streams() { // Set video export options void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Only encoders the probe opened are safe; some drivers and loaders crash. + const auto gpu_encoders = IsVulkanEncoder(codec) ? GpuEncoders() : std::vector(); + if (IsVulkanEncoder(codec) && (!USE_HW_ACCEL || std::find(gpu_encoders.begin(), gpu_encoders.end(), codec) == gpu_encoders.end())) { + const auto fallback = SoftwareEncoder(codec); + if (!fallback.empty()) { + Logger::Instance()->Log("Vulkan encoder unavailable; using CPU encoder " + fallback, Logger::LevelWarning); + codec = fallback; + } + } +#endif // Set the video options if (codec.length() > 0) { const AVCodec *new_codec; // Check if the codec selected is a hardware accelerated codec #if USE_HW_ACCEL +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (IsVulkanEncoder(codec)) { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = hw_en_supported = true; + hw_en_av_pix_fmt = AV_PIX_FMT_VULKAN; + hw_en_av_device_type = AV_HWDEVICE_TYPE_VULKAN; + } else +#endif + { #if defined(__linux__) if (strstr(codec.c_str(), "_vaapi") != NULL) { new_codec = avcodec_find_encoder_by_name(codec.c_str()); @@ -223,6 +294,7 @@ void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction f #else // unknown OS new_codec = avcodec_find_encoder_by_name(codec.c_str()); #endif //__linux__/_WIN32/__APPLE__ + } #else // USE_HW_ACCEL new_codec = avcodec_find_encoder_by_name(codec.c_str()); #endif // USE_HW_ACCEL @@ -1068,6 +1140,12 @@ void FFmpegWriter::WriteHeader() { } // Add a frame to the queue waiting to be encoded. +void FFmpegWriter::WriteMeasuredFrame(ReaderBase* reader, int64_t number) { + gpu_metrics::Scope scope(true); + gpu_metrics::RecordExportFrame(); + WriteFrame(reader->GetFrame(number)); +} + void FFmpegWriter::WriteFrame(std::shared_ptr frame) { // Check for open reader (or throw exception) if (!is_open) @@ -1119,6 +1197,7 @@ void FFmpegWriter::WriteFrameAt(std::shared_ptr frame, int64_t // Write all frames in the queue to the video file. void FFmpegWriter::write_frame(std::shared_ptr frame) { + gpu_metrics::Reason gpu_reason("Encoder"); // Flip writing flag is_writing = true; @@ -1145,6 +1224,9 @@ void FFmpegWriter::write_frame(std::shared_ptr frame) { } // Deallocate buffer and AVFrame +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (!frame_final->hw_frames_ctx) +#endif av_freep(&(frame_final->data[0])); AV_FREE_FRAME(&frame_final); av_frames.erase(frame); @@ -1466,6 +1548,10 @@ void FFmpegWriter::Close() { close_video(oc, video_st); if (audio_st) close_audio(oc, audio_st); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Free pooled frames still tied to this encoder's CUDA context + ReleaseVulkanMemory(); +#endif // Remove single software scaler if (img_convert_ctx) @@ -1640,7 +1726,7 @@ AVStream *FFmpegWriter::add_audio_stream() { } // Add a video output stream -AVStream *FFmpegWriter::add_video_stream() { +AVStream *FFmpegWriter::add_video_stream(AVStream* existing) { // Find the video codec const AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str()); if (codec == NULL) @@ -1652,7 +1738,7 @@ AVStream *FFmpegWriter::add_video_stream() { } // Create a new video stream - AVStream* st = avformat_new_stream(oc, codec); + AVStream* st = existing ? existing : avformat_new_stream(oc, codec); if (!st) throw OutOfMemory("Could not allocate memory for the video stream.", path); @@ -1672,9 +1758,11 @@ AVStream *FFmpegWriter::add_video_stream() { c->sample_aspect_ratio.den = info.pixel_ratio.den; /* Init video encoder options */ + // VA-API AV1 supports bitrate control; the legacy AV1 quality conversion + // below clears the bitrate and uses a qp option this encoder does not expose. if (info.video_bit_rate >= 1000 #if (LIBAVCODEC_VERSION_MAJOR >= 58) - && c->codec_id != AV_CODEC_ID_AV1 + && (c->codec_id != AV_CODEC_ID_AV1 || info.vcodec == "av1_vaapi" || info.vcodec == "av1_vulkan") #endif ) { c->bit_rate = info.video_bit_rate; @@ -1954,7 +2042,27 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { Logger::Instance()->AppendDebugMethod( "Encode Device not present, using default"); } - if (av_hwdevice_ctx_create(&hw_device_ctx, + // Encode on the card that renders, so VA-API can share its frames. That card + // may have no VA-API driver (e.g. NVIDIA in a hybrid laptop); then use the + // device chosen above. + bool device_created = false; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (hw_en_av_device_type == AV_HWDEVICE_TYPE_VULKAN) { + if (!VulkanRenderingEnabled() || CreateSharedVulkanDevice(&hw_device_ctx) < 0) + throw InvalidCodec("No Vulkan rendering device is available", path); + device_created = true; + } +#endif +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__linux__) + if (hw_en_av_device_type == AV_HWDEVICE_TYPE_VAAPI && VulkanRenderingEnabled()) { + const std::string render_node = ActiveGpuDevice().render_node; + device_created = !render_node.empty() && + av_hwdevice_ctx_create(&hw_device_ctx, hw_en_av_device_type, render_node.c_str(), NULL, 0) >= 0; + if (!render_node.empty() && !device_created) + Logger::Instance()->Log("No VA-API on " + render_node + "; encoding on the default VA-API device", Logger::LevelWarning); + } +#endif + if (!device_created && av_hwdevice_ctx_create(&hw_device_ctx, hw_en_av_device_type, adapter_ptr, NULL, 0) < 0) { Logger::Instance()->AppendDebugMethod( @@ -2006,13 +2114,23 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { } } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const bool vulkan_encoder = hw_en_av_pix_fmt == AV_PIX_FMT_VULKAN; +#else + const bool vulkan_encoder = false; // Older FFmpeg has no AV_PIX_FMT_VULKAN +#endif switch (video_codec_ctx->codec_id) { case AV_CODEC_ID_H264: video_codec_ctx->max_b_frames = 0; // At least this GPU doesn't support b-frames - video_codec_ctx->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE; - av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0); - av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0); - av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN); + if (!vulkan_encoder) { + video_codec_ctx->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE; + av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0); + if (hw_en_av_pix_fmt == AV_PIX_FMT_CUDA) + av_opt_set_int(video_codec_ctx->priv_data, "zerolatency", 1, 0); + else + av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0); + av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN); + } break; case AV_CODEC_ID_HEVC: // tested to work with defaults @@ -2029,7 +2147,7 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { // set hw_frames_ctx for encoder's AVCodecContext int err; - if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height)) < 0) + if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height, hw_en_av_pix_fmt)) < 0) { Logger::Instance()->AppendDebugMethod( "FFmpegWriter::open_video (set_hwframe_ctx) ERROR faled to set hwframe context", @@ -2051,6 +2169,24 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { } #endif + // Tag YUV outputs so players decode them as rendered: SDR BT.709 color, + // with the matrix players assume for this size (BT.709 for HD, BT.601 for + // SD). RGB outputs are left untagged. + { + const AVPixFmtDescriptor* desc = av_pix_fmt_desc_get(video_codec_ctx->pix_fmt); + const bool hardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL); + const bool rgb = desc && !hardware && (desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_PAL)); + const bool gray = desc && !hardware && desc->nb_components <= 2; + if (desc && !rgb && !gray) { + bool full_range = false; + NormalizeDeprecatedPixFmt(video_codec_ctx->pix_fmt, full_range); + video_codec_ctx->color_primaries = AVCOL_PRI_BT709; + video_codec_ctx->color_trc = AVCOL_TRC_BT709; + video_codec_ctx->colorspace = color::EncodeMatrix(info.width, info.height); + video_codec_ctx->color_range = full_range ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + } + } + /* open the codec */ if (avcodec_open2(video_codec_ctx, codec, &opts) < 0) throw InvalidCodec("Could not open video codec", path); @@ -2610,6 +2746,102 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { if (src_w == 1 && src_h == 1) return; +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__) && USE_HW_ACCEL + if (!gpu_export_failed && hw_en_on && hw_en_supported && + video_codec_ctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX && video_codec_ctx->hw_frames_ctx) { + auto surface = frame->GetDeferredImage(); + if (surface) { + AVFrame* encoded = av_frame_alloc(); + try { + if (!encoded) throw std::bad_alloc(); + auto native = std::dynamic_pointer_cast(surface); + const auto* context = native ? reinterpret_cast(native->GetFrame()->hw_frames_ctx->data) : nullptr; + if (src_w != info.width || src_h != info.height || !context || context->sw_format != AV_PIX_FMT_RGBA) { + QTransform scale; + scale.scale(double(info.width) / src_w, double(info.height) / src_h); + surface = RenderVulkanImage(surface, nullptr, info.width, info.height, scale, 1.0, Qt::black); + } + // Render directly into the encoder's own Core Video pixel buffer. + // No CPU download, upload, or separate NV12 image copy is needed. + const int allocated = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0); + if (allocated < 0) throw std::runtime_error("Allocate VideoToolbox encoder frame: " + av_err2string(allocated)); + std::unique_ptr target(ImportVideoToolboxFrame(encoded, true), + [](AVFrame* value) { av_frame_free(&value); }); + auto converted = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface, target.get())); + if (!converted) throw std::runtime_error("No shared VideoToolbox conversion result"); + // VideoToolbox does not consume Vulkan timeline semaphores. Retire + // GPU writes before handing the same IOSurface to its encoder. + WaitMacVulkanFrame(converted->GetFrame()); + const int copied = av_frame_copy_props(encoded, converted->GetFrame()); + if (copied < 0) throw std::bad_alloc(); + add_avframe(frame, encoded); + encoded = nullptr; // add_avframe owns the retained encoder frame. + if (!gpu_export_logged) { + gpu_export_logged = true; + Logger::Instance()->Log("Vulkan export: encoder-owned VideoToolbox IOSurface (no CPU copy)", Logger::LevelInfo); + } + return; + } catch (const std::exception& error) { + av_frame_free(&encoded); + gpu_export_failed = true; + Logger::Instance()->Log(std::string("VideoToolbox GPU export CPU fallback: ") + error.what(), Logger::LevelWarning); + } + } + } +#endif +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL + // Native Vulkan writes directly into encoder-owned NV12. NVENC retains its + // device-to-device transfer path; other encoders use the established CPU path. + const bool native_vulkan = video_codec_ctx->pix_fmt == AV_PIX_FMT_VULKAN; + if (!gpu_export_failed && hw_en_on && hw_en_supported && video_codec_ctx->hw_frames_ctx && + (native_vulkan || video_codec_ctx->pix_fmt == AV_PIX_FMT_CUDA)) { + auto surface = frame->GetDeferredImage(); + if (surface) { + AVFrame* encoded = nullptr; + try { + auto native = std::dynamic_pointer_cast(surface); + const auto* context = native ? reinterpret_cast(native->GetFrame()->hw_frames_ctx->data) : nullptr; + if (src_w != info.width || src_h != info.height || !context || context->sw_format != AV_PIX_FMT_RGBA) { + QTransform scale; + scale.scale(double(info.width) / src_w, double(info.height) / src_h); + surface = RenderVulkanImage(surface, nullptr, info.width, info.height, scale, 1.0, Qt::black); + } + // NVENC keeps its existing separate-plane conversion and GPU + // transfer. Native Vulkan writes into the encoder's own frame. + std::shared_ptr nv12; + if (!native_vulkan) + nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface)); + encoded = av_frame_alloc(); + if (!encoded) throw std::bad_alloc(); + const int allocated = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0); + if (allocated < 0) throw std::runtime_error("Allocate encoder frame: " + av_err2string(allocated)); + if (native_vulkan) { + nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface, encoded)); + } else { + const int status = av_hwframe_transfer_data(encoded, nv12->GetFrame(), 0); + if (status < 0) throw std::runtime_error("Vulkan/CUDA export transfer failed: " + av_err2string(status)); + } + + av_frame_copy_props(encoded, nv12->GetFrame()); + add_avframe(frame, encoded); + if (!gpu_export_logged) { + gpu_export_logged = true; + Logger::Instance()->Log(native_vulkan ? "Vulkan export: encoder-owned GPU frames (no copy)" + : "Vulkan export: GPU transfer to NVENC", Logger::LevelInfo); + } + return; + } catch (const std::exception& error) { + av_frame_free(&encoded); + // Interop only touches the temporary scaled/NV12 surfaces. The readback + // below waits on the frame surface's semaphores, so it sees finished + // pixels; keep this export on the CPU path. + gpu_export_failed = true; + Logger::Instance()->Log(std::string("Vulkan export CPU fallback: ") + error.what(), Logger::LevelWarning); + } + } + } +#endif + // Point persistent_src_frame->data to RGBA pixels const uchar* pixels = frame->GetPixels(); if (!persistent_src_frame) { @@ -2684,6 +2916,13 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { ); if (!img_convert_ctx) throw ErrorEncodingVideo("Could not initialize sws context", -1); + // Encode with the matrix and range tagged in open_video(). + bool dst_full_range = false; + NormalizeDeprecatedPixFmt(dst_fmt, dst_full_range); + const int* coefficients = sws_getCoefficients( + color::SwsColorspace(color::EncodeMatrix(info.width, info.height))); + sws_setColorspaceDetails(img_convert_ctx, coefficients, 1, coefficients, + dst_full_range ? 1 : 0, 0, 1 << 16, 1 << 16); } // Scale RGBA → dst_fmt into persistent_dst_buffer @@ -2798,21 +3037,25 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra // Assign the initial AVFrame PTS from the frame counter frame_final->pts = video_timestamp; #if USE_HW_ACCEL - if (hw_en_on && hw_en_supported) { + if (hw_en_on && hw_en_supported +#ifdef OPENSHOT_ENABLE_GPU_FRAME + && !frame_final->hw_frames_ctx +#endif + ) { if (!(hw_frame = av_frame_alloc())) { - std::clog << "Error code: av_hwframe_alloc\n"; + throw OutOfMemory("Could not allocate hardware upload frame", path); } if (av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, hw_frame, 0) < 0) { - std::clog << "Error code: av_hwframe_get_buffer\n"; - } - if (!hw_frame->hw_frames_ctx) { - std::clog << "Error hw_frames_ctx.\n"; + av_frame_free(&hw_frame); + throw ErrorEncodingVideo("Could not allocate hardware upload buffer", -1); } // Keep the hardware format assigned by av_hwframe_get_buffer(). // NV12 describes the CPU source, not the encoder surface. if ( av_hwframe_transfer_data(hw_frame, frame_final, 0) < 0) { - std::clog << "Error while transferring frame data to surface.\n"; + av_frame_free(&hw_frame); + throw ErrorEncodingVideo("Could not upload CPU frame to encoder", -1); } + gpu_metrics::Record(gpu_metrics::Upload); av_frame_copy_props(hw_frame, frame_final); } #endif // USE_HW_ACCEL @@ -2825,12 +3068,23 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra #if USE_HW_ACCEL if (hw_en_on && hw_en_supported) { - ret = avcodec_send_frame(video_codec_ctx, hw_frame); //hw_frame!!! +#ifdef OPENSHOT_ENABLE_GPU_FRAME + ret = avcodec_send_frame(video_codec_ctx, frame_final->hw_frames_ctx ? frame_final : hw_frame); +#else + ret = avcodec_send_frame(video_codec_ctx, hw_frame); +#endif } else #endif // USE_HW_ACCEL { ret = avcodec_send_frame(video_codec_ctx, frame_final); } + if (ret >= 0) { +#if USE_HW_ACCEL + gpu_metrics::Record(hw_en_on && hw_en_supported ? gpu_metrics::EncodeGpu : gpu_metrics::EncodeCpu); +#else + gpu_metrics::Record(gpu_metrics::EncodeCpu); +#endif + } error_code = ret; if (ret < 0 ) { Logger::Instance()->AppendDebugMethod( diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index 1859cb866..c05c66cb6 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -136,10 +136,23 @@ namespace openshot { uint8_t *audio_outbuf; uint8_t *audio_encoder_buffer; +#if USE_HW_ACCEL + // Hardware encoder ownership belongs to this export, never another writer. + bool hw_en_on = false; + bool hw_en_supported = false; + AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE; + AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_NONE; + AVBufferRef* hw_device_ctx = nullptr; + AVFrame* hw_frame = nullptr; +#endif AVFrame *persistent_src_frame = nullptr; AVFrame *persistent_dst_frame = nullptr; uint8_t *persistent_dst_buffer = nullptr; int persistent_dst_size = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + bool gpu_export_failed = false; ///< Encode remaining frames from CPU pixels + bool gpu_export_logged = false; ///< The GPU export path was reported once +#endif int audio_outbuf_size; int audio_input_frame_size; @@ -163,7 +176,7 @@ namespace openshot { AVStream *add_audio_stream(); /// Add a video output stream - AVStream *add_video_stream(); + AVStream *add_video_stream(AVStream* existing = nullptr); /// Allocate an AVFrame object AVFrame *allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer); @@ -202,6 +215,7 @@ namespace openshot { void write_frame(std::shared_ptr frame); public: + void WriteMeasuredFrame(ReaderBase* reader, int64_t number); /// @brief Constructor for FFmpegWriter. /// Throws an exception on failure to open path. diff --git a/src/Frame.cpp b/src/Frame.cpp index 1862a82d0..6447326b2 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -14,11 +14,13 @@ #include // for std::chrono::milliseconds #include #include +#include #include "Frame.h" #include "AudioBufferSource.h" #include "AudioResampler.h" #include "QtUtilities.h" +#include "GpuMetrics.h" #include #include @@ -86,6 +88,13 @@ Frame& Frame::operator= (const Frame& other) // Copy data and pointers from another Frame instance void Frame::DeepCopy(const Frame& other) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (this == &other) + return; + std::scoped_lock lock(addingImageMutex, other.addingImageMutex); + deferred_image = other.deferred_image; + image.reset(); +#endif number = other.number; capture_timestamp = other.capture_timestamp; channels = other.channels; @@ -381,11 +390,18 @@ juce::AudioBuffer *Frame::GetAudioSampleBuffer() // Get the size in bytes of this frame (rough estimate) int64_t Frame::GetBytes() { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const std::lock_guard lock(addingImageMutex); +#endif int64_t total_bytes = 0; if (image) { total_bytes += static_cast( width * height * sizeof(char) * 4); } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (deferred_image) + total_bytes += deferred_image->GetBytes(); +#endif if (audio) { // approximate audio size (sample rate / 24 fps) total_bytes += (sample_rate / 24.0) * sizeof(float); @@ -398,6 +414,9 @@ int64_t Frame::GetBytes() // Get pixel data (as packets) const unsigned char* Frame::GetPixels() { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + GetImage(); +#endif // Check for blank image if (!image) // Fill with black @@ -410,6 +429,9 @@ const unsigned char* Frame::GetPixels() // Get pixel data (for only a single scan-line) const unsigned char* Frame::GetPixels(int row) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + GetImage(); +#endif // Check for blank image if (!image) // Fill with black @@ -421,6 +443,9 @@ const unsigned char* Frame::GetPixels(int row) // Check a specific pixel color value (returns True/False) bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (deferred_image) GetImage(); +#endif int col_pos = col * 4; // Find column array position if (!image || row < 0 || row >= (height - 1) || col_pos < 0 || col_pos >= (width - 1) ) { @@ -516,6 +541,7 @@ ChannelLayout Frame::ChannelsLayout() // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG) void Frame::Save(std::string path, float scale, std::string format, int quality) { + gpu_metrics::Reason gpu_reason("Image file"); // Get preview image std::shared_ptr previewImage = GetImage(); @@ -544,6 +570,7 @@ void Frame::Save(std::string path, float scale, std::string format, int quality) // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG) void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate, ScaleType scale_mode) { + gpu_metrics::Reason gpu_reason("Thumbnail"); // Create blank thumbnail image & fill background color auto thumbnail = std::make_shared( @@ -703,6 +730,9 @@ void Frame::AddColor(const QColor& new_color) { // Create new image object, and fill with pixel data const std::lock_guard lock(addingImageMutex); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + deferred_image.reset(); +#endif image = std::make_shared(width, height, QImage::Format_RGBA8888_Premultiplied); // Fill with solid color @@ -741,6 +771,9 @@ void Frame::AddImage(std::shared_ptr new_image) // assign image data const std::lock_guard lock(addingImageMutex); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + deferred_image.reset(); +#endif image = new_image; // Always convert to Format_RGBA8888_Premultiplied (if different) @@ -760,6 +793,9 @@ void Frame::AddImage(std::shared_ptr new_image, bool only_odd_lines) if (!new_image) return; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (deferred_image) GetImage(); +#endif // Check for blank source image if (!image) { // Replace the blank source image @@ -871,6 +907,17 @@ void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, // Get pointer to Magick++ image object std::shared_ptr Frame::GetImage() { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const std::lock_guard lock(addingImageMutex); + if (deferred_image) { + auto materialized = deferred_image->Materialize(); + if (materialized.isNull() || materialized.width() != width || materialized.height() != height) + throw std::runtime_error("Deferred frame image returned invalid dimensions"); + // GetImage exposes mutable pixels. Make CPU storage authoritative so a + // later GPU consumer cannot accidentally reuse stale surface contents. + AddImage(std::make_shared(std::move(materialized))); + } +#endif // Check for blank image if (!image) // Fill with black @@ -895,6 +942,9 @@ cv::Mat Frame::Qimage2mat( std::shared_ptr& qimage) { // Get pointer to OpenCV image object cv::Mat Frame::GetImageCV() { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + GetImage(); +#endif // Check for blank image if (!image) // Fill with black @@ -924,7 +974,11 @@ std::shared_ptr Frame::Mat2Qimage(cv::Mat img){ void Frame::SetImageCV(cv::Mat _image) { imagecv = _image; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + AddImage(Mat2Qimage(_image)); +#else image = Mat2Qimage(_image); +#endif } #endif @@ -1015,3 +1069,33 @@ void Frame::AddAudioSilence(int numSamples) // Reset audio direction audio_is_increasing = true; } + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +void Frame::AddDeferredImage(std::shared_ptr new_image) +{ + if (!new_image) + return; + if (new_image->GetWidth() <= 0 || new_image->GetHeight() <= 0 || new_image->GetBytes() < 0) + throw std::invalid_argument("Invalid deferred frame image"); + const std::lock_guard lock(addingImageMutex); + width = new_image->GetWidth(); + height = new_image->GetHeight(); + image.reset(); + deferred_image = std::move(new_image); + has_image_data = true; +} + +std::shared_ptr Frame::GetDeferredImage() const +{ + const std::lock_guard lock(addingImageMutex); + return deferred_image; +} + +bool Frame::GetBlankColor(QColor& result) const +{ + const std::lock_guard lock(addingImageMutex); + if (image || deferred_image) return false; + result = QColor(QString::fromStdString(color)); + return result.isValid(); +} +#endif diff --git a/src/Frame.h b/src/Frame.h index 82106f685..bc440a53e 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -13,6 +13,8 @@ #ifndef OPENSHOT_FRAME_H #define OPENSHOT_FRAME_H +#include "OpenShotVersion.h" + #ifdef USE_OPENCV #define int64 opencv_broken_int #define uint64 opencv_broken_uint @@ -27,6 +29,9 @@ #include "ChannelLayouts.h" #include "Enums.h" #include "Fraction.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "FrameImage.h" +#endif #include #include @@ -90,10 +95,13 @@ namespace openshot { private: std::shared_ptr image; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + std::shared_ptr deferred_image; +#endif std::shared_ptr wave_image; std::shared_ptr previewApp; - std::recursive_mutex addingImageMutex; + mutable std::recursive_mutex addingImageMutex; std::recursive_mutex addingAudioMutex; openshot::Fraction pixel_ratio; int channels; @@ -120,6 +128,15 @@ namespace openshot bool has_image_data; ///< This frame has been loaded with pixel data +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && !defined(SWIG) + /// Attach an immutable deferred image without materializing CPU pixels. + void AddDeferredImage(std::shared_ptr new_image); + /// Returns the surface only while it remains authoritative. + std::shared_ptr GetDeferredImage() const; + /// Report an unmaterialized solid canvas without allocating CPU pixels. + bool GetBlankColor(QColor& result) const; +#endif + /// Constructor - blank frame Frame(); diff --git a/src/FrameImage.h b/src/FrameImage.h new file mode 100644 index 000000000..199e89f02 --- /dev/null +++ b/src/FrameImage.h @@ -0,0 +1,25 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_FRAME_IMAGE_H +#define OPENSHOT_FRAME_IMAGE_H + +#include +#include + +namespace openshot { + +/// Immutable image storage, including a hardware surface with deferred readback. +/// Implementations own their resources and synchronization. Materialize must be +/// safe for concurrent callers and return independent/COW CPU image storage. +class FrameImage { +public: + virtual ~FrameImage() = default; + virtual int GetWidth() const = 0; + virtual int GetHeight() const = 0; + /// Conservative resident allocation size, for cache budgeting. + virtual int64_t GetBytes() const = 0; + virtual QImage Materialize() const = 0; +}; + +} // namespace openshot +#endif diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index eb5164163..c5031045b 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -52,6 +52,9 @@ FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType targe // Adjust cache size based on size of frame and audio const int initial_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS); final_cache.SetMaxBytesFromInfo(initial_cache_frames, info.width, info.height, info.sample_rate, info.channels); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + final_cache.SetMaxBytes(std::min(final_cache.GetMaxBytes(), 128LL * 1024 * 1024)); +#endif } // Destructor @@ -548,8 +551,17 @@ std::shared_ptr FrameMapper::GetFrame(int64_t requested_frame) // Copy the image from the odd field std::shared_ptr odd_frame = mapped_frame; - if (odd_frame && odd_frame->has_image_data) - frame->AddImage(std::make_shared(*odd_frame->GetImage()), true); + if (odd_frame && odd_frame->has_image_data) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Progressive mapping needs no pixel access. Preserve immutable GPU + // storage while remapping the frame number and audio independently. + auto surface = odd_frame->GetDeferredImage(); + if (surface && mapped.Odd.Frame == mapped.Even.Frame) + frame->AddDeferredImage(std::move(surface)); + else +#endif + frame->AddImage(std::make_shared(*odd_frame->GetImage()), true); + } if (mapped.Odd.Frame != mapped.Even.Frame) { // Add even lines (if different than the previous image) std::shared_ptr even_frame; @@ -858,6 +870,9 @@ void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldow // Adjust cache size based on size of frame and audio const int reset_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4); final_cache.SetMaxBytesFromInfo(reset_cache_frames, info.width, info.height, info.sample_rate, info.channels); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + final_cache.SetMaxBytes(std::min(final_cache.GetMaxBytes(), 128LL * 1024 * 1024)); +#endif // Deallocate resample buffer if (avr) { diff --git a/src/FrameScope.cpp b/src/FrameScope.cpp index db2d54639..5ec41d85b 100644 --- a/src/FrameScope.cpp +++ b/src/FrameScope.cpp @@ -11,6 +11,10 @@ // SPDX-License-Identifier: LGPL-3.0-or-later #include "FrameScope.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "Logger.h" +#endif #include #include @@ -236,13 +240,29 @@ void FrameScope::analyze() { void FrameScope::analyze_video() { // Frame images are always QImage::Format_RGBA8888_Premultiplied (enforced // by Frame::AddImage). Pixel byte order is [R=0, G=1, B=2, A=3]. - std::shared_ptr image = frame->GetImage(); - if (!image || image->isNull()) + std::shared_ptr image; + int width = 0; + int height = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Analysis is read-only. Do not replace the shared preview/cache surface + // with mutable CPU pixels just because the scopes need to inspect it. + auto surface = frame->GetDeferredImage(); + if (surface) { + width = surface->GetWidth(); + height = surface->GetHeight(); + } else +#endif + { + image = frame->GetImage(); + if (!image || image->isNull()) + return; + width = image->width(); + height = image->height(); + } + if (width <= 0 || height <= 0) return; video_present = true; - const int width = image->width(); - const int height = image->height(); int start_x = 0; int end_x = width; int start_y = 0; @@ -265,6 +285,44 @@ void FrameScope::analyze_video() { clipped_green = 0; clipped_blue = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (surface) { + try { + // Count on the GPU; only the counters return to the CPU. + const auto counts = ScopeVulkanImage(surface, QRect(start_x, start_y, video_width, video_height), + waveform_columns, waveform_bins, vectorscope_size); + auto next = counts.begin(); + for (auto* values : {&histogram_luma, &histogram_red, &histogram_green, &histogram_blue, + &waveform_luma, &waveform_red, &waveform_green, &waveform_blue, &vectorscope}) { + std::copy_n(next, values->size(), values->begin()); + next += static_cast(values->size()); + } + // The summary follows from the histograms. Average luma uses bin + // centers, within half a bin of the CPU's exact sum. + double weighted = 0.0; + int64_t pixels = 0; + for (int bin = 0; bin < 256; ++bin) { + pixels += histogram_luma[bin]; + weighted += static_cast(bin) * histogram_luma[bin]; + } + avg_luma = pixels > 0 ? weighted / (255.0 * static_cast(pixels)) : 0.0; + for (int bin = 0; bin < 3; ++bin) + clipped_shadows += histogram_luma[bin]; + for (int bin = 253; bin < 256; ++bin) { + clipped_highlights += histogram_luma[bin]; + clipped_red += histogram_red[bin]; + clipped_green += histogram_green[bin]; + clipped_blue += histogram_blue[bin]; + } + return; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan scope CPU fallback: ") + error.what()); + } + image = std::make_shared(surface->Materialize()); + if (image->isNull()) + return; + } +#endif const int bytes_per_line = image->bytesPerLine(); const unsigned char* bits = image->constBits(); const auto& inv_alpha = inv_alpha_lut(); diff --git a/src/GpuDebug.h b/src/GpuDebug.h new file mode 100644 index 000000000..e71383ff8 --- /dev/null +++ b/src/GpuDebug.h @@ -0,0 +1,30 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later + +// Temporary switches for debugging GPU test builds; search for "GpuDebug" +// to find (and later remove) every use. +// OPENSHOT_GPU_DEBUG=1 extra logging only, no behavior change +// OPENSHOT_PRESENTATION_TEST=pattern preview shows solid magenta instead of video +// OPENSHOT_PRESENTATION_SYNC=wait CPU waits instead of Vulkan/OpenGL semaphores +// OPENSHOT_PRESENTATION_DEDICATED=0|1 force the preview buffer's memory allocation style +#ifndef OPENSHOT_GPU_DEBUG_H +#define OPENSHOT_GPU_DEBUG_H + +#include + +namespace openshot { + +// OPENSHOT_TRACE_PRESENTATION is the older name for the same logging. +inline bool GpuDebug() { + static const bool enabled = qEnvironmentVariableIsSet("OPENSHOT_GPU_DEBUG") || + qEnvironmentVariableIsSet("OPENSHOT_TRACE_PRESENTATION"); + return enabled; +} + +inline QString GpuDebugSetting(const char* name) { + return qEnvironmentVariable(name).trimmed().toLower(); +} + +} + +#endif diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp new file mode 100644 index 000000000..48bcd432d --- /dev/null +++ b/src/GpuDevices.cpp @@ -0,0 +1,393 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuDevices.h" +#include "Logger.h" +#include "VulkanFrameImage.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +#include +#include +} +#include "VulkanLoader.h" +namespace openshot { +namespace { +// A temporary instance for listing devices; FFmpeg keeps its own for rendering. +struct ListingInstance { + VkInstance instance = VK_NULL_HANDLE; + ListingInstance() { + if (!LoadVulkan()) + return; + VkApplicationInfo app = {}; + app.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; + app.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo create = {}; + create.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; + create.pApplicationInfo = &app; + if (vkCreateInstance(&create, nullptr, &instance) != VK_SUCCESS) + instance = VK_NULL_HANDLE; + } + ~ListingInstance() { if (instance) vkDestroyInstance(instance, nullptr); } + std::vector Devices() const { + uint32_t count = 0; + if (!instance) return {}; + vkEnumeratePhysicalDevices(instance, &count, nullptr); + std::vector devices(count); + vkEnumeratePhysicalDevices(instance, &count, devices.data()); + return devices; + } +}; + +bool HasExtension(VkPhysicalDevice device, const char* name) { + uint32_t count = 0; + vkEnumerateDeviceExtensionProperties(device, nullptr, &count, nullptr); + std::vector extensions(count); + vkEnumerateDeviceExtensionProperties(device, nullptr, &count, extensions.data()); + return std::any_of(extensions.begin(), extensions.end(), + [name](const VkExtensionProperties& e) { return std::strcmp(e.extensionName, name) == 0; }); +} + +GpuDevice Describe(VkPhysicalDevice device) { + VkPhysicalDeviceProperties basic; + vkGetPhysicalDeviceProperties(device, &basic); + VkPhysicalDeviceIDProperties ids = {}; + ids.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES; + VkPhysicalDeviceDriverProperties driver = {}; + driver.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES; + VkPhysicalDeviceDrmPropertiesEXT drm = {}; + drm.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT; + VkPhysicalDeviceProperties2 props = {}; + props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + props.pNext = &ids; + // Chain only the structures this device understands. + void** next = &ids.pNext; + const bool has_driver = basic.apiVersion >= VK_API_VERSION_1_2; + if (has_driver) { *next = &driver; next = &driver.pNext; } + const bool has_drm = HasExtension(device, VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME); + if (has_drm) { *next = &drm; next = &drm.pNext; } + vkGetPhysicalDeviceProperties2(device, &props); + + GpuDevice gpu; + gpu.name = basic.deviceName; + gpu.vulkan_api = std::to_string(VK_API_VERSION_MAJOR(basic.apiVersion)) + "." + + std::to_string(VK_API_VERSION_MINOR(basic.apiVersion)) + "." + + std::to_string(VK_API_VERSION_PATCH(basic.apiVersion)); + switch (basic.vendorID) { + case 0x10DE: gpu.vendor = "NVIDIA"; break; + case 0x1002: gpu.vendor = "AMD"; break; + case 0x8086: gpu.vendor = "Intel"; break; + default: { + char id[16]; + std::snprintf(id, sizeof(id), "0x%04x", basic.vendorID); + gpu.vendor = id; + } + } + switch (basic.deviceType) { + case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: gpu.type = "discrete"; break; + case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: gpu.type = "integrated"; break; + case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: gpu.type = "virtual"; break; + case VK_PHYSICAL_DEVICE_TYPE_CPU: gpu.type = "software"; break; + default: gpu.type = "other"; + } + if (has_driver) + gpu.driver = std::string(driver.driverName) + " " + driver.driverInfo; + for (uint8_t byte : ids.deviceUUID) { + char hex[3]; + std::snprintf(hex, sizeof(hex), "%02x", byte); + gpu.uuid += hex; + } + if (has_drm && drm.hasRender) + gpu.render_node = "/dev/dri/renderD" + std::to_string(drm.renderMinor); + + uint32_t families = 0; + vkGetPhysicalDeviceQueueFamilyProperties(device, &families, nullptr); + std::vector queues(families); + vkGetPhysicalDeviceQueueFamilyProperties(device, &families, queues.data()); + gpu.video_decode = std::any_of(queues.begin(), queues.end(), + [](const VkQueueFamilyProperties& q) { return q.queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR; }); + gpu.video_encode = std::any_of(queues.begin(), queues.end(), + [](const VkQueueFamilyProperties& q) { return q.queueFlags & VK_QUEUE_VIDEO_ENCODE_BIT_KHR; }); + return gpu; +} + +// FFmpeg uses the first Vulkan device, which can be a software driver such as +// llvmpipe. List real GPUs instead, discrete before integrated. Software (CPU) +// devices are skipped: they cannot decode video and are slower than our CPU path. +std::vector VulkanDeviceCandidates() { + const char* chosen = std::getenv("OPENSHOT_VULKAN_DEVICE"); + if (chosen && chosen[0]) + return {chosen}; + ListingInstance listing; + if (!listing.instance) + return {""}; // Let FFmpeg report why Vulkan is unavailable + std::vector> ranked; + for (VkPhysicalDevice device : listing.Devices()) { + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(device, &props); + if (props.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU) + continue; + const int rank = props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? 0 + : props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? 1 : 2; + // Select by name: FFmpeg's device indices can be in a different order + ranked.emplace_back(rank, props.deviceName); + } + std::stable_sort(ranked.begin(), ranked.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + std::vector candidates; + std::transform(ranked.begin(), ranked.end(), std::back_inserter(candidates), + [](const auto& device) { return device.second; }); + return candidates; +} + +VkPhysicalDevice PhysicalDevice(AVBufferRef* device) { + auto* context = reinterpret_cast(device->data); + return static_cast(context->hwctx)->phys_dev; +} + +// Try opening an encoder at a tiny size on the given hardware device. +bool EncoderOpens(const char* name, AVBufferRef* device, AVPixelFormat hw_format) { + const AVCodec* codec = avcodec_find_encoder_by_name(name); + if (!codec) { + Logger::Instance()->Log(std::string("GPU encoder probe: ") + name + " is not compiled into FFmpeg", Logger::LevelDebug); + return false; + } + AVCodecContext* context = avcodec_alloc_context3(codec); + if (!context) return false; + context->width = 640; + context->height = 360; + context->time_base = {1, 30}; + context->framerate = {30, 1}; + context->bit_rate = 1000000; + bool opened = false; + if (hw_format == AV_PIX_FMT_VAAPI || hw_format == AV_PIX_FMT_VULKAN || hw_format == AV_PIX_FMT_VIDEOTOOLBOX) { + AVBufferRef* frames = av_hwframe_ctx_alloc(device); + if (frames) { + auto* fc = reinterpret_cast(frames->data); + fc->format = hw_format; + fc->sw_format = AV_PIX_FMT_NV12; + fc->width = context->width; + fc->height = context->height; + if (av_hwframe_ctx_init(frames) >= 0) { + context->pix_fmt = hw_format; + context->hw_frames_ctx = av_buffer_ref(frames); + } + av_buffer_unref(&frames); + } + } else { + // NVENC takes system-memory frames on the CUDA device given here + context->pix_fmt = AV_PIX_FMT_NV12; + context->hw_device_ctx = av_buffer_ref(device); + } + if (context->hw_frames_ctx || context->hw_device_ctx) { + const int result = avcodec_open2(context, codec, nullptr); + opened = result >= 0; + if (result < 0) { + char message[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(result, message, sizeof(message)); + Logger::Instance()->Log(std::string("GPU encoder probe failed: ") + name + ": " + message, Logger::LevelDebug); + } + } + avcodec_free_context(&context); + return opened; +} +} // namespace + +int CreateSharedVulkanDevice(AVBufferRef** output) { + // All readers share one device so their surfaces can be composed without interop. + // Keep an owning reference for the process lifetime; each caller gets its own ref. + // Never destroyed: destroying the device during process exit crashes drivers + // that are already unloading. + struct SharedDevice { + std::mutex mutex; + AVBufferRef* device = nullptr; + bool tried = false; + }; + static auto& shared = *new SharedDevice; + const std::lock_guard lock(shared.mutex); + if (!shared.device) { + // Only search once; without a usable GPU every reader decodes on the CPU. + if (shared.tried) + return AVERROR(ENODEV); + shared.tried = true; + if (!LoadVulkan()) + return AVERROR(ENODEV); +#ifdef __APPLE__ + // Work around imported-texture argument-buffer writes on older Intel + // Metal drivers. An explicit user setting still takes precedence. + setenv("MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS", "0", 0); +#endif + int result = AVERROR(ENODEV); + for (const auto& device : VulkanDeviceCandidates()) { + Logger::Instance()->Log("Vulkan device initialization: " + (device.empty() ? std::string("default") : device), Logger::LevelInfo); + AVDictionary* options = nullptr; +#ifdef __APPLE__ + av_dict_set(&options, "device_extensions", "VK_EXT_metal_objects", 0); +#endif + result = av_hwdevice_ctx_create(&shared.device, AV_HWDEVICE_TYPE_VULKAN, + device.empty() ? nullptr : device.c_str(), options, 0); + av_dict_free(&options); + if (result >= 0) + break; + char message[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(result, message, sizeof(message)); + Logger::Instance()->Log("Vulkan device initialization failed: " + device + ": " + message + + " (" + std::to_string(result) + ")", Logger::LevelWarning); + } + if (result < 0) { + Logger::Instance()->Log("No usable Vulkan GPU; decoding on the CPU", Logger::LevelWarning); + return result; + } + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(PhysicalDevice(shared.device), &props); + Logger::Instance()->Log(std::string("Vulkan device: ") + props.deviceName, Logger::LevelInfo); + } + *output = av_buffer_ref(shared.device); + return *output ? 0 : AVERROR(ENOMEM); +} + +GpuDevice ActiveGpuDevice() { + AVBufferRef* device = nullptr; + if (!VulkanRenderingEnabled() || CreateSharedVulkanDevice(&device) < 0) + return {}; + GpuDevice gpu = Describe(PhysicalDevice(device)); + gpu.active = true; + av_buffer_unref(&device); + return gpu; +} + +void ConfigureGpuEnvironment(const std::string& backend, const std::string& device) { +#ifdef _WIN32 + if (_putenv_s("OPENSHOT_RENDER_BACKEND", backend.c_str()) != 0 || + _putenv_s("OPENSHOT_VULKAN_DEVICE", device.c_str()) != 0) + throw std::runtime_error("Unable to apply native GPU environment"); +#else + const auto apply = [](const char* key, const std::string& value) { + return value.empty() ? unsetenv(key) : setenv(key, value.c_str(), 1); + }; + if (apply("OPENSHOT_RENDER_BACKEND", backend) != 0 || apply("OPENSHOT_VULKAN_DEVICE", device) != 0) + throw std::runtime_error("Unable to apply native GPU environment"); +#endif +} + +bool VulkanAv1DecodeSupported() { + AVBufferRef* device = nullptr; + if (!VulkanRenderingEnabled() || CreateSharedVulkanDevice(&device) < 0) + return false; + const auto physical = PhysicalDevice(device); + const bool supported = Describe(physical).video_decode && + HasExtension(physical, "VK_KHR_video_decode_av1"); + av_buffer_unref(&device); + return supported; +} + +std::vector ListGpuDevices() { + const std::string active = ActiveGpuDevice().uuid; + ListingInstance listing; + std::vector devices; + for (VkPhysicalDevice device : listing.Devices()) { + devices.push_back(Describe(device)); + devices.back().active = !active.empty() && devices.back().uuid == active; + } + return devices; +} + +std::vector GpuEncoders() { + static std::mutex mutex; + static bool tested = false; + static std::vector encoders; + const std::lock_guard lock(mutex); + if (tested) + return encoders; + tested = true; + AVBufferRef* vulkan = nullptr; + if (!VulkanRenderingEnabled() || CreateSharedVulkanDevice(&vulkan) < 0) + return encoders; + const GpuDevice gpu = Describe(PhysicalDevice(vulkan)); + // Encoders that cannot open print errors; this is only a capability test. + const int log_level = av_log_get_level(); + av_log_set_level(AV_LOG_QUIET); + // Prefer the same Vulkan device/API for every vendor, when its driver and + // this FFmpeg build support the requested codec. Probe, never assume. + // Old loaders (1.2.131 on Ubuntu 20.04) return a stub without a driver + // function for the encode queries, and FFmpeg crashes calling it. + // 1.3.270 is the oldest loader verified to fail cleanly instead. + uint32_t loader_version = 0; + if (vkEnumerateInstanceVersion(&loader_version) == VK_SUCCESS && + loader_version >= VK_MAKE_API_VERSION(0, 1, 3, 270)) { + for (const char* name : {"h264_vulkan", "hevc_vulkan", "av1_vulkan"}) + if (EncoderOpens(name, vulkan, AV_PIX_FMT_VULKAN)) encoders.push_back(name); + } + // CUDA derived from Vulkan is the same card (matched by UUID) + AVBufferRef* cuda = nullptr; + if (av_hwdevice_ctx_create_derived(&cuda, AV_HWDEVICE_TYPE_CUDA, vulkan, 0) >= 0) { + for (const char* name : {"h264_nvenc", "hevc_nvenc", "av1_nvenc"}) + if (EncoderOpens(name, cuda, AV_PIX_FMT_CUDA)) encoders.push_back(name); + av_buffer_unref(&cuda); + } +#ifdef __APPLE__ + AVBufferRef* videotoolbox = nullptr; + if (av_hwdevice_ctx_create(&videotoolbox, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr, nullptr, 0) >= 0) { + for (const char* name : {"h264_videotoolbox", "hevc_videotoolbox"}) + if (EncoderOpens(name, videotoolbox, AV_PIX_FMT_VIDEOTOOLBOX)) encoders.push_back(name); + av_buffer_unref(&videotoolbox); + } +#endif + AVBufferRef* vaapi = nullptr; + if (!gpu.render_node.empty() && + av_hwdevice_ctx_create(&vaapi, AV_HWDEVICE_TYPE_VAAPI, gpu.render_node.c_str(), nullptr, 0) >= 0) { + for (const char* name : {"h264_vaapi", "hevc_vaapi", "av1_vaapi", "vp9_vaapi"}) + if (EncoderOpens(name, vaapi, AV_PIX_FMT_VAAPI)) encoders.push_back(name); + av_buffer_unref(&vaapi); + } + av_log_set_level(log_level); + av_buffer_unref(&vulkan); + std::string list; + for (const auto& name : encoders) + list += (list.empty() ? "" : ", ") + name; + Logger::Instance()->Log("GPU encoders on " + gpu.name + ": " + (list.empty() ? "none" : list), Logger::LevelInfo); + return encoders; +} + +namespace { std::atomic device_lost{false}; } +bool VulkanDeviceLost() { return device_lost.load(std::memory_order_relaxed); } +void ReportVulkanResult(int result) { + if (result != VK_ERROR_DEVICE_LOST || device_lost.exchange(true)) return; + Logger::Instance()->Log("GPU device lost (driver reset or GPU timeout); using the CPU for the rest of this session", + Logger::LevelWarning); +} + +std::string VulkanResultName(int result) { + switch (static_cast(result)) { + case VK_SUCCESS: return "VK_SUCCESS"; + case VK_NOT_READY: return "VK_NOT_READY"; + case VK_TIMEOUT: return "VK_TIMEOUT"; + case VK_INCOMPLETE: return "VK_INCOMPLETE"; + case VK_ERROR_OUT_OF_HOST_MEMORY: return "VK_ERROR_OUT_OF_HOST_MEMORY"; + case VK_ERROR_OUT_OF_DEVICE_MEMORY: return "VK_ERROR_OUT_OF_DEVICE_MEMORY"; + case VK_ERROR_INITIALIZATION_FAILED: return "VK_ERROR_INITIALIZATION_FAILED"; + case VK_ERROR_DEVICE_LOST: return "VK_ERROR_DEVICE_LOST"; + case VK_ERROR_MEMORY_MAP_FAILED: return "VK_ERROR_MEMORY_MAP_FAILED"; + case VK_ERROR_LAYER_NOT_PRESENT: return "VK_ERROR_LAYER_NOT_PRESENT"; + case VK_ERROR_EXTENSION_NOT_PRESENT: return "VK_ERROR_EXTENSION_NOT_PRESENT"; + case VK_ERROR_FEATURE_NOT_PRESENT: return "VK_ERROR_FEATURE_NOT_PRESENT"; + case VK_ERROR_INCOMPATIBLE_DRIVER: return "VK_ERROR_INCOMPATIBLE_DRIVER"; + case VK_ERROR_TOO_MANY_OBJECTS: return "VK_ERROR_TOO_MANY_OBJECTS"; + case VK_ERROR_FORMAT_NOT_SUPPORTED: return "VK_ERROR_FORMAT_NOT_SUPPORTED"; + case VK_ERROR_FRAGMENTED_POOL: return "VK_ERROR_FRAGMENTED_POOL"; + case VK_ERROR_OUT_OF_POOL_MEMORY: return "VK_ERROR_OUT_OF_POOL_MEMORY"; + case VK_ERROR_INVALID_EXTERNAL_HANDLE: return "VK_ERROR_INVALID_EXTERNAL_HANDLE"; + case VK_ERROR_FRAGMENTATION: return "VK_ERROR_FRAGMENTATION"; + default: return "VkResult " + std::to_string(result); + } +} +} diff --git a/src/GpuDevices.h b/src/GpuDevices.h new file mode 100644 index 000000000..66de5e085 --- /dev/null +++ b/src/GpuDevices.h @@ -0,0 +1,51 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_GPU_DEVICES_H +#define OPENSHOT_GPU_DEVICES_H +#include +#include +#ifndef SWIG +struct AVBufferRef; +#endif +namespace openshot { +/// A Vulkan graphics card as OpenShot sees it. +struct GpuDevice { + std::string name; + std::string vendor; ///< "NVIDIA", "AMD", "Intel", or the PCI vendor id + std::string type; ///< "discrete", "integrated", "virtual", "software" or "other" + std::string driver; ///< Driver name and version, when the driver reports them + std::string vulkan_api; ///< Vulkan API version advertised by the physical device + std::string uuid; ///< Stable id (hex), the same one CUDA reports for the card + std::string render_node; ///< Linux DRM render node such as /dev/dri/renderD128, if known + bool video_decode = false; ///< Has a Vulkan video decode queue + bool video_encode = false; ///< Has a Vulkan video encode queue (not codec-specific support) + bool active = false; ///< The card OpenShot renders with +}; +/// Apply before any device/worker initialization. Keep the native C runtime's +/// environment consistent with Python when Windows uses different runtimes. +void ConfigureGpuEnvironment(const std::string& backend, const std::string& device); +/// Every Vulkan device on this system, including software ones. Empty without Vulkan. +std::vector ListGpuDevices(); +/// The card OpenShot renders with, choosing it on first use. Empty name when +/// Vulkan is off (OPENSHOT_RENDER_BACKEND=cpu) or no usable GPU was found. +GpuDevice ActiveGpuDevice(); +/// Hardware encoders (e.g. "h264_nvenc", "hevc_vaapi") that open on the active +/// card. Tested once per process with a tiny frame size; takes a fraction of a second. +std::vector GpuEncoders(); +/// True after a Vulkan call reported VK_ERROR_DEVICE_LOST (a driver reset or GPU +/// timeout). Processing then stays on the CPU for the rest of the session. +bool VulkanDeviceLost(); +#ifndef SWIG +/// The shared Vulkan device every reader decodes on, created on first use with the +/// best GPU (OPENSHOT_VULKAN_DEVICE, an index or a name, overrides it). Returns a new +/// reference, or a negative FFmpeg error when there is no usable GPU. +int CreateSharedVulkanDevice(AVBufferRef** output); +/// A VkResult's name (e.g. "VK_ERROR_DEVICE_LOST"), for error messages. +std::string VulkanResultName(int result); +/// Record a failed Vulkan call; VK_ERROR_DEVICE_LOST sets VulkanDeviceLost(). +void ReportVulkanResult(int result); +/// Whether the active Vulkan driver advertises AV1 hardware decoding. +bool VulkanAv1DecodeSupported(); +#endif +} +#endif diff --git a/src/GpuMetrics.cpp b/src/GpuMetrics.cpp new file mode 100644 index 000000000..5539c6f28 --- /dev/null +++ b/src/GpuMetrics.cpp @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" +#include "Frame.h" +#include "Json.h" +#include +#include +#include +#include +#include +namespace openshot { namespace gpu_metrics { +struct Counters { + std::array, Count> values{}; + std::atomic resident{0}, requests{0}; + std::atomic export_frames{0}, export_resident{0}; + std::array, 5> reasons{}; +}; +namespace { +std::shared_ptr sessions[2] = {std::make_shared(), std::make_shared()}; +thread_local Scope* current = nullptr; +thread_local const char* reason = "Other"; +const char* names[] = {"decode_gpu", "decode_cpu", "image_sources", "render_gpu", "render_cpu", + "readbacks", "uploads", "encode_gpu", "encode_cpu", "upload_failures"}; +const char* reasons[] = {"Effect", "Transform", "Composite", "Encoder", "Other"}; +} +Scope::Scope(bool exporting) : previous(current), exceptions(std::uncaught_exceptions()) { + if (!current) { + counters = std::atomic_load(&sessions[exporting ? 1 : 0]); + current = this; + } +} +Scope::~Scope() { + if (counters) { + if (export_frame && std::uncaught_exceptions() == exceptions) { + counters->export_frames.fetch_add(1, std::memory_order_relaxed); + if (gpu && !readback) counters->export_resident.fetch_add(1, std::memory_order_relaxed); + } + if (gpu && std::uncaught_exceptions() == exceptions) { + counters->requests.fetch_add(1, std::memory_order_relaxed); + if (!readback) counters->resident.fetch_add(1, std::memory_order_relaxed); + } + current = previous; + } +} +Reason::Reason(const char* name) : previous(reason) { reason = name; } +Reason::~Reason() { reason = previous; } +const char* CurrentReason() { return reason; } +void RecordExportFrame() { + if (current) current->export_frame = true; +} +void Record(Event event) { + if (!current) return; + current->counters->values[event].fetch_add(1, std::memory_order_relaxed); + if (event == DecodeGpu || event == RenderGpu || event == EncodeGpu) current->gpu = true; + if (event == Readback) { + current->readback = true; + size_t index = 4; + for (size_t i = 0; i < 4; ++i) if (std::strcmp(reason, reasons[i]) == 0) { index = i; break; } + current->counters->reasons[index].fetch_add(1, std::memory_order_relaxed); + } +} +void RecordRender(const Frame& frame) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + Record(frame.GetDeferredImage() ? RenderGpu : RenderCpu); +#else + Record(RenderCpu); +#endif +} +} // namespace gpu_metrics +std::string GpuMetricsSnapshot(bool exporting) { + using namespace gpu_metrics; + auto counters = std::atomic_load(&sessions[exporting ? 1 : 0]); + Json::Value root(Json::objectValue); + for (int i = 0; i < Count; ++i) root[names[i]] = Json::UInt64(counters->values[i].load(std::memory_order_relaxed)); + root["copy_free"] = Json::UInt64(counters->resident.load(std::memory_order_relaxed)); + root["gpu_requests"] = Json::UInt64(counters->requests.load(std::memory_order_relaxed)); + root["export_frames"] = Json::UInt64(counters->export_frames.load(std::memory_order_relaxed)); + root["export_copy_free"] = Json::UInt64(counters->export_resident.load(std::memory_order_relaxed)); + for (int i = 0; i < 5; ++i) root["readback_reasons"][reasons[i]] = Json::UInt64(counters->reasons[i].load(std::memory_order_relaxed)); + root["definition"] = "Operation counts exclude cache hits; copy-free counts GPU work requests without GPU-to-CPU readbacks, not end-to-end host-free frames"; + root["export_copy_free_definition"] = "Measured writer frames with GPU work and no CPU readback; includes cached frames and excludes separate background rendering requests"; + return root.toStyledString(); +} +void ResetGpuMetrics(bool exporting) { + std::atomic_store(&gpu_metrics::sessions[exporting ? 1 : 0], std::make_shared()); +} +} diff --git a/src/GpuMetrics.h b/src/GpuMetrics.h new file mode 100644 index 000000000..2a255fa57 --- /dev/null +++ b/src/GpuMetrics.h @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_GPU_METRICS_H +#define OPENSHOT_GPU_METRICS_H +#include +#include +namespace openshot { +class Frame; +/// Bounded, operation counters. Export and playback have independent generations. +std::string GpuMetricsSnapshot(bool exporting = false); +void ResetGpuMetrics(bool exporting = false); +#ifndef SWIG +namespace gpu_metrics { +enum Event { DecodeGpu, DecodeCpu, ImageSource, RenderGpu, RenderCpu, + Readback, Upload, EncodeGpu, EncodeCpu, UploadFailure, Count }; +/// Scope follows synchronous timeline work; unrelated background readers are ignored. +struct Counters; +class Scope { + Scope* previous; + std::shared_ptr counters; + bool gpu = false, readback = false; + bool export_frame = false; + int exceptions; + friend void Record(Event event); + friend void RecordExportFrame(); +public: + explicit Scope(bool exporting); + ~Scope(); + Scope(const Scope&) = delete; + Scope& operator=(const Scope&) = delete; +}; +class Reason { + const char* previous; +public: + explicit Reason(const char* name); + ~Reason(); +}; +void Record(Event event); +void RecordRender(const Frame& frame); +/// Mark a measured writer request, including one served from the timeline cache. +void RecordExportFrame(); +/// The current thread's reason for CPU work ("Other" outside a Reason scope). +const char* CurrentReason(); +} +#endif +} +#endif diff --git a/src/MacVulkanInterop.h b/src/MacVulkanInterop.h new file mode 100644 index 000000000..cf0a76cb1 --- /dev/null +++ b/src/MacVulkanInterop.h @@ -0,0 +1,29 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_MAC_VULKAN_INTEROP_H +#define OPENSHOT_MAC_VULKAN_INTEROP_H +#include +extern "C" { +#include +} +struct AVFrame; +struct AVBufferRef; +namespace openshot { +// Internal Apple bridge. Returned AVFrames are owned by the caller. Hardware +// surfaces retain their CVPixelBuffers until all Vulkan submissions have retired. +AVFrame* ImportVideoToolboxFrame(const AVFrame* frame, bool writable = false); +AVFrame* AllocateMacVulkanFrame(AVBufferRef* device, int width, int height, bool nv12); +/// The FFmpeg layout of a Core Video pixel format (OSType) that can be shared +/// with Vulkan: NV12, P010 or P210, else AV_PIX_FMT_NONE. +AVPixelFormat MacSurfaceFormat(uint32_t core_video_format); +uint64_t MacVulkanDeviceRegistryID(const AVFrame* frame); +int64_t MacVulkanFrameBytes(const AVFrame* frame); +bool IsMacVulkanFrame(const AVFrame* frame); +void WaitMacVulkanFrame(const AVFrame* frame); +// Borrowed CVPixelBufferRef; valid while the AVFrame is retained. +void* MacVulkanPixelBuffer(const AVFrame* frame); +// Explicit CPU fallback only. Returns an FFmpeg status; never used by GPU export. +int ReadMacVulkanFrame(AVFrame* destination, const AVFrame* source); +void ReleaseMacVulkanPools(); +} +#endif diff --git a/src/MacVulkanInterop.mm b/src/MacVulkanInterop.mm new file mode 100644 index 000000000..ed7e6a594 --- /dev/null +++ b/src/MacVulkanInterop.mm @@ -0,0 +1,338 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#define VK_USE_PLATFORM_METAL_EXT +#import +#import +#import +#include "MacVulkanInterop.h" +#include "GpuDevices.h" +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +#include +#include +#include +} +#include "VulkanLoader.h" +namespace openshot { +namespace { +const char native_pool_tag = 0; +void AvCheck(int r, const char* what) { if (r < 0) throw std::runtime_error(what); } +void VkCheck(VkResult r, const char* what) { if (r != VK_SUCCESS) throw std::runtime_error(what); } +// Shared surface layouts: RGBA, or two-plane YUV with 8-bit (NV12) or +// MSB-aligned 10-bit (P010 4:2:0, P210 4:2:2) samples, as VideoToolbox decodes. +struct SurfaceFormat { + AVPixelFormat sw_format; + int planes; + MTLPixelFormat metal[2]; + VkFormat vulkan[2]; +}; +SurfaceFormat Describe(AVPixelFormat sw_format) { + switch (sw_format) { + case AV_PIX_FMT_NV12: + return {sw_format, 2, {MTLPixelFormatR8Unorm, MTLPixelFormatRG8Unorm}, {VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM}}; + case AV_PIX_FMT_P010: + case AV_PIX_FMT_P210: + return {sw_format, 2, {MTLPixelFormatR16Unorm, MTLPixelFormatRG16Unorm}, {VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM}}; + case AV_PIX_FMT_RGBA: + return {sw_format, 1, {MTLPixelFormatRGBA8Unorm, MTLPixelFormatInvalid}, {VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_UNDEFINED}}; + default: + throw std::runtime_error("Unsupported Mac Vulkan surface format"); + } +} +struct Pools { + std::mutex mutex; + std::map, AVBufferRef*> frames; + ~Pools() { for (auto& pair : frames) av_buffer_unref(&pair.second); } +}; +Pools& NativePools() { static Pools pools; return pools; } +AVBufferRef* Pool(AVBufferRef* device, int width, int height, AVPixelFormat sw_format) { + auto& pools = NativePools(); + std::lock_guard lock(pools.mutex); + auto key = std::make_tuple(reinterpret_cast(device->data), width, height, sw_format); + auto& ref = pools.frames[key]; + if (!ref) { + ref = av_hwframe_ctx_alloc(device); + if (!ref) throw std::bad_alloc(); + auto* fc = reinterpret_cast(ref->data); + fc->format = AV_PIX_FMT_VULKAN; + fc->sw_format = sw_format; + fc->width = width; fc->height = height; + fc->user_opaque = const_cast(&native_pool_tag); + auto* vc = static_cast(fc->hwctx); + vc->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; + // These allocations only carry FFmpeg's synchronization; Wrap substitutes + // the imported images. 10-bit surfaces are only read (decoded video). + const bool wide = sw_format == AV_PIX_FMT_P010 || sw_format == AV_PIX_FMT_P210; + vc->usage = static_cast(VK_IMAGE_USAGE_SAMPLED_BIT | (wide ? 0 : VK_IMAGE_USAGE_STORAGE_BIT) | + VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); + const int result = av_hwframe_ctx_init(ref); + if (result < 0) { av_buffer_unref(&ref); pools.frames.erase(key); AvCheck(result, "Initialize Mac Vulkan frame pool"); } + } + AVBufferRef* result = av_buffer_ref(ref); + if (!result) throw std::bad_alloc(); + return result; +} +struct NativeFrame { + AVFrame* allocation = nullptr; + AVVkFrame original{}; + AVVkFrame* image = nullptr; + AVHWFramesContext* fc = nullptr; + AVVulkanFramesContext* vc = nullptr; + AVVulkanDeviceContext* vk = nullptr; + CVPixelBufferRef pixel = nullptr; + CVMetalTextureCacheRef cache = nullptr; + std::array textures{}; + std::array, 2> direct_textures{}; + id Texture(int plane) const { + return direct_textures[plane] ?: CVMetalTextureGetTexture(textures[plane]); + } + std::array imported{}; + uint64_t registry_id = 0; + int count = 0; + bool substituted = false; + void Wait() const { + vc->lock_frame(fc, image); + VkSemaphoreWaitInfo wait{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + wait.semaphoreCount = count; + wait.pSemaphores = image->sem; + wait.pValues = image->sem_value; + const auto result = vkWaitSemaphores(vk->act_dev, &wait, UINT64_MAX); + vc->unlock_frame(fc, image); + VkCheck(result, "Wait for Mac surface GPU access"); + } + ~NativeFrame() { + if (substituted) { + // Keep the normal FFmpeg allocation and its private synchronization + // internals intact. Its pool callback is the only supported way to + // release an AVVkFrame; no reliance on FFmpeg's private struct ABI. + try { Wait(); } catch (...) { /* Device loss: retire owned objects. */ } + } + if (vk) for (auto img : imported) if (img) vkDestroyImage(vk->act_dev, img, nullptr); + if (substituted) { + for (int i = 0; i < count; ++i) original.sem_value[i] = image->sem_value[i]; + *image = original; + } + av_frame_free(&allocation); + for (auto texture : textures) if (texture) CFRelease(texture); + for (auto texture : direct_textures) [texture release]; + if (cache) CFRelease(cache); + if (pixel) CVPixelBufferRelease(pixel); + } +}; +NativeFrame* State(const AVFrame* frame) { + if (!frame || frame->format != AV_PIX_FMT_VULKAN || !frame->hw_frames_ctx || !frame->buf[0]) return nullptr; + auto* fc = reinterpret_cast(frame->hw_frames_ctx->data); + if (fc->user_opaque != &native_pool_tag) return nullptr; + return static_cast(av_buffer_get_opaque(frame->buf[0])); +} +AVFrame* Wrap(AVBufferRef* device, CVPixelBufferRef pixel, AVPixelFormat sw_format, bool writable) { + const SurfaceFormat surface = Describe(sw_format); + const bool planar = surface.planes == 2; + @autoreleasepool { + const int width = static_cast(CVPixelBufferGetWidth(pixel)); + const int height = static_cast(CVPixelBufferGetHeight(pixel)); + if (width <= 0 || height <= 0 || !CVPixelBufferGetIOSurface(pixel)) + throw std::invalid_argument("Mac GPU frames require IOSurface-backed pixels"); + auto state = std::make_unique(); + state->pixel = CVPixelBufferRetain(pixel); + state->count = surface.planes; + state->allocation = av_frame_alloc(); + if (!state->allocation) throw std::bad_alloc(); + AVBufferRef* pool = Pool(device, width, height, sw_format); + const int allocated = av_hwframe_get_buffer(pool, state->allocation, 0); + av_buffer_unref(&pool); + AvCheck(allocated, "Allocate Mac Vulkan frame synchronization"); + state->fc = reinterpret_cast(state->allocation->hw_frames_ctx->data); + state->vc = static_cast(state->fc->hwctx); + state->vk = static_cast(state->fc->device_ctx->hwctx); + state->image = reinterpret_cast(state->allocation->data[0]); + state->original = *state->image; + auto exportObjects = reinterpret_cast(vkGetDeviceProcAddr(state->vk->act_dev, "vkExportMetalObjectsEXT")); + if (!exportObjects) throw std::runtime_error("Vulkan device lacks VK_EXT_metal_objects"); + VkExportMetalDeviceInfoEXT metal{VK_STRUCTURE_TYPE_EXPORT_METAL_DEVICE_INFO_EXT}; + VkExportMetalObjectsInfoEXT objects{VK_STRUCTURE_TYPE_EXPORT_METAL_OBJECTS_INFO_EXT}; + objects.pNext = &metal; + exportObjects(state->vk->act_dev, &objects); + state->registry_id = metal.mtlDevice.registryID; + if (!metal.mtlDevice) throw std::runtime_error("No Metal device for Vulkan"); + if (CVMetalTextureCacheCreate(kCFAllocatorDefault, nullptr, metal.mtlDevice, nullptr, &state->cache) != kCVReturnSuccess) + throw std::runtime_error("Create Core Video Metal texture cache"); + NSDictionary* attributes = @{(id)kCVMetalTextureUsage: @(MTLTextureUsageShaderRead | + (writable ? MTLTextureUsageShaderWrite : 0))}; + for (int i = 0; i < state->count; ++i) { + const auto w = planar ? CVPixelBufferGetWidthOfPlane(pixel, i) : size_t(width); + const auto h = planar ? CVPixelBufferGetHeightOfPlane(pixel, i) : size_t(height); + const auto format = surface.metal[i]; + if (planar) { + if (CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, state->cache, pixel, + (CFDictionaryRef)attributes, format, w, h, i, &state->textures[i]) != kCVReturnSuccess) + throw std::runtime_error("Map Core Video pixel planes to Metal"); + } else { + // Core Video cannot allocate Metal-compatible 32RGBA on older + // Macs. Allocate a 32BGRA IOSurface and give Metal an RGBA view; + // the compositor and CPU fallback both retain RGBA byte order. + auto* descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:format + width:w height:h mipmapped:NO]; + descriptor.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite; + state->direct_textures[i] = [metal.mtlDevice newTextureWithDescriptor:descriptor + iosurface:CVPixelBufferGetIOSurface(pixel) plane:i]; + if (!state->direct_textures[i]) throw std::runtime_error("Create RGBA IOSurface texture view"); + } + VkImportMetalTextureInfoEXT texture{VK_STRUCTURE_TYPE_IMPORT_METAL_TEXTURE_INFO_EXT}; + texture.plane = VK_IMAGE_ASPECT_COLOR_BIT; + texture.mtlTexture = state->Texture(i); + VkImageCreateInfo create{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + create.pNext = &texture; create.imageType = VK_IMAGE_TYPE_2D; + create.format = surface.vulkan[i]; + create.extent = {uint32_t(w), uint32_t(h), 1}; create.mipLevels = create.arrayLayers = 1; + create.samples = VK_SAMPLE_COUNT_1_BIT; create.tiling = VK_IMAGE_TILING_OPTIMAL; + create.usage = VK_IMAGE_USAGE_SAMPLED_BIT | (writable ? VK_IMAGE_USAGE_STORAGE_BIT : 0); + create.initialLayout = writable ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_PREINITIALIZED; + VkCheck(vkCreateImage(state->vk->act_dev, &create, nullptr, &state->imported[i]), "Import shared Mac texture into Vulkan"); + } + for (int i = 0; i < state->count; ++i) { + state->image->img[i] = state->imported[i]; + state->image->mem[i] = VK_NULL_HANDLE; + state->image->size[i] = planar ? CVPixelBufferGetBytesPerRowOfPlane(pixel, i) * CVPixelBufferGetHeightOfPlane(pixel, i) + : CVPixelBufferGetBytesPerRow(pixel) * height; + state->image->offset[i] = 0; + state->image->layout[i] = writable ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_PREINITIALIZED; + state->image->access[i] = static_cast(writable ? 0 : VK_ACCESS_MEMORY_WRITE_BIT); + state->image->queue_family[i] = VK_QUEUE_FAMILY_IGNORED; + } + state->substituted = true; + AVFrame* result = av_frame_alloc(); + if (!result) throw std::bad_alloc(); + result->format = AV_PIX_FMT_VULKAN; result->width = width; result->height = height; + result->hw_frames_ctx = av_buffer_ref(state->allocation->hw_frames_ctx); + result->data[0] = reinterpret_cast(state->image); + result->buf[0] = av_buffer_create(result->data[0], sizeof(AVVkFrame), + [](void* opaque, uint8_t*) { delete static_cast(opaque); }, state.get(), 0); + if (!result->hw_frames_ctx || !result->buf[0]) { + // Do not let the AVBuffer callback race the still-owning unique_ptr. + if (result->buf[0]) state.release(); + av_frame_free(&result); + throw std::bad_alloc(); + } + state.release(); + return result; + } +} +} +uint64_t MacVulkanDeviceRegistryID(const AVFrame* frame) { auto* state = State(frame); return state ? state->registry_id : 0; } +int64_t MacVulkanFrameBytes(const AVFrame* frame) { + auto* state = State(frame); + if (!state) return 0; + uint64_t size = CVPixelBufferGetDataSize(state->pixel); + // The FFmpeg synchronization allocation is retained alongside the imported + // surface. Include both, rather than letting GPU frames escape cache limits. + for (int i = 0; i < state->count; ++i) { + bool counted = false; + for (int j = 0; j < i; ++j) counted |= state->original.mem[j] == state->original.mem[i]; + if (!counted) size += state->original.size[i]; + } + return static_cast(std::min(size, std::numeric_limits::max())); +} +bool IsMacVulkanFrame(const AVFrame* frame) { return State(frame) != nullptr; } +AVPixelFormat MacSurfaceFormat(uint32_t core_video_format) { + switch (core_video_format) { + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + return AV_PIX_FMT_NV12; + // 10-bit HEVC/AV1 decode (iPhone HDR): 16-bit samples, data in the high bits. + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: + return AV_PIX_FMT_P010; + case kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_422YpCbCr10BiPlanarFullRange: + return AV_PIX_FMT_P210; + default: + return AV_PIX_FMT_NONE; + } +} +void WaitMacVulkanFrame(const AVFrame* frame) { + auto* state = State(frame); + if (!state) throw std::invalid_argument("Not a shared Mac Vulkan frame"); + state->Wait(); +} +void* MacVulkanPixelBuffer(const AVFrame* frame) { auto* state = State(frame); return state ? state->pixel : nullptr; } +AVFrame* ImportVideoToolboxFrame(const AVFrame* frame, bool writable) { + if (!frame || frame->format != AV_PIX_FMT_VIDEOTOOLBOX || !frame->data[3]) + throw std::invalid_argument("Expected a VideoToolbox frame"); + auto pixel = reinterpret_cast(frame->data[3]); + const AVPixelFormat sw_format = MacSurfaceFormat(CVPixelBufferGetPixelFormatType(pixel)); + if (sw_format == AV_PIX_FMT_NONE) + throw std::runtime_error("Mac Vulkan supports NV12, P010 and P210 decode/encode surfaces"); + AVBufferRef* device = nullptr; + AvCheck(CreateSharedVulkanDevice(&device), "Create Mac compositor device"); + AVFrame* result = nullptr; + try { result = Wrap(device, pixel, sw_format, writable); } catch (...) { av_buffer_unref(&device); throw; } + av_buffer_unref(&device); + const int props = av_frame_copy_props(result, frame); + if (props < 0) { av_frame_free(&result); AvCheck(props, "Copy VideoToolbox metadata"); } + return result; +} +AVFrame* AllocateMacVulkanFrame(AVBufferRef* device, int width, int height, bool nv12) { + @autoreleasepool { + NSDictionary* attrs = @{(id)kCVPixelBufferMetalCompatibilityKey: @YES, (id)kCVPixelBufferIOSurfacePropertiesKey: @{}}; + CVPixelBufferRef pixel = nullptr; + const auto type = nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_32BGRA; + if (CVPixelBufferCreate(kCFAllocatorDefault, width, height, type, (CFDictionaryRef)attrs, &pixel) != kCVReturnSuccess) + throw std::runtime_error("Allocate shared compositor IOSurface"); + AVFrame* frame = nullptr; + try { frame = Wrap(device, pixel, nv12 ? AV_PIX_FMT_NV12 : AV_PIX_FMT_RGBA, true); } catch (...) { CVPixelBufferRelease(pixel); throw; } + CVPixelBufferRelease(pixel); + return frame; + } +} +int ReadMacVulkanFrame(AVFrame* destination, const AVFrame* source) { + auto* state = State(source); + if (!state) return AVERROR(EINVAL); + try { state->Wait(); } catch (...) { return AVERROR_EXTERNAL; } + destination->format = state->fc->sw_format; + destination->width = source->width; destination->height = source->height; + int result = av_frame_get_buffer(destination, 32); + if (result < 0) return result; + // Managed Metal textures need an explicit GPU-to-CPU synchronization for + // materialization. This is intentionally absent from preview and export. + @autoreleasepool { + auto first = state->Texture(0); + id queue = [first.device newCommandQueue]; + if (!queue) return AVERROR_EXTERNAL; + id command = [queue commandBuffer]; + id blit = [command blitCommandEncoder]; + for (int i = 0; i < state->count; ++i) { + auto texture = state->Texture(i); + if (texture.storageMode == MTLStorageModeManaged) [blit synchronizeResource:texture]; + } + [blit endEncoding]; [command commit]; [command waitUntilCompleted]; + const bool completed = command.status == MTLCommandBufferStatusCompleted; + [queue release]; + if (!completed) return AVERROR_EXTERNAL; + } + if (CVPixelBufferLockBaseAddress(state->pixel, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) return AVERROR_EXTERNAL; + const bool planar = state->count == 2; + const uint8_t* data[4]{}; int strides[4]{}; + for (int i = 0; i < state->count; ++i) { + data[i] = static_cast(planar ? CVPixelBufferGetBaseAddressOfPlane(state->pixel, i) : CVPixelBufferGetBaseAddress(state->pixel)); + strides[i] = static_cast(planar ? CVPixelBufferGetBytesPerRowOfPlane(state->pixel, i) : CVPixelBufferGetBytesPerRow(state->pixel)); + } + av_image_copy(destination->data, destination->linesize, data, strides, AVPixelFormat(destination->format), source->width, source->height); + CVPixelBufferUnlockBaseAddress(state->pixel, kCVPixelBufferLock_ReadOnly); + return 0; +} +void ReleaseMacVulkanPools() { + auto& pools = NativePools(); std::lock_guard lock(pools.mutex); + for (auto& pair : pools.frames) av_buffer_unref(&pair.second); + pools.frames.clear(); +} +} diff --git a/src/OpenShotVersion.h.in b/src/OpenShotVersion.h.in index 7452cba01..3c03d9212 100644 --- a/src/OpenShotVersion.h.in +++ b/src/OpenShotVersion.h.in @@ -34,6 +34,10 @@ #cmakedefine01 HAVE_OPENCV #cmakedefine01 FFMPEG_USE_SWRESAMPLE #cmakedefine01 APPIMAGE_BUILD +#cmakedefine ENABLE_GPU_FRAME +#ifdef ENABLE_GPU_FRAME +#define OPENSHOT_ENABLE_GPU_FRAME 1 +#endif #include diff --git a/src/Qt/ExternalOpenGL.h b/src/Qt/ExternalOpenGL.h new file mode 100644 index 000000000..14296e97f --- /dev/null +++ b/src/Qt/ExternalOpenGL.h @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: MIT +/* +** Copyright (c) 2013-2018 The Khronos Group Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and/or associated documentation files (the +** "Materials"), to deal in the Materials without restriction, including +** without limitation the rights to use, copy, modify, merge, publish, +** distribute, sublicense, and/or sell copies of the Materials, and to +** permit persons to whom the Materials are furnished to do so, subject to +** the following conditions: +** +** The above copyright notice and this permission notice shall be included +** in all copies or substantial portions of the Materials. +** +** THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +** MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS. +*/ +// Subset of Khronos OpenGL-Registry api/GL/glext.h for Qt 5 SDKs whose +// extension headers predate external memory and semaphore interop. +#ifndef OPENSHOT_EXTERNAL_OPENGL_H +#define OPENSHOT_EXTERNAL_OPENGL_H +#include + +#ifndef GL_EXT_memory_object +#define GL_EXT_memory_object 1 +#define GL_TEXTURE_TILING_EXT 0x9580 +#define GL_DEDICATED_MEMORY_OBJECT_EXT 0x9581 +#define GL_PROTECTED_MEMORY_OBJECT_EXT 0x959B +#define GL_NUM_TILING_TYPES_EXT 0x9582 +#define GL_TILING_TYPES_EXT 0x9583 +#define GL_OPTIMAL_TILING_EXT 0x9584 +#define GL_LINEAR_TILING_EXT 0x9585 +#define GL_NUM_DEVICE_UUIDS_EXT 0x9596 +#define GL_DEVICE_UUID_EXT 0x9597 +#define GL_DRIVER_UUID_EXT 0x9598 +#define GL_UUID_SIZE_EXT 16 +typedef void (APIENTRYP PFNGLGETUNSIGNEDBYTEVEXTPROC) (GLenum pname, GLubyte *data); +typedef void (APIENTRYP PFNGLGETUNSIGNEDBYTEI_VEXTPROC) (GLenum target, GLuint index, GLubyte *data); +typedef void (APIENTRYP PFNGLDELETEMEMORYOBJECTSEXTPROC) (GLsizei n, const GLuint *memoryObjects); +typedef GLboolean (APIENTRYP PFNGLISMEMORYOBJECTEXTPROC) (GLuint memoryObject); +typedef void (APIENTRYP PFNGLCREATEMEMORYOBJECTSEXTPROC) (GLsizei n, GLuint *memoryObjects); +typedef void (APIENTRYP PFNGLMEMORYOBJECTPARAMETERIVEXTPROC) (GLuint memoryObject, GLenum pname, const GLint *params); +typedef void (APIENTRYP PFNGLGETMEMORYOBJECTPARAMETERIVEXTPROC) (GLuint memoryObject, GLenum pname, GLint *params); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM2DEXTPROC) (GLenum target, GLsizei levels, GLenum internalFormat, GLsizei width, GLsizei height, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM2DMULTISAMPLEEXTPROC) (GLenum target, GLsizei samples, GLenum internalFormat, GLsizei width, GLsizei height, GLboolean fixedSampleLocations, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM3DEXTPROC) (GLenum target, GLsizei levels, GLenum internalFormat, GLsizei width, GLsizei height, GLsizei depth, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM3DMULTISAMPLEEXTPROC) (GLenum target, GLsizei samples, GLenum internalFormat, GLsizei width, GLsizei height, GLsizei depth, GLboolean fixedSampleLocations, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLBUFFERSTORAGEMEMEXTPROC) (GLenum target, GLsizeiptr size, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM2DEXTPROC) (GLuint texture, GLsizei levels, GLenum internalFormat, GLsizei width, GLsizei height, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM2DMULTISAMPLEEXTPROC) (GLuint texture, GLsizei samples, GLenum internalFormat, GLsizei width, GLsizei height, GLboolean fixedSampleLocations, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM3DEXTPROC) (GLuint texture, GLsizei levels, GLenum internalFormat, GLsizei width, GLsizei height, GLsizei depth, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM3DMULTISAMPLEEXTPROC) (GLuint texture, GLsizei samples, GLenum internalFormat, GLsizei width, GLsizei height, GLsizei depth, GLboolean fixedSampleLocations, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLNAMEDBUFFERSTORAGEMEMEXTPROC) (GLuint buffer, GLsizeiptr size, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM1DEXTPROC) (GLenum target, GLsizei levels, GLenum internalFormat, GLsizei width, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM1DEXTPROC) (GLuint texture, GLsizei levels, GLenum internalFormat, GLsizei width, GLuint memory, GLuint64 offset); +#endif /* GL_EXT_memory_object */ + +#ifndef GL_EXT_memory_object_fd +#define GL_EXT_memory_object_fd 1 +#define GL_HANDLE_TYPE_OPAQUE_FD_EXT 0x9586 +typedef void (APIENTRYP PFNGLIMPORTMEMORYFDEXTPROC) (GLuint memory, GLuint64 size, GLenum handleType, GLint fd); +#endif /* GL_EXT_memory_object_fd */ + +#ifndef GL_EXT_memory_object_win32 +#define GL_EXT_memory_object_win32 1 +#define GL_HANDLE_TYPE_OPAQUE_WIN32_EXT 0x9587 +#define GL_HANDLE_TYPE_OPAQUE_WIN32_KMT_EXT 0x9588 +#define GL_DEVICE_LUID_EXT 0x9599 +#define GL_DEVICE_NODE_MASK_EXT 0x959A +#define GL_LUID_SIZE_EXT 8 +#define GL_HANDLE_TYPE_D3D12_TILEPOOL_EXT 0x9589 +#define GL_HANDLE_TYPE_D3D12_RESOURCE_EXT 0x958A +#define GL_HANDLE_TYPE_D3D11_IMAGE_EXT 0x958B +#define GL_HANDLE_TYPE_D3D11_IMAGE_KMT_EXT 0x958C +typedef void (APIENTRYP PFNGLIMPORTMEMORYWIN32HANDLEEXTPROC) (GLuint memory, GLuint64 size, GLenum handleType, void *handle); +typedef void (APIENTRYP PFNGLIMPORTMEMORYWIN32NAMEEXTPROC) (GLuint memory, GLuint64 size, GLenum handleType, const void *name); +#endif /* GL_EXT_memory_object_win32 */ + +#ifndef GL_EXT_semaphore +#define GL_EXT_semaphore 1 +#define GL_LAYOUT_GENERAL_EXT 0x958D +#define GL_LAYOUT_COLOR_ATTACHMENT_EXT 0x958E +#define GL_LAYOUT_DEPTH_STENCIL_ATTACHMENT_EXT 0x958F +#define GL_LAYOUT_DEPTH_STENCIL_READ_ONLY_EXT 0x9590 +#define GL_LAYOUT_SHADER_READ_ONLY_EXT 0x9591 +#define GL_LAYOUT_TRANSFER_SRC_EXT 0x9592 +#define GL_LAYOUT_TRANSFER_DST_EXT 0x9593 +#define GL_LAYOUT_DEPTH_READ_ONLY_STENCIL_ATTACHMENT_EXT 0x9530 +#define GL_LAYOUT_DEPTH_ATTACHMENT_STENCIL_READ_ONLY_EXT 0x9531 +typedef void (APIENTRYP PFNGLGENSEMAPHORESEXTPROC) (GLsizei n, GLuint *semaphores); +typedef void (APIENTRYP PFNGLDELETESEMAPHORESEXTPROC) (GLsizei n, const GLuint *semaphores); +typedef GLboolean (APIENTRYP PFNGLISSEMAPHOREEXTPROC) (GLuint semaphore); +typedef void (APIENTRYP PFNGLSEMAPHOREPARAMETERUI64VEXTPROC) (GLuint semaphore, GLenum pname, const GLuint64 *params); +typedef void (APIENTRYP PFNGLGETSEMAPHOREPARAMETERUI64VEXTPROC) (GLuint semaphore, GLenum pname, GLuint64 *params); +typedef void (APIENTRYP PFNGLWAITSEMAPHOREEXTPROC) (GLuint semaphore, GLuint numBufferBarriers, const GLuint *buffers, GLuint numTextureBarriers, const GLuint *textures, const GLenum *srcLayouts); +typedef void (APIENTRYP PFNGLSIGNALSEMAPHOREEXTPROC) (GLuint semaphore, GLuint numBufferBarriers, const GLuint *buffers, GLuint numTextureBarriers, const GLuint *textures, const GLenum *dstLayouts); +#endif /* GL_EXT_semaphore */ + +#ifndef GL_EXT_semaphore_fd +#define GL_EXT_semaphore_fd 1 +typedef void (APIENTRYP PFNGLIMPORTSEMAPHOREFDEXTPROC) (GLuint semaphore, GLenum handleType, GLint fd); +#endif /* GL_EXT_semaphore_fd */ + +#ifndef GL_EXT_semaphore_win32 +#define GL_EXT_semaphore_win32 1 +#define GL_HANDLE_TYPE_D3D12_FENCE_EXT 0x9594 +#define GL_D3D12_FENCE_VALUE_EXT 0x9595 +typedef void (APIENTRYP PFNGLIMPORTSEMAPHOREWIN32HANDLEEXTPROC) (GLuint semaphore, GLenum handleType, void *handle); +typedef void (APIENTRYP PFNGLIMPORTSEMAPHOREWIN32NAMEEXTPROC) (GLuint semaphore, GLenum handleType, const void *name); +#endif /* GL_EXT_semaphore_win32 */ +#endif diff --git a/src/Qt/VideoRenderer.cpp b/src/Qt/VideoRenderer.cpp index fdedcdb38..c5173c002 100644 --- a/src/Qt/VideoRenderer.cpp +++ b/src/Qt/VideoRenderer.cpp @@ -12,11 +12,15 @@ #include "VideoRenderer.h" #include +#include #include VideoRenderer::VideoRenderer(QObject *parent) : QObject(parent), override_widget(nullptr) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + qRegisterMetaType>(); +#endif } VideoRenderer::~VideoRenderer() @@ -26,6 +30,9 @@ VideoRenderer::~VideoRenderer() /// Override QWidget which needs to be painted void VideoRenderer::OverrideWidget(uintptr_t qwidget_address) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + frame_presenter.store(false); +#endif if (override_present_connection) QObject::disconnect(override_present_connection); @@ -34,6 +41,24 @@ void VideoRenderer::OverrideWidget(uintptr_t qwidget_address) if (!override_widget) return; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + QObject* presenter_object = override_widget; + auto presenter = dynamic_cast(presenter_object); + if (!presenter) { + presenter_object = override_widget->property("openshot_gpu_presenter").value(); + presenter = dynamic_cast(presenter_object); + } + if (presenter) { + override_present_connection = QObject::connect( + this, &VideoRenderer::presentFrame, presenter_object, + [presenter](std::shared_ptr frame) { + presenter->PresentFrame(frame); + }, Qt::QueuedConnection); + frame_presenter.store(true); + return; + } +#endif + override_present_connection = QObject::connect( this, &VideoRenderer::present, override_widget, @@ -55,3 +80,13 @@ void VideoRenderer::render(std::shared_ptr image) emit present(*image); } + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +void VideoRenderer::renderFrame(const std::shared_ptr& frame) +{ + if (frame_presenter.load()) + emit presentFrame(frame); + else + openshot::RendererBase::renderFrame(frame); +} +#endif diff --git a/src/Qt/VideoRenderer.h b/src/Qt/VideoRenderer.h index 1bdbdc893..a1db7ee34 100644 --- a/src/Qt/VideoRenderer.h +++ b/src/Qt/VideoRenderer.h @@ -19,11 +19,23 @@ #include #include #include +#include class QPainter; class QWidget; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +/// Implement on a native QWidget to receive retained frames on the GUI thread. +/// The presenter owns presentation synchronization and may retain the frame. +class FramePresenter { +public: + virtual ~FramePresenter() = default; + virtual void PresentFrame(const std::shared_ptr& frame) = 0; +}; +Q_DECLARE_METATYPE(std::shared_ptr) +#endif + class VideoRenderer : public QObject, public openshot::RendererBase { Q_OBJECT @@ -37,16 +49,25 @@ class VideoRenderer : public QObject, public openshot::RendererBase signals: void present(const QImage &image); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + void presentFrame(std::shared_ptr frame); +#endif protected: //void render(openshot::OSPixelFormat format, int width, int height, int bytesPerLine, unsigned char *data); void render(std::shared_ptr image); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + void renderFrame(const std::shared_ptr& frame) override; +#endif private slots: private: QWidget* override_widget; QMetaObject::Connection override_present_connection; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + std::atomic frame_presenter{false}; +#endif }; #endif //OPENSHOT_VIDEO_RENDERER_H diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp new file mode 100644 index 000000000..752476a63 --- /dev/null +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -0,0 +1,838 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "VulkanOpenGLPresenter.h" +#include "VideoRenderer.h" +#include "../VulkanFrameImage.h" +#include "../VulkanCompositor.h" +#include "../GpuDevices.h" +#include "../GpuDebug.h" +#include "../Logger.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ExternalOpenGL.h" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#define VK_USE_PLATFORM_WIN32_KHR +#else +#include +#endif +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} +#include "../VulkanLoader.h" + +namespace openshot { +namespace { +void check(VkResult result, const char* operation) { + if (result == VK_SUCCESS) return; + ReportVulkanResult(result); + throw std::runtime_error(std::string(operation) + ": " + VulkanResultName(result)); +} +// Qt shares this context with its widget texture cache. Leaving our video +// texture bound can interfere with later Qt popup/overlay texture uploads. +class TextureBindingGuard { + QOpenGLFunctions* gl; + GLint active = GL_TEXTURE0; + GLint texture = 0; +public: + explicit TextureBindingGuard(QOpenGLFunctions* functions) : gl(functions) { + gl->glGetIntegerv(GL_ACTIVE_TEXTURE, &active); + gl->glActiveTexture(GL_TEXTURE0); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &texture); + } + ~TextureBindingGuard() { + gl->glActiveTexture(GL_TEXTURE0); + gl->glBindTexture(GL_TEXTURE_2D, texture); + gl->glActiveTexture(active); + } +}; +template T glFunction(QOpenGLContext* context, const char* name) { + auto function = reinterpret_cast(context->getProcAddress(name)); + if (!function) throw std::runtime_error(name); + return function; +} +#ifdef _WIN32 +// GpuDebug: OPENSHOT_PRESENTATION_HANDLE=kmt shares through the older global +// (KMT) handles, which are never closed, instead of NT handles. +const bool kKmtHandles = GpuDebugSetting("OPENSHOT_PRESENTATION_HANDLE") == "kmt"; +const auto kMemoryHandle = kKmtHandles ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT + : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT; +const auto kSemaphoreHandle = kKmtHandles ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT + : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT; +const GLenum kGLHandle = kKmtHandles ? GL_HANDLE_TYPE_OPAQUE_WIN32_KMT_EXT : GL_HANDLE_TYPE_OPAQUE_WIN32_EXT; +constexpr const char* kMemoryExtension = "GL_EXT_memory_object_win32"; +constexpr const char* kSemaphoreExtension = "GL_EXT_semaphore_win32"; +using ImportMemory = PFNGLIMPORTMEMORYWIN32HANDLEEXTPROC; +using ImportSemaphore = PFNGLIMPORTSEMAPHOREWIN32HANDLEEXTPROC; +#else +constexpr auto kMemoryHandle = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; +constexpr auto kSemaphoreHandle = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; +constexpr GLenum kGLHandle = GL_HANDLE_TYPE_OPAQUE_FD_EXT; +constexpr const char* kMemoryExtension = "GL_EXT_memory_object_fd"; +constexpr const char* kSemaphoreExtension = "GL_EXT_semaphore_fd"; +using ImportMemory = PFNGLIMPORTMEMORYFDEXTPROC; +using ImportSemaphore = PFNGLIMPORTSEMAPHOREFDEXTPROC; +#endif +constexpr VkImageSubresourceRange kColor = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; +std::string pixelText(const unsigned char* pixel) { + char text[16]; + std::snprintf(text, sizeof(text), "#%02x%02x%02x%02x", pixel[0], pixel[1], pixel[2], pixel[3]); + return text; +} +} +// Frames reach OpenGL through a presenter-owned linear buffer, never through +// shared image memory. NVIDIA's Windows driver returns zeros when FFmpeg or +// compositor frames are read through an exported alias of their memory, and +// Maxwell GPUs raise a TEX LAYOUT exception (then a TDR) when OpenGL samples +// an imported optimal-tiling Vulkan image. Vulkan copies each frame into the +// buffer, and OpenGL unpacks it into its own texture. The buffer and its +// OpenGL import persist until the frame size, device, or context changes. +struct SharedTarget { + AVBufferRef* device_ref = nullptr; + VkDevice device = VK_NULL_HANDLE; + QPointer context; + int width = 0, height = 0; + uint32_t family = 0; + VkQueue queue = VK_NULL_HANDLE; + VkBuffer pixels = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + // Separate directions: reusing one binary semaphore lets Vulkan's + // acquire consume its own release signal before GL can wait on it. + VkSemaphore handoff[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + VkCommandPool pool = VK_NULL_HANDLE; + VkCommandBuffer commands[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + VkFence fence = VK_NULL_HANDLE; + bool pending = false; + GLuint gl_memory = 0, gl_pixels = 0, texture = 0, gl_semaphores[2] = {0, 0}, buffer = 0, vertex_array = 0; + std::unique_ptr program; + PFNGLWAITSEMAPHOREEXTPROC wait_semaphore = nullptr; + PFNGLSIGNALSEMAPHOREEXTPROC signal_semaphore = nullptr; + // GpuDebug: a host-visible copy of the frame's centre pixel. + VkBuffer probe = VK_NULL_HANDLE; + VkDeviceMemory probe_memory = VK_NULL_HANDLE; + const unsigned char* probe_pixel = nullptr; +#ifdef _WIN32 + // Imported Win32 handles stay open while OpenGL uses them: a driver may + // resolve the handle after the import call returns. + std::vector handles; +#endif +}; +struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { + QPointer widget; + std::shared_ptr frame; + std::shared_ptr display_surface; + bool fallback_delivered = false; + uint64_t native_draws = 0; + uint64_t fallback_frames = 0; + std::string last_failure; + // After a failed draw the CPU shows frames until retry_after, and each + // further failure in a row doubles the wait (up to 30 seconds). + int failures_in_a_row = 0; + std::chrono::steady_clock::time_point retry_after; + std::string paused_reason; + QPointer logged_context; + bool identity_logged = false; + uint64_t debug_frames = 0; + std::string debug_geometry; + SharedTarget target; + bool unavailable(const std::string& reason) { last_failure = reason; return false; } + explicit Impl(QWidget* value) : widget(value) { + value->setProperty("openshot_gpu_presenter", QVariant::fromValue(static_cast(this))); + } + ~Impl() override { + releaseTarget(); + if (widget) widget->setProperty("openshot_gpu_presenter", QVariant()); + } + void PresentFrame(const std::shared_ptr& value) override { + frame = value; + display_surface.reset(); + fallback_delivered = false; + if (frame && !frame->GetDeferredImage()) { + last_failure = "Frame is CPU-backed (decode, rendering, or an effect required CPU pixels)"; + fallback(); + } + if (widget) { + if (frame && frame->GetDeferredImage() && widget->metaObject()->indexOfMethod("gpu_presented(int)") >= 0) + QMetaObject::invokeMethod(widget, "gpu_presented", Qt::DirectConnection, + Q_ARG(int, static_cast(frame->number))); + widget->update(); + } + } + void fallback() { + if (!fallback_delivered && frame && widget) { + fallback_delivered = true; + ++fallback_frames; + display_surface.reset(); + // A lost device cannot provide CPU pixels. Let the GUI clear its + // caches and request a fresh CPU-rendered frame instead. + if (VulkanDeviceLost() && frame->GetDeferredImage()) return; + try { + const QImage image = *frame->GetImage(); + QMetaObject::invokeMethod(widget, "present", Qt::QueuedConnection, Q_ARG(QImage, image)); + } catch (const std::exception& error) { + last_failure = std::string("GPU preview fallback could not read the frame: ") + error.what(); + Logger::Instance()->Log(last_failure, Logger::LevelWarning); + } + } + } + void waitPending() { + if (!target.pending) return; + target.pending = false; + check(vkWaitForFences(target.device, 1, &target.fence, VK_TRUE, UINT64_MAX), "Wait for presentation transfer"); + check(vkResetFences(target.device, 1, &target.fence), "Reset presentation fence"); + } + // OpenGL objects are deleted only while their context is current; otherwise + // they are released with that context. + void releaseTarget() { + if (target.device) { + try { waitPending(); } catch (const std::exception&) { vkDeviceWaitIdle(target.device); } + } + auto* context = QOpenGLContext::currentContext(); + if (target.context && context == target.context) { + auto gl = context->functions(); + if (target.vertex_array) context->extraFunctions()->glDeleteVertexArrays(1, &target.vertex_array); + if (target.buffer) gl->glDeleteBuffers(1, &target.buffer); + if (target.gl_pixels) gl->glDeleteBuffers(1, &target.gl_pixels); + if (target.texture) gl->glDeleteTextures(1, &target.texture); + if (target.gl_memory) + reinterpret_cast(context->getProcAddress("glDeleteMemoryObjectsEXT"))(1, &target.gl_memory); + if (target.gl_semaphores[0] || target.gl_semaphores[1]) + reinterpret_cast(context->getProcAddress("glDeleteSemaphoresEXT"))(2, target.gl_semaphores); + } + target.program.reset(); // Qt defers deletion until the context is current. + if (target.device) { + if (target.fence) vkDestroyFence(target.device, target.fence, nullptr); + if (target.pool) vkDestroyCommandPool(target.device, target.pool, nullptr); + for (auto semaphore : target.handoff) + if (semaphore) vkDestroySemaphore(target.device, semaphore, nullptr); + if (target.pixels) vkDestroyBuffer(target.device, target.pixels, nullptr); + if (target.memory) vkFreeMemory(target.device, target.memory, nullptr); + if (target.probe) vkDestroyBuffer(target.device, target.probe, nullptr); + if (target.probe_memory) vkFreeMemory(target.device, target.probe_memory, nullptr); + } +#ifdef _WIN32 + for (auto handle : target.handles) CloseHandle(handle); +#endif + av_buffer_unref(&target.device_ref); + target = SharedTarget(); + } + // GpuDebug: a 4-byte host-visible buffer that receives the frame's centre pixel. + void createProbe(const VkPhysicalDeviceMemoryProperties& memoryProperties) { + VkBufferCreateInfo create{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + create.size = 4; + create.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + check(vkCreateBuffer(target.device, &create, nullptr, &target.probe), "Create debug probe"); + VkMemoryRequirements needs; + vkGetBufferMemoryRequirements(target.device, target.probe, &needs); + const VkMemoryPropertyFlags wanted = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocate.allocationSize = needs.size; + allocate.memoryTypeIndex = UINT32_MAX; + for (uint32_t i = 0; i < memoryProperties.memoryTypeCount && allocate.memoryTypeIndex == UINT32_MAX; ++i) + if ((needs.memoryTypeBits & (1u << i)) && (memoryProperties.memoryTypes[i].propertyFlags & wanted) == wanted) + allocate.memoryTypeIndex = i; + if (allocate.memoryTypeIndex == UINT32_MAX) throw std::runtime_error("No host-visible debug probe memory"); + check(vkAllocateMemory(target.device, &allocate, nullptr, &target.probe_memory), "Allocate debug probe"); + check(vkBindBufferMemory(target.device, target.probe, target.probe_memory, 0), "Bind debug probe"); + void* mapped = nullptr; + check(vkMapMemory(target.device, target.probe_memory, 0, VK_WHOLE_SIZE, 0, &mapped), "Map debug probe"); + target.probe_pixel = static_cast(mapped); + } + void createTarget(QOpenGLContext* context, const AVHWFramesContext* fc, int width, int height) { + auto vk = static_cast(fc->device_ctx->hwctx); + target.device_ref = av_buffer_ref(fc->device_ref); + if (!target.device_ref) throw std::bad_alloc(); + target.device = vk->act_dev; + target.context = context; + target.width = width; + target.height = height; + target.family = vk->queue_family_comp_index; + vkGetDeviceQueue(target.device, target.family, 0, &target.queue); + auto gl = context->functions(); + auto extra = context->extraFunctions(); + auto createMemory = glFunction(context, "glCreateMemoryObjectsEXT"); + auto memoryParameter = glFunction(context, "glMemoryObjectParameterivEXT"); + auto bufferMemory = glFunction(context, "glBufferStorageMemEXT"); + auto createSemaphores = glFunction(context, "glGenSemaphoresEXT"); + glFunction(context, "glDeleteMemoryObjectsEXT"); + glFunction(context, "glDeleteSemaphoresEXT"); + target.wait_semaphore = glFunction(context, "glWaitSemaphoreEXT"); + target.signal_semaphore = glFunction(context, "glSignalSemaphoreEXT"); +#ifdef _WIN32 + auto importMemory = glFunction(context, "glImportMemoryWin32HandleEXT"); + auto importSemaphore = glFunction(context, "glImportSemaphoreWin32HandleEXT"); + auto getMemoryHandle = reinterpret_cast(vkGetDeviceProcAddr(target.device, "vkGetMemoryWin32HandleKHR")); + auto getSemaphoreHandle = reinterpret_cast(vkGetDeviceProcAddr(target.device, "vkGetSemaphoreWin32HandleKHR")); +#else + auto importMemory = glFunction(context, "glImportMemoryFdEXT"); + auto importSemaphore = glFunction(context, "glImportSemaphoreFdEXT"); + auto getMemoryHandle = reinterpret_cast(vkGetDeviceProcAddr(target.device, "vkGetMemoryFdKHR")); + auto getSemaphoreHandle = reinterpret_cast(vkGetDeviceProcAddr(target.device, "vkGetSemaphoreFdKHR")); +#endif + if (!getMemoryHandle || !getSemaphoreHandle) + throw std::runtime_error("Vulkan device lacks external memory/semaphore export"); + + const VkDeviceSize size = VkDeviceSize(width) * height * 4; + VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; + external.handleTypes = kMemoryHandle; + VkBufferCreateInfo create{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; + create.size = size; + create.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + check(vkCreateBuffer(target.device, &create, nullptr, &target.pixels), "Create presentation buffer"); + VkMemoryDedicatedRequirements dedicated{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; + VkMemoryRequirements2 requirements{VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, &dedicated}; + VkBufferMemoryRequirementsInfo2 requirementInfo{VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2}; + requirementInfo.buffer = target.pixels; + vkGetBufferMemoryRequirements2(target.device, &requirementInfo, &requirements); + VkPhysicalDeviceMemoryProperties memoryProperties; + vkGetPhysicalDeviceMemoryProperties(vk->phys_dev, &memoryProperties); + uint32_t memoryType = UINT32_MAX; + for (uint32_t i = 0; i < memoryProperties.memoryTypeCount && memoryType == UINT32_MAX; ++i) + if ((requirements.memoryRequirements.memoryTypeBits & (1u << i)) && + (memoryProperties.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) + memoryType = i; + if (memoryType == UINT32_MAX) throw std::runtime_error("No device-local presentation memory"); + bool useDedicated = dedicated.prefersDedicatedAllocation || dedicated.requiresDedicatedAllocation; + // GpuDebug: OPENSHOT_PRESENTATION_DEDICATED=0|1 overrides the driver's preference. + const auto forceDedicated = GpuDebugSetting("OPENSHOT_PRESENTATION_DEDICATED"); + if (forceDedicated == "1") + useDedicated = true; + else if (forceDedicated == "0" && !dedicated.requiresDedicatedAllocation) + useDedicated = false; + VkMemoryDedicatedAllocateInfo dedicatedInfo{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO}; + dedicatedInfo.buffer = target.pixels; + VkExportMemoryAllocateInfo exportInfo{VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, useDedicated ? &dedicatedInfo : nullptr}; + exportInfo.handleTypes = kMemoryHandle; + VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &exportInfo}; + allocate.allocationSize = requirements.memoryRequirements.size; + allocate.memoryTypeIndex = memoryType; + check(vkAllocateMemory(target.device, &allocate, nullptr, &target.memory), "Allocate presentation memory"); + check(vkBindBufferMemory(target.device, target.pixels, target.memory, 0), "Bind presentation memory"); + if (GpuDebug()) { + const auto& type = memoryProperties.memoryTypes[memoryType]; + char flags[64]; + std::snprintf(flags, sizeof(flags), "0x%x; heap %u flags=0x%x", type.propertyFlags, type.heapIndex, + memoryProperties.memoryHeaps[type.heapIndex].flags); + Logger::Instance()->Log("GPU debug presentation buffer: " + std::to_string(width) + "x" + + std::to_string(height) + "; size=" + std::to_string(size) + "; allocation=" + + std::to_string(requirements.memoryRequirements.size) + "; memory type " + std::to_string(memoryType) + + " flags=" + flags + "; dedicated prefers=" + std::to_string(dedicated.prefersDedicatedAllocation) + + " requires=" + std::to_string(dedicated.requiresDedicatedAllocation) + " used=" + + std::to_string(useDedicated) + "; override=" + forceDedicated.toStdString() +#ifdef _WIN32 + + "; handles=" + (kKmtHandles ? "kmt" : "nt") +#endif + , Logger::LevelInfo); + createProbe(memoryProperties); + } + // GpuDebug: check OpenGL errors after each import step, not only at the end. + GLenum importError = GL_NO_ERROR; + const auto glStep = [&](const char* name) { + if (!GpuDebug()) return; + const GLenum error = context->functions()->glGetError(); + if (importError == GL_NO_ERROR) importError = error; + Logger::Instance()->Log(std::string("GPU debug OpenGL ") + name + ": error=" + std::to_string(error), + Logger::LevelInfo); + }; + + createMemory(1, &target.gl_memory); + GLint glDedicated = useDedicated; + memoryParameter(target.gl_memory, GL_DEDICATED_MEMORY_OBJECT_EXT, &glDedicated); + glStep("create memory object"); +#ifdef _WIN32 + VkMemoryGetWin32HandleInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR}; + memoryInfo.memory = target.memory; + memoryInfo.handleType = kMemoryHandle; + HANDLE memoryHandle = nullptr; + check(getMemoryHandle(target.device, &memoryInfo, &memoryHandle), "Export presentation memory"); + importMemory(target.gl_memory, requirements.memoryRequirements.size, kGLHandle, memoryHandle); + if (!kKmtHandles) target.handles.push_back(memoryHandle); + glStep("import memory"); +#else + VkMemoryGetFdInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; + memoryInfo.memory = target.memory; + memoryInfo.handleType = kMemoryHandle; + int memoryFd = -1; + check(getMemoryHandle(target.device, &memoryInfo, &memoryFd), "Export presentation memory"); + importMemory(target.gl_memory, requirements.memoryRequirements.size, kGLHandle, memoryFd); // GL owns the fd. + glStep("import memory"); +#endif + GLint previousUnpack = 0, previousTexture = 0; + gl->glGetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &previousUnpack); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &previousTexture); + gl->glGenBuffers(1, &target.gl_pixels); + gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, target.gl_pixels); + bufferMemory(GL_PIXEL_UNPACK_BUFFER, size, target.gl_memory, 0); + glStep("buffer storage"); + gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, previousUnpack); + gl->glGenTextures(1, &target.texture); + gl->glBindTexture(GL_TEXTURE_2D, target.texture); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + gl->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + glStep("texture"); + gl->glBindTexture(GL_TEXTURE_2D, previousTexture); + + VkExportSemaphoreCreateInfo exportSemaphore{VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO}; + exportSemaphore.handleTypes = kSemaphoreHandle; + VkSemaphoreCreateInfo semaphoreInfo{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &exportSemaphore}; + createSemaphores(2, target.gl_semaphores); + for (int direction = 0; direction < 2; ++direction) { + check(vkCreateSemaphore(target.device, &semaphoreInfo, nullptr, &target.handoff[direction]), "Create presentation semaphore"); +#ifdef _WIN32 + VkSemaphoreGetWin32HandleInfoKHR handleInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR}; + handleInfo.semaphore = target.handoff[direction]; + handleInfo.handleType = kSemaphoreHandle; + HANDLE semaphoreHandle = nullptr; + check(getSemaphoreHandle(target.device, &handleInfo, &semaphoreHandle), "Export presentation semaphore"); + importSemaphore(target.gl_semaphores[direction], kGLHandle, semaphoreHandle); + if (!kKmtHandles) target.handles.push_back(semaphoreHandle); +#else + VkSemaphoreGetFdInfoKHR handleInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR}; + handleInfo.semaphore = target.handoff[direction]; + handleInfo.handleType = kSemaphoreHandle; + int semaphoreFd = -1; + check(getSemaphoreHandle(target.device, &handleInfo, &semaphoreFd), "Export presentation semaphore"); + importSemaphore(target.gl_semaphores[direction], kGLHandle, semaphoreFd); +#endif + glStep("import semaphore"); + } + if (importError != GL_NO_ERROR || gl->glGetError() != GL_NO_ERROR) + throw std::runtime_error("OpenGL external memory import failed"); + + VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + poolInfo.queueFamilyIndex = target.family; + check(vkCreateCommandPool(target.device, &poolInfo, nullptr, &target.pool), "Create presentation command pool"); + VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + allocation.commandPool = target.pool; + allocation.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocation.commandBufferCount = 2; + check(vkAllocateCommandBuffers(target.device, &allocation, target.commands), "Allocate presentation commands"); + VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + check(vkCreateFence(target.device, &fenceInfo, nullptr, &target.fence), "Create presentation fence"); + // The acquire from OpenGL is the same every frame. + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + check(vkBeginCommandBuffer(target.commands[1], &begin), "Begin presentation acquire"); + VkBufferMemoryBarrier acquire{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; + acquire.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + acquire.srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + acquire.dstQueueFamilyIndex = target.family; + acquire.buffer = target.pixels; + acquire.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(target.commands[1], VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + 0, 0, nullptr, 1, &acquire, 0, nullptr); + check(vkEndCommandBuffer(target.commands[1]), "End presentation acquire"); + + target.program.reset(new QOpenGLShaderProgram); + if (!target.program->addShaderFromSourceCode(QOpenGLShader::Vertex, + "attribute vec2 vertex; varying vec2 uv; void main(){uv=vec2((vertex.x+1.0)*0.5,(1.0-vertex.y)*0.5);gl_Position=vec4(vertex,0,1);}") || + !target.program->addShaderFromSourceCode(QOpenGLShader::Fragment, + "uniform sampler2D video; varying vec2 uv; void main(){gl_FragColor=texture2D(video,uv);}") || !target.program->link()) + throw std::runtime_error("OpenGL presentation shader: " + target.program->log().toStdString()); + const float vertices[] = {-1,-1, 1,-1, -1,1, 1,1}; + GLint previousVertexArray = 0; + gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previousVertexArray); + extra->glGenVertexArrays(1, &target.vertex_array); + extra->glBindVertexArray(target.vertex_array); + gl->glGenBuffers(1, &target.buffer); + gl->glBindBuffer(GL_ARRAY_BUFFER, target.buffer); + gl->glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); + const int vertex = target.program->attributeLocation("vertex"); + target.program->enableAttributeArray(vertex); + target.program->setAttributeBuffer(vertex, GL_FLOAT, 0, 2); + gl->glBindBuffer(GL_ARRAY_BUFFER, 0); + extra->glBindVertexArray(previousVertexArray); + } + bool draw(int x, int y, int width, int height) { + if (qEnvironmentVariable("OPENSHOT_PRESENTATION_BACKEND") == "cpu") + return unavailable("OPENSHOT_PRESENTATION_BACKEND=cpu (Vulkan processing remains enabled)"); + auto lastStage = std::chrono::steady_clock::now(); + const bool trace = GpuDebug(); + // GpuDebug switches: draw solid magenta instead of the video, or have + // the CPU wait for each side instead of sharing semaphores. + static const bool pattern = GpuDebugSetting("OPENSHOT_PRESENTATION_TEST") == "pattern"; + static const bool cpuWait = GpuDebugSetting("OPENSHOT_PRESENTATION_SYNC") == "wait"; + auto stage = [&](const char* name) { + if (!trace) return; + auto now = std::chrono::steady_clock::now(); + qWarning() << "GPU presentation" << name << std::chrono::duration(now-lastStage).count(); + lastStage = now; + }; + auto context = QOpenGLContext::currentContext(); + if (!context) return unavailable("No current Qt OpenGL context"); + if (logged_context != context) { + auto text = [&](GLenum name) { + const auto* value = context->functions()->glGetString(name); + return value ? reinterpret_cast(value) : "unknown"; + }; + Logger::Instance()->Log(std::string("GPU presentation Qt context: vendor=") + text(GL_VENDOR) + + "; renderer=" + text(GL_RENDERER) + "; version=" + text(GL_VERSION) + + "; OpenGL ES=" + (context->isOpenGLES() ? "yes" : "no") + + "; external memory=" + (context->hasExtension(kMemoryExtension) ? "yes" : "no") + + "; external semaphore=" + (context->hasExtension(kSemaphoreExtension) ? "yes" : "no"), Logger::LevelInfo); + logged_context = context; + identity_logged = false; + } + stage("begin draw"); + if (context->isOpenGLES()) + return unavailable("Vulkan presentation requires desktop OpenGL; Qt selected OpenGL ES/ANGLE (try QT_OPENGL=desktop)"); + if (!frame || !widget) return unavailable("No video frame or preview widget"); + if (fallback_delivered) return false; + if (VulkanDeviceLost()) + return unavailable("GPU device lost (driver reset or GPU timeout); the CPU is used for the rest of this session"); + if (std::chrono::steady_clock::now() < retry_after) return unavailable(paused_reason); + auto authoritative = frame->GetDeferredImage(); + if (!authoritative) return unavailable("Frame has no deferred GPU surface"); + auto surface = display_surface ? display_surface : std::dynamic_pointer_cast(authoritative); + if (!surface) return unavailable("Frame cannot be presented as a Vulkan RGBA surface"); + auto inputContext = reinterpret_cast(surface->GetFrame()->hw_frames_ctx->data); + if (inputContext->sw_format != AV_PIX_FMT_RGBA) { + stage("begin RGBA conversion"); + surface = std::dynamic_pointer_cast(RenderVulkanImage( + surface, nullptr, surface->GetWidth(), surface->GetHeight(), QTransform(), 1.0, Qt::black)); + if (!surface) return unavailable("Frame cannot be presented as a Vulkan RGBA surface"); + } + display_surface = surface; + stage("surface"); + const AVFrame* source = surface->GetFrame(); + auto fc = reinterpret_cast(source->hw_frames_ctx->data); + auto vfc = static_cast(fc->hwctx); + auto dc = fc->device_ctx; + auto vk = static_cast(dc->hwctx); + auto image = reinterpret_cast(source->data[0]); + // Multi-plane decode surfaces must first go through the RGBA compositor. + if (fc->sw_format != AV_PIX_FMT_RGBA || vfc->format[0] != VK_FORMAT_R8G8B8A8_UNORM || image->img[1]) + return unavailable("Unsupported Vulkan image format for OpenGL sharing"); + if (!(vfc->usage & VK_IMAGE_USAGE_TRANSFER_SRC_BIT)) + return unavailable("Vulkan frame cannot be copied for OpenGL sharing"); + VkPhysicalDeviceIDProperties identity{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &identity}; + vkGetPhysicalDeviceProperties2(vk->phys_dev, &properties); + const bool logIdentity = !identity_logged; + if (logIdentity) { + Logger::Instance()->Log(std::string("GPU presentation Vulkan device: ") + properties.properties.deviceName + + "; UUID=" + QByteArray(reinterpret_cast(identity.deviceUUID), VK_UUID_SIZE).toHex().toStdString(), + Logger::LevelInfo); + identity_logged = true; + } + if (!context->hasExtension(kMemoryExtension) || !context->hasExtension(kSemaphoreExtension)) + return unavailable("OpenGL driver lacks external memory or semaphore sharing extensions"); + auto gl = context->functions(); + auto extra = context->extraFunctions(); + TextureBindingGuard textureBinding(gl); + auto getUuid = glFunction(context, "glGetUnsignedBytei_vEXT"); + GLint deviceCount = 0; + gl->glGetIntegerv(GL_NUM_DEVICE_UUIDS_EXT, &deviceCount); + bool sameDevice = false; + for (int i = 0; i < deviceCount; ++i) { + GLubyte uuid[GL_UUID_SIZE_EXT]; + getUuid(GL_DEVICE_UUID_EXT, i, uuid); + if (logIdentity) + Logger::Instance()->Log("GPU presentation OpenGL device UUID=" + + QByteArray(reinterpret_cast(uuid), GL_UUID_SIZE_EXT).toHex().toStdString(), Logger::LevelInfo); + sameDevice = sameDevice || std::memcmp(uuid, identity.deviceUUID, VK_UUID_SIZE) == 0; + } + if (!sameDevice) { + const auto* renderer = gl->glGetString(GL_RENDERER); + return unavailable(std::string("Qt/OpenGL and Vulkan selected different GPUs (OpenGL: ") + + (renderer ? reinterpret_cast(renderer) : "unknown") + + "; Vulkan: " + properties.properties.deviceName + ")"); + } + if (vk->queue_family_comp_index < 0) + return unavailable("Vulkan device lacks a compute queue"); + + bool locked = false, copySubmitted = false; + uint64_t copyValue = 0; + // GpuDebug: centre pixel at each step, for the first 10 frames and every 60th. + const bool probeFrame = trace && (++debug_frames <= 10 || debug_frames % 60 == 0); + std::string pixels; + try { + if (target.context != context || target.device != vk->act_dev || + target.width != source->width || target.height != source->height) { + releaseTarget(); + createTarget(context, fc, source->width, source->height); + stage("create shared buffer"); + } + // The previous frame's commands must retire before re-recording. + waitPending(); + stage("previous frame"); + const auto submit = [&](const VkSubmitInfo& info, VkFence signalFence) { + vk->lock_queue(dc, target.family, 0); + const auto result = vkQueueSubmit(target.queue, 1, &info, signalFence); + vk->unlock_queue(dc, target.family, 0); + check(result, "Submit presentation transfer"); + }; + vfc->lock_frame(fc, image); + locked = true; + VkCommandBuffer copy = target.commands[0]; + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + check(vkBeginCommandBuffer(copy, &begin), "Begin presentation copy"); + // FFmpeg frames are concurrent or owned by the compute family, so + // the copy needs no ownership transfer of the source. + VkImageMemoryBarrier toCopy{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toCopy.srcAccessMask = image->access[0]; + toCopy.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toCopy.oldLayout = image->layout[0]; + toCopy.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + toCopy.srcQueueFamilyIndex = toCopy.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toCopy.image = image->img[0]; + toCopy.subresourceRange = kColor; + vkCmdPipelineBarrier(copy, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 0, nullptr, 1, &toCopy); + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {static_cast(target.width), static_cast(target.height), 1}; + if (pattern) + vkCmdFillBuffer(copy, target.pixels, 0, VK_WHOLE_SIZE, 0xFFFF00FFu); // RGBA ff00ffff + else + vkCmdCopyImageToBuffer(copy, image->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, target.pixels, 1, ®ion); + if (target.probe) { + VkBufferImageCopy centre{}; + centre.imageSubresource = region.imageSubresource; + centre.imageOffset = {target.width / 2, target.height / 2, 0}; + centre.imageExtent = {1, 1, 1}; + vkCmdCopyImageToBuffer(copy, image->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, target.probe, 1, ¢re); + VkMemoryBarrier toHost{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + toHost.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toHost.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + vkCmdPipelineBarrier(copy, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, + 0, 1, &toHost, 0, nullptr, 0, nullptr); + } + VkBufferMemoryBarrier release{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; + release.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + release.srcQueueFamilyIndex = target.family; + release.dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + release.buffer = target.pixels; + release.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(copy, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + 0, 0, nullptr, 1, &release, 0, nullptr); + check(vkEndCommandBuffer(copy), "End presentation copy"); + // Advance the frame's timeline and release the copy to OpenGL. + const uint64_t waitValue = image->sem_value[0]; + const uint64_t signalValues[2] = {image->sem_value[0] + 1, 0}; + const VkSemaphore copySignals[2] = {image->sem[0], target.handoff[0]}; + VkTimelineSemaphoreSubmitInfo timeline{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; + timeline.waitSemaphoreValueCount = 1; + timeline.pWaitSemaphoreValues = &waitValue; + timeline.signalSemaphoreValueCount = cpuWait ? 1 : 2; + timeline.pSignalSemaphoreValues = signalValues; + const VkPipelineStageFlags waitStage = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + VkSubmitInfo copyInfo{VK_STRUCTURE_TYPE_SUBMIT_INFO, &timeline}; + copyInfo.waitSemaphoreCount = 1; + copyInfo.pWaitSemaphores = &image->sem[0]; + copyInfo.pWaitDstStageMask = &waitStage; + copyInfo.commandBufferCount = 1; + copyInfo.pCommandBuffers = © + copyInfo.signalSemaphoreCount = cpuWait ? 1 : 2; + copyInfo.pSignalSemaphores = copySignals; + submit(copyInfo, VK_NULL_HANDLE); + copySubmitted = true; + copyValue = ++image->sem_value[0]; + image->layout[0] = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + image->access[0] = VK_ACCESS_TRANSFER_READ_BIT; + vfc->unlock_frame(fc, image); + locked = false; + stage("copy submit"); + if (cpuWait || probeFrame) { + VkSemaphoreWaitInfo copyDone{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + copyDone.semaphoreCount = 1; + copyDone.pSemaphores = &image->sem[0]; + copyDone.pValues = ©Value; + check(vkWaitSemaphores(target.device, ©Done, UINT64_MAX), "Wait for presentation copy"); + if (probeFrame && target.probe_pixel) + pixels += "vulkan=" + pixelText(target.probe_pixel); + } + + if (!cpuWait) + target.wait_semaphore(target.gl_semaphores[0], 1, &target.gl_pixels, 0, nullptr, nullptr); + if (probeFrame) { + using GetBufferSubData = void (QOPENGLF_APIENTRYP)(GLenum, GLintptr, GLsizeiptr, void*); + auto getBufferSubData = reinterpret_cast(context->getProcAddress("glGetBufferSubData")); + unsigned char pixel[4] = {0, 0, 0, 0}; + GLint previousCopy = 0; + gl->glGetIntegerv(GL_COPY_READ_BUFFER_BINDING, &previousCopy); + gl->glBindBuffer(GL_COPY_READ_BUFFER, target.gl_pixels); + if (getBufferSubData) + getBufferSubData(GL_COPY_READ_BUFFER, (GLintptr(target.height / 2) * target.width + target.width / 2) * 4, 4, pixel); + gl->glBindBuffer(GL_COPY_READ_BUFFER, previousCopy); + pixels += " buffer=" + pixelText(pixel); + } + GLint viewport[4], previousVertexArray = 0, previousUnpack = 0, unpackAlignment = 4, unpackRowLength = 0; + gl->glGetIntegerv(GL_VIEWPORT, viewport); + gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previousVertexArray); + gl->glGetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &previousUnpack); + gl->glGetIntegerv(GL_UNPACK_ALIGNMENT, &unpackAlignment); + gl->glGetIntegerv(GL_UNPACK_ROW_LENGTH, &unpackRowLength); + gl->glActiveTexture(GL_TEXTURE0); + gl->glBindTexture(GL_TEXTURE_2D, target.texture); + gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, target.gl_pixels); + gl->glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + gl->glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + gl->glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, target.width, target.height, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, previousUnpack); + if (probeFrame) { + unsigned char pixel[4] = {0, 0, 0, 0}; + GLint previousFramebuffer = 0; + GLuint probe = 0; + gl->glGetIntegerv(GL_FRAMEBUFFER_BINDING, &previousFramebuffer); + gl->glGenFramebuffers(1, &probe); + gl->glBindFramebuffer(GL_FRAMEBUFFER, probe); + gl->glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, target.texture, 0); + gl->glReadPixels(target.width / 2, target.height / 2, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, pixel); + gl->glBindFramebuffer(GL_FRAMEBUFFER, previousFramebuffer); + gl->glDeleteFramebuffers(1, &probe); + const GLenum readError = gl->glGetError(); + pixels += " texture=" + pixelText(pixel) + (readError ? " (GL error " + std::to_string(readError) + ")" : ""); + } + gl->glPixelStorei(GL_UNPACK_ALIGNMENT, unpackAlignment); + gl->glPixelStorei(GL_UNPACK_ROW_LENGTH, unpackRowLength); + gl->glViewport(x, y, width, height); + gl->glDisable(GL_DEPTH_TEST); + gl->glDisable(GL_SCISSOR_TEST); + gl->glDisable(GL_STENCIL_TEST); + gl->glDisable(GL_BLEND); + gl->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); + target.program->bind(); + target.program->setUniformValue("video", 0); + extra->glBindVertexArray(target.vertex_array); + gl->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + if (trace) { + const auto error = gl->glGetError(); + GLint framebuffer = 0, activeProgram = 0, boundTexture = 0; + gl->glGetIntegerv(GL_FRAMEBUFFER_BINDING, &framebuffer); + gl->glGetIntegerv(GL_CURRENT_PROGRAM, &activeProgram); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &boundTexture); + qWarning() << "GPU presentation draw state: error=" << error + << "framebuffer=" << framebuffer << "program=" << activeProgram + << "texture=" << boundTexture; + // Log where the frame lands whenever that changes. + const qreal ratio = widget ? widget->devicePixelRatioF() : 0.0; + const std::string geometry = "draw " + std::to_string(x) + "," + std::to_string(y) + " " + + std::to_string(width) + "x" + std::to_string(height) + "; frame " + std::to_string(target.width) + + "x" + std::to_string(target.height) + "; GL viewport " + std::to_string(viewport[0]) + "," + + std::to_string(viewport[1]) + " " + std::to_string(viewport[2]) + "x" + std::to_string(viewport[3]) + + "; framebuffer " + std::to_string(framebuffer) + "; widget " + + std::to_string(widget ? widget->width() : 0) + "x" + std::to_string(widget ? widget->height() : 0) + + " at scale " + std::to_string(ratio) + "; pattern=" + std::to_string(pattern) + + " cpu wait=" + std::to_string(cpuWait) + "; GL error=" + std::to_string(error); + if (geometry != debug_geometry) { + debug_geometry = geometry; + Logger::Instance()->Log("GPU debug presentation " + geometry, Logger::LevelInfo); + } + } + if (probeFrame) { + unsigned char pixel[4] = {0, 0, 0, 0}; + gl->glReadPixels(x + width / 2, y + height / 2, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, pixel); + const GLenum readError = gl->glGetError(); + pixels += " screen=" + pixelText(pixel) + (readError ? " (GL error " + std::to_string(readError) + ")" : ""); + Logger::Instance()->Log("GPU debug preview pixels, frame " + std::to_string(debug_frames) + ": " + pixels, + Logger::LevelInfo); + } + extra->glBindVertexArray(previousVertexArray); + target.program->release(); + gl->glViewport(viewport[0], viewport[1], viewport[2], viewport[3]); + // Signals once the unpack above has finished reading the buffer. + if (cpuWait) { + gl->glFinish(); + } else { + target.signal_semaphore(target.gl_semaphores[1], 1, &target.gl_pixels, 0, nullptr, nullptr); + gl->glFlush(); + } + stage("GL draw flush"); + + VkSubmitInfo acquireInfo{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + acquireInfo.waitSemaphoreCount = cpuWait ? 0 : 1; + acquireInfo.pWaitSemaphores = &target.handoff[1]; + acquireInfo.pWaitDstStageMask = &waitStage; + acquireInfo.commandBufferCount = 1; + acquireInfo.pCommandBuffers = &target.commands[1]; + submit(acquireInfo, target.fence); + target.pending = true; + stage("acquire submit"); + return true; + } catch (...) { + if (locked) vfc->unlock_frame(fc, image); + // A half-finished handoff leaves the semaphores unusable. + if (copySubmitted && !target.pending) { + VkSemaphoreWaitInfo copyDone{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + copyDone.semaphoreCount = 1; + copyDone.pSemaphores = &image->sem[0]; + copyDone.pValues = ©Value; + vkWaitSemaphores(target.device, ©Done, UINT64_MAX); + } + releaseTarget(); + throw; + } + } +}; +VulkanOpenGLPresenter::VulkanOpenGLPresenter(uintptr_t widget) + : impl(new Impl(reinterpret_cast(widget))) {} +VulkanOpenGLPresenter::~VulkanOpenGLPresenter() = default; +bool VulkanOpenGLPresenter::Draw(int x, int y, int width, int height) { + try { + if (impl->draw(x, y, width, height)) { + impl->last_failure.clear(); + impl->failures_in_a_row = 0; + ++impl->native_draws; + return true; + } + } catch (const std::exception& error) { + impl->last_failure = error.what(); + qWarning() << "Vulkan presentation fallback:" << error.what(); + if (!VulkanDeviceLost()) { + // Usually low GPU memory: free cached GPU images, then retry later + // instead of failing again on every frame. + ReleaseVulkanMemory(); + const int failures = ++impl->failures_in_a_row; + const auto delay = std::chrono::milliseconds(std::min(30000, 500 << std::min(failures - 1, 6))); + impl->retry_after = std::chrono::steady_clock::now() + delay; + impl->paused_reason = std::string("GPU preview paused after an error; retrying shortly (") + + error.what() + ")"; + Logger::Instance()->Log("GPU preview failed (" + std::to_string(failures) + " in a row); retrying in " + + std::to_string(delay.count()) + " ms: " + error.what(), Logger::LevelWarning); + } + } + impl->fallback(); + return false; +} +int VulkanOpenGLPresenter::Width() const { return impl->frame ? impl->frame->GetWidth() : 0; } +int VulkanOpenGLPresenter::Height() const { return impl->frame ? impl->frame->GetHeight() : 0; } +void VulkanOpenGLPresenter::Clear() { + impl->frame.reset(); + impl->display_surface.reset(); +} +uint64_t VulkanOpenGLPresenter::NativeDraws() const { return impl->native_draws; } +uint64_t VulkanOpenGLPresenter::FallbackFrames() const { return impl->fallback_frames; } +std::string VulkanOpenGLPresenter::FallbackReason() const { return impl->last_failure; } +void VulkanOpenGLPresenter::Present(std::shared_ptr frame) { + if (QThread::currentThread() == impl->thread()) { + impl->PresentFrame(frame); + } else { + QMetaObject::invokeMethod(impl.get(), [receiver=impl.get(), frame]() { + receiver->PresentFrame(frame); + }, Qt::QueuedConnection); + } +} +} diff --git a/src/Qt/VulkanOpenGLPresenter.h b/src/Qt/VulkanOpenGLPresenter.h new file mode 100644 index 000000000..2f02225d3 --- /dev/null +++ b/src/Qt/VulkanOpenGLPresenter.h @@ -0,0 +1,29 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_VULKAN_OPENGL_PRESENTER_H +#define OPENSHOT_VULKAN_OPENGL_PRESENTER_H +#include +#include +#include +namespace openshot { +class Frame; +/// GUI-thread adapter for a QOpenGLWidget. Draw requires its current GL context. +class VulkanOpenGLPresenter { +public: + explicit VulkanOpenGLPresenter(uintptr_t widget); + ~VulkanOpenGLPresenter(); + bool Draw(int x, int y, int width, int height); + int Width() const; + int Height() const; + void Clear(); + uint64_t NativeDraws() const; + uint64_t FallbackFrames() const; + /// Most recent native presentation failure; empty after a successful draw. + std::string FallbackReason() const; + void Present(std::shared_ptr frame); +private: + struct Impl; + std::unique_ptr impl; +}; +} +#endif diff --git a/src/Qt/VulkanOpenGLPresenterMac.mm b/src/Qt/VulkanOpenGLPresenterMac.mm new file mode 100644 index 000000000..70c07b01a --- /dev/null +++ b/src/Qt/VulkanOpenGLPresenterMac.mm @@ -0,0 +1,214 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#define GL_SILENCE_DEPRECATION +#include "VulkanOpenGLPresenter.h" +#include "VideoRenderer.h" +#include "../VulkanFrameImage.h" +#include "../VulkanCompositor.h" +#include "../GpuDevices.h" +#include "../Logger.h" +#include "../MacVulkanInterop.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#import +#import +#import +#import +#include +extern "C" { +#include +} +namespace openshot { +namespace { +// Qt shares the texture units with native painting. Restore both targets used +// by Qt and the IOSurface bridge, including when importing or drawing fails. +class TextureBindingGuard { + QOpenGLFunctions* gl; + GLint active = GL_TEXTURE0, texture = 0, rectangle = 0; +public: + explicit TextureBindingGuard(QOpenGLFunctions* functions) : gl(functions) { + gl->glGetIntegerv(GL_ACTIVE_TEXTURE, &active); + gl->glActiveTexture(GL_TEXTURE0); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &texture); + gl->glGetIntegerv(GL_TEXTURE_BINDING_RECTANGLE, &rectangle); + } + ~TextureBindingGuard() { + gl->glActiveTexture(GL_TEXTURE0); + gl->glBindTexture(GL_TEXTURE_2D, texture); + gl->glBindTexture(GL_TEXTURE_RECTANGLE, rectangle); + gl->glActiveTexture(active); + } +}; +bool SameDevice(CGLContextObj context, uint64_t registry) { + if (!context || !registry) return false; + GLint current = 0; + if (CGLGetParameter(context, kCGLCPCurrentRendererID, ¤t) != kCGLNoError) return false; + CGLRendererInfoObj info = nullptr; + GLint count = 0; + if (CGLQueryRendererInfo(0xffffffffu, &info, &count) != kCGLNoError) return false; + bool matched = false; + for (GLint i = 0; i < count; ++i) { + GLint renderer = 0, low = 0, high = 0; + CGLDescribeRenderer(info, i, kCGLRPRendererID, &renderer); + if ((renderer & kCGLRendererIDMatchingMask) != (current & kCGLRendererIDMatchingMask)) continue; + if (CGLDescribeRenderer(info, i, kCGLRPRegistryIDLow, &low) == kCGLNoError && + CGLDescribeRenderer(info, i, kCGLRPRegistryIDHigh, &high) == kCGLNoError) + matched |= ((uint64_t(uint32_t(high)) << 32) | uint32_t(low)) == registry; + } + CGLDestroyRendererInfo(info); + return matched; +} +} +struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { + QPointer widget; + std::shared_ptr frame; + std::shared_ptr display_surface; + bool fallback_delivered = false; + uint64_t native_draws = 0, fallback_frames = 0; + std::string last_failure; + bool unavailable(const std::string& reason) { last_failure = reason; return false; } + explicit Impl(QWidget* value) : widget(value) { + value->setProperty("openshot_gpu_presenter", QVariant::fromValue(static_cast(this))); + } + ~Impl() override { if (widget) widget->setProperty("openshot_gpu_presenter", QVariant()); } + void fallback() { + if (!fallback_delivered && frame && widget) { + fallback_delivered = true; ++fallback_frames; display_surface.reset(); + // A dead GPU cannot provide CPU pixels. The next rendered frame + // will use the CPU path instead of reading this stale surface. + if (VulkanDeviceLost() && frame->GetDeferredImage()) return; + try { + const QImage image = *frame->GetImage(); + QMetaObject::invokeMethod(widget, "present", Qt::QueuedConnection, Q_ARG(QImage, image)); + } catch (const std::exception& error) { + last_failure = std::string("GPU preview fallback could not read the frame: ") + error.what(); + Logger::Instance()->Log(last_failure, Logger::LevelWarning); + } + } + } + void PresentFrame(const std::shared_ptr& value) override { + frame = value; display_surface.reset(); fallback_delivered = false; + if (frame && !frame->GetDeferredImage()) { + last_failure = "Frame is CPU-backed (decode, rendering, or an effect required CPU pixels)"; + fallback(); + } + if (widget) { + if (frame && frame->GetDeferredImage() && widget->metaObject()->indexOfMethod("gpu_presented(int)") >= 0) + QMetaObject::invokeMethod(widget, "gpu_presented", Qt::DirectConnection, + Q_ARG(int, static_cast(frame->number))); + widget->update(); + } + } + bool draw(int x, int y, int width, int height) { + auto* context = QOpenGLContext::currentContext(); + auto cgl = CGLGetCurrentContext(); + if (!context || !cgl) return unavailable("No current Qt/CGL OpenGL context"); + if (!frame || !widget) return unavailable("No video frame or preview widget"); + if (fallback_delivered) return false; + if (VulkanDeviceLost()) + return unavailable("GPU device lost (driver reset or GPU timeout); the CPU is used for the rest of this session"); + auto surface = display_surface ? display_surface : std::dynamic_pointer_cast(frame->GetDeferredImage()); + if (!surface) return unavailable("Frame cannot be presented as a shared Mac Vulkan surface"); + auto* fc = reinterpret_cast(surface->GetFrame()->hw_frames_ctx->data); + if (fc->sw_format != AV_PIX_FMT_RGBA || !IsMacVulkanFrame(surface->GetFrame())) { + surface = std::dynamic_pointer_cast(RenderVulkanImage( + surface, nullptr, surface->GetWidth(), surface->GetHeight(), QTransform(), 1.0, Qt::black)); + if (!surface) return unavailable("Frame cannot be presented as a shared Mac Vulkan surface"); + } + const AVFrame* source = surface->GetFrame(); + auto pixel = static_cast(MacVulkanPixelBuffer(source)); + if (!pixel) return unavailable("Vulkan frame has no shared Core Video pixel buffer"); + if (!SameDevice(cgl, MacVulkanDeviceRegistryID(source))) + return unavailable("Qt/OpenGL and Vulkan/Metal selected different GPUs"); + display_surface = surface; + WaitMacVulkanFrame(source); + auto io = CVPixelBufferGetIOSurface(pixel); + if (!io) return unavailable("Core Video pixel buffer has no IOSurface"); + auto* gl = context->functions(); + auto* extra = context->extraFunctions(); + TextureBindingGuard textureBinding(gl); + const bool core = context->format().profile() == QSurfaceFormat::CoreProfile; + QOpenGLShaderProgram program; + const char* vertex = core ? + "#version 150\nin vec2 vertex;out vec2 uv;void main(){uv=vec2((vertex.x+1.0)*0.5,(1.0-vertex.y)*0.5);gl_Position=vec4(vertex,0,1);}" : + "#version 120\nattribute vec2 vertex;varying vec2 uv;void main(){uv=vec2((vertex.x+1.0)*0.5,(1.0-vertex.y)*0.5);gl_Position=vec4(vertex,0,1);}"; + // CGL's supported 32-bit IOSurface view is BGRA. Our compositor stores + // RGBA bytes, so undo the view's red/blue interpretation in the sampler. + const char* fragment = core ? + "#version 150\nuniform sampler2DRect video;uniform vec2 size;in vec2 uv;out vec4 color;void main(){color=texture(video,uv*size).bgra;}" : + "#version 120\n#extension GL_ARB_texture_rectangle : enable\nuniform sampler2DRect video;uniform vec2 size;varying vec2 uv;void main(){gl_FragColor=texture2DRect(video,uv*size).bgra;}"; + if (!program.addShaderFromSourceCode(QOpenGLShader::Vertex, vertex) || + !program.addShaderFromSourceCode(QOpenGLShader::Fragment, fragment) || !program.link()) + throw std::runtime_error(program.log().toStdString()); + GLuint texture = 0, buffer = 0, vao = 0; + GLint oldViewport[4], oldVao = 0; + gl->glGetIntegerv(GL_VIEWPORT, oldViewport); + if (core) gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &oldVao); + auto cleanup = [&]() { + if (buffer) gl->glDeleteBuffers(1, &buffer); + if (vao) { extra->glBindVertexArray(oldVao); extra->glDeleteVertexArrays(1, &vao); } + if (texture) gl->glDeleteTextures(1, &texture); + gl->glViewport(oldViewport[0], oldViewport[1], oldViewport[2], oldViewport[3]); + }; + try { + gl->glActiveTexture(GL_TEXTURE0); + gl->glGenTextures(1, &texture); + gl->glBindTexture(GL_TEXTURE_RECTANGLE, texture); + const auto imported = CGLTexImageIOSurface2D(cgl, GL_TEXTURE_RECTANGLE, GL_RGBA8, + source->width, source->height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, io, 0); + if (imported != kCGLNoError) throw std::runtime_error("CGL shared IOSurface import failed"); + gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + if (core) { extra->glGenVertexArrays(1, &vao); extra->glBindVertexArray(vao); } + const float vertices[] = {-1,-1, 1,-1, -1,1, 1,1}; + gl->glGenBuffers(1, &buffer); gl->glBindBuffer(GL_ARRAY_BUFFER, buffer); + gl->glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); + if (!program.bind()) throw std::runtime_error("Bind Mac presentation shader"); + program.setUniformValue("video", 0); + program.setUniformValue("size", QVector2D(source->width, source->height)); + const int attribute = program.attributeLocation("vertex"); + program.enableAttributeArray(attribute); program.setAttributeBuffer(attribute, GL_FLOAT, 0, 2); + gl->glViewport(x, y, width, height); + gl->glDisable(GL_DEPTH_TEST); gl->glDisable(GL_SCISSOR_TEST); + gl->glDisable(GL_STENCIL_TEST); gl->glDisable(GL_BLEND); + gl->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); + gl->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + // The next renderer may reuse the retained surface after PresentFrame. + // Complete GL sampling before releasing our reference to that IOSurface. + gl->glFinish(); + const auto error = gl->glGetError(); + program.disableAttributeArray(attribute); program.release(); + gl->glBindBuffer(GL_ARRAY_BUFFER, 0); + if (error != GL_NO_ERROR) throw std::runtime_error("Mac shared-texture presentation failed"); + cleanup(); + return true; + } catch (...) { cleanup(); throw; } + } +}; +VulkanOpenGLPresenter::VulkanOpenGLPresenter(uintptr_t widget) : impl(new Impl(reinterpret_cast(widget))) {} +VulkanOpenGLPresenter::~VulkanOpenGLPresenter() = default; +bool VulkanOpenGLPresenter::Draw(int x, int y, int width, int height) { + try { if (impl->draw(x,y,width,height)) { impl->last_failure.clear(); ++impl->native_draws; return true; } } + catch (const std::exception& error) { impl->last_failure = error.what(); qWarning() << "Mac Vulkan presentation fallback:" << error.what(); } + impl->fallback(); return false; +} +int VulkanOpenGLPresenter::Width() const { return impl->frame ? impl->frame->GetWidth() : 0; } +int VulkanOpenGLPresenter::Height() const { return impl->frame ? impl->frame->GetHeight() : 0; } +void VulkanOpenGLPresenter::Clear() { impl->frame.reset(); impl->display_surface.reset(); } +uint64_t VulkanOpenGLPresenter::NativeDraws() const { return impl->native_draws; } +uint64_t VulkanOpenGLPresenter::FallbackFrames() const { return impl->fallback_frames; } +std::string VulkanOpenGLPresenter::FallbackReason() const { return impl->last_failure; } +void VulkanOpenGLPresenter::Present(std::shared_ptr frame) { + if (QThread::currentThread() == impl->thread()) impl->PresentFrame(frame); + else QMetaObject::invokeMethod(impl.get(), [receiver=impl.get(), frame]() { receiver->PresentFrame(frame); }, Qt::QueuedConnection); +} +} diff --git a/src/RendererBase.cpp b/src/RendererBase.cpp index c172600a2..2b7250f38 100644 --- a/src/RendererBase.cpp +++ b/src/RendererBase.cpp @@ -24,5 +24,16 @@ RendererBase::~RendererBase() void RendererBase::paint(const std::shared_ptr & frame) { if (frame) +#ifdef OPENSHOT_ENABLE_GPU_FRAME + this->renderFrame(frame); +#else this->render(frame->GetImage()); +#endif } + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +void RendererBase::renderFrame(const std::shared_ptr& frame) +{ + render(frame->GetImage()); +} +#endif diff --git a/src/RendererBase.h b/src/RendererBase.h index 49ed3cc31..b060e24b5 100644 --- a/src/RendererBase.h +++ b/src/RendererBase.h @@ -43,6 +43,10 @@ namespace openshot virtual ~RendererBase(); virtual void render(std::shared_ptr image) = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + /// Override to consume retained surfaces before CPU materialization. + virtual void renderFrame(const std::shared_ptr& frame); +#endif }; } diff --git a/src/Timeline.cpp b/src/Timeline.cpp index c34097e24..971aec481 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -10,6 +10,7 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" #include "Timeline.h" #include "PreviewSize.h" @@ -76,6 +77,9 @@ Timeline::Timeline(int width, int height, Fraction fps, int sample_rate, int cha final_cache = new CacheMemory(); const int cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4); final_cache->SetMaxBytesFromInfo(cache_frames, info.width, info.height, info.sample_rate, info.channels); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + final_cache->SetMaxBytes(std::min(final_cache->GetMaxBytes(), 256LL * 1024 * 1024)); +#endif } // Delegating constructor that copies parameters from a provided ReaderInfo @@ -209,6 +213,9 @@ Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) final_cache = new CacheMemory(); const int cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4); final_cache->SetMaxBytesFromInfo(cache_frames, info.width, info.height, info.sample_rate, info.channels); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + final_cache->SetMaxBytes(std::min(final_cache->GetMaxBytes(), 256LL * 1024 * 1024)); +#endif } Timeline::~Timeline() { @@ -599,7 +606,9 @@ std::shared_ptr Timeline::apply_effects(std::shared_ptr frame, int "does_effect_intersect", does_effect_intersect); // Apply the effect to this frame + gpu_metrics::Reason gpu_reason("Effect"); frame = effect->ProcessFrame(frame, effect_frame_number); + if (effect->info.has_video) gpu_metrics::RecordRender(*frame); } } // end effect loop @@ -979,6 +988,7 @@ bool Timeline::isEqual(double a, double b) // Get an openshot::Frame object for a specific frame number of this reader. std::shared_ptr Timeline::GetFrame(int64_t requested_frame) { + gpu_metrics::Scope gpu_scope(gpu_metrics_export); // Adjust out of bounds frame number if (requested_frame < 1) requested_frame = 1; diff --git a/src/Timeline.h b/src/Timeline.h index b5d270c78..7da1d2b2f 100644 --- a/src/Timeline.h +++ b/src/Timeline.h @@ -224,6 +224,7 @@ namespace openshot { void InvalidateCacheForClip(const openshot::Clip* clip); public: + bool gpu_metrics_export = false; // Set once before this timeline starts serving export work. /// @brief Constructor for the timeline (which configures the default frame properties) /// @param width The image width of generated openshot::Frame objects diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp new file mode 100644 index 000000000..b4cba575e --- /dev/null +++ b/src/VulkanCompositor.cpp @@ -0,0 +1,774 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "VulkanCompositor.h" +#include "VulkanEncodeConverter.h" +#include "GpuDevices.h" +#include "VulkanFrameImage.h" +#include "ColorManagement.h" +#ifdef __APPLE__ +#include "MacVulkanInterop.h" +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +#include +#include +} +#include "VulkanLoader.h" +// SPIR-V generated from src/vulkan/*.comp at build time. +#include "affine.comp.h" +#include "encode_nv12.comp.h" +namespace openshot { +namespace { +std::atomic pipeline_creations{0}; +void VkCheck(VkResult r, const char* operation) { + if(r == VK_SUCCESS) return; + ReportVulkanResult(r); + throw std::runtime_error(std::string(operation)+": "+VulkanResultName(r)); +} +void AvCheck(int r, const char* operation) { + if(r < 0) { + char error[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(r, error, sizeof(error)); + throw std::runtime_error(std::string(operation)+": FFmpeg error "+std::to_string(r)+" ("+error+")"); + } +} +struct FrameDelete { void operator()(AVFrame* f) const { av_frame_free(&f); } }; +using FramePtr=std::unique_ptr; +// Submissions in flight per pipeline before Render waits for the oldest one. +constexpr int kSlots=4; +// Output pools are shared by every thread so ReleaseVulkanMemory() can drop them. +// A pool keeps freed frames for reuse; FFmpeg frees a dropped pool's memory once +// its last frame is released. +struct OutputPools { + std::mutex mutex; + struct Entry { AVBufferRef* frames=nullptr; uint64_t used=0; }; + std::map,Entry> pools; + uint64_t clock=0; +}; +// Never destroyed: at process exit the Windows NVIDIA driver may already be +// unloading, and freeing Vulkan frames from a static destructor crashes it. +OutputPools& SharedPools(){ static auto* pools=new OutputPools; return *pools; } +constexpr VkImageUsageFlags kOutputUsage=VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_STORAGE_BIT|VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT; +// Returns a new reference the caller must release. +AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12){ + auto& shared=SharedPools(); + const std::lock_guard lock(shared.mutex); + auto& entry=shared.pools[{reinterpret_cast(device_ref->data),w,h,encode_nv12}]; + entry.used=++shared.clock; + auto& pool=entry.frames; + if(!pool){ + AVBufferRef* frames=av_hwframe_ctx_alloc(device_ref); + if(!frames) throw std::bad_alloc(); + auto* fc=reinterpret_cast(frames->data); + fc->format=AV_PIX_FMT_VULKAN; fc->sw_format=encode_nv12?AV_PIX_FMT_NV12:AV_PIX_FMT_RGBA; + fc->width=w; fc->height=h; + auto* vc=reinterpret_cast(fc->hwctx); + if(encode_nv12) vc->flags=AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; + vc->usage=static_cast(kOutputUsage); + const int status=av_hwframe_ctx_init(frames); + if(status<0){ + av_buffer_unref(&frames); shared.pools.erase({reinterpret_cast(device_ref->data),w,h,encode_nv12}); + AvCheck(status,"Initialize Vulkan compositor output"); + } + pool=frames; + } + // Resizing effects can produce a new size every frame. Keep retained pools + // bounded; frames still in use retain their own references to evicted pools. + while(shared.pools.size()>16) { + auto oldest=std::min_element(shared.pools.begin(),shared.pools.end(), + [](const auto& a,const auto& b){return a.second.usedsecond.frames); + shared.pools.erase(oldest); + } + AVBufferRef* ref=av_buffer_ref(pool); + if(!ref) throw std::bad_alloc(); + return ref; +} +struct EffectParameters { + int mode; // 1 saturation, 2 crop, 3 grading, 4 mask, 5 blur, 6 LUT, 7 mask blend, 8 scope + std::array values{}; + const void* data=nullptr; // copied into the slot's shader buffer (binding 5) + size_t data_bytes=0; + std::shared_ptr data_owner; // unchanged data: skip the copy while a slot holds it + const VulkanFrameImage* mask=nullptr; // third input (binding 6) + std::vector* counts=nullptr; // scope: zeroed, counted on the GPU, then read back +}; +struct Parameters { + float matrix_x[4], matrix_y[4], coeffs[4], extra[4], clear[4]; + int sizes[4], modes[4]; + float effect[4]; +}; +static_assert(sizeof(VulkanGradeParameters)==1045*sizeof(float), "Grading buffer must match shader scalar layout"); +static_assert(sizeof(Parameters)==128, "Vulkan guarantees 128 bytes of push constants"); +struct LockedFrame { + AVHWFramesContext* context; + AVVulkanFramesContext* vkcontext; + AVVkFrame* frame; + explicit LockedFrame(const AVFrame* f) { + context=reinterpret_cast(f->hw_frames_ctx->data); + vkcontext=reinterpret_cast(context->hwctx); + frame=reinterpret_cast(f->data[0]); + vkcontext->lock_frame(context,frame); + } + ~LockedFrame(){vkcontext->unlock_frame(context,frame);} +}; +class Pipeline { + AVBufferRef* device_ref=nullptr; + AVHWDeviceContext* device_context=nullptr; + AVVulkanDeviceContext* vkcontext=nullptr; + VkDevice device=VK_NULL_HANDLE; + VkQueue queue=VK_NULL_HANDLE; + int queue_family=-1; + VkCommandPool commands=VK_NULL_HANDLE; + VkDescriptorSetLayout descriptors=VK_NULL_HANDLE; + VkDescriptorPool pool=VK_NULL_HANDLE; + VkPipelineLayout layout=VK_NULL_HANDLE; + VkPipeline pipeline=VK_NULL_HANDLE; + VkSampler sampler=VK_NULL_HANDLE; + // Consumers order submissions through the frames' timeline semaphores. + // Keep encoder-owned targets alive until their views and work are retired. + struct Buffer { + VkBuffer buffer=VK_NULL_HANDLE; + VkDeviceMemory memory=VK_NULL_HANDLE; + void* mapped=nullptr; + VkDeviceSize size=0; + bool coherent=true; + }; + struct Slot { + VkDescriptorSet set=VK_NULL_HANDLE; + VkCommandBuffer cmd=VK_NULL_HANDLE; + VkFence fence=VK_NULL_HANDLE; + bool busy=false; + Buffer data; // host-written shader parameters and tables + Buffer counts; // GPU-only scope counters + Buffer staging; // host-read copy of the counters + std::shared_ptr data_owner; + std::vector views; + FramePtr target; + std::vector inputs; + }; + std::array ring; + int next_slot=0; + bool encode_nv12; + void Release(Buffer& b) { + if(b.mapped) vkUnmapMemory(device,b.memory); + if(b.buffer) vkDestroyBuffer(device,b.buffer,nullptr); + if(b.memory) vkFreeMemory(device,b.memory,nullptr); + b=Buffer{}; + } + // Grow-only. Choose the memory type with the most preferred properties: + // device-local for data shaders read per pixel, cached for host readback. + void Reserve(Buffer& b,VkDeviceSize size,VkBufferUsageFlags usage,VkMemoryPropertyFlags required,VkMemoryPropertyFlags preferred) { + if(b.size>=size) return; + Release(b); + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + info.size=size; info.usage=usage; + VkCheck(vkCreateBuffer(device,&info,nullptr,&b.buffer),"Create shader buffer"); + VkMemoryRequirements requirements; + vkGetBufferMemoryRequirements(device,b.buffer,&requirements); + VkPhysicalDeviceMemoryProperties memory; + vkGetPhysicalDeviceMemoryProperties(vkcontext->phys_dev,&memory); + uint32_t type=UINT32_MAX; int best=-1; + for(uint32_t i=0;i(flags&preferred).count()); + if(score>best) { best=score; type=i; } + } + if(type==UINT32_MAX) throw std::runtime_error("No suitable Vulkan shader buffer memory"); + b.coherent=memory.memoryTypes[type].propertyFlags&VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize=requirements.size; allocation.memoryTypeIndex=type; + VkCheck(vkAllocateMemory(device,&allocation,nullptr,&b.memory),"Allocate shader buffer"); + VkCheck(vkBindBufferMemory(device,b.buffer,b.memory,0),"Bind shader buffer"); + if(required&VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) + VkCheck(vkMapMemory(device,b.memory,0,VK_WHOLE_SIZE,0,&b.mapped),"Map shader buffer"); + b.size=size; + } + void ReserveData(Slot& slot,VkDeviceSize size) { + Reserve(slot.data,size,VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + } + void Sync(const Buffer& b,bool to_device) { + if(b.coherent) return; + VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; + range.memory=b.memory; range.size=VK_WHOLE_SIZE; + VkCheck(to_device?vkFlushMappedMemoryRanges(device,1,&range):vkInvalidateMappedMemoryRanges(device,1,&range),"Synchronize shader buffer"); + } +public: + explicit Pipeline(AVBufferRef* device_owner,bool encode=false):encode_nv12(encode) { + device_ref=av_buffer_ref(device_owner); + if(!device_ref) throw std::bad_alloc(); + device_context=reinterpret_cast(device_ref->data); + vkcontext=reinterpret_cast(device_context->hwctx); + device=vkcontext->act_dev; + queue_family=vkcontext->queue_family_comp_index; + if(queue_family<0) throw std::runtime_error("No Vulkan compute queue"); + try { + vkGetDeviceQueue(device,queue_family,0,&queue); + VkCommandPoolCreateInfo ci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + ci.queueFamilyIndex=queue_family; ci.flags=VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + VkCheck(vkCreateCommandPool(device,&ci,nullptr,&commands),"Create command pool"); + std::array bindings{}; + for(int i=0;i<7;++i) { + bindings[i].binding=i; bindings[i].descriptorCount=1; + bindings[i].descriptorType=i==5?VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:(i==4 || (encode_nv12 && i==3))?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + bindings[i].stageFlags=VK_SHADER_STAGE_COMPUTE_BIT; + } + VkDescriptorSetLayoutCreateInfo di{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + di.bindingCount=7; di.pBindings=bindings.data(); + VkCheck(vkCreateDescriptorSetLayout(device,&di,nullptr,&descriptors),"Create descriptors"); + VkPushConstantRange range{VK_SHADER_STAGE_COMPUTE_BIT,0,sizeof(Parameters)}; + VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + li.setLayoutCount=1; li.pSetLayouts=&descriptors; li.pushConstantRangeCount=1; li.pPushConstantRanges=⦥ + VkCheck(vkCreatePipelineLayout(device,&li,nullptr,&layout),"Create pipeline layout"); + std::array sizes{{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,5*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,2*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,kSlots}}}; + VkDescriptorPoolCreateInfo pi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + pi.maxSets=kSlots; pi.poolSizeCount=3; pi.pPoolSizes=sizes.data(); + VkCheck(vkCreateDescriptorPool(device,&pi,nullptr,&pool),"Create descriptor pool"); + for(auto& slot:ring){ + if(!encode_nv12) ReserveData(slot,sizeof(VulkanGradeParameters)); + VkDescriptorSetAllocateInfo ai{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + ai.descriptorPool=pool; ai.descriptorSetCount=1; ai.pSetLayouts=&descriptors; + VkCheck(vkAllocateDescriptorSets(device,&ai,&slot.set),"Allocate descriptors"); + VkCommandBufferAllocateInfo ci{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + ci.commandPool=commands; ci.level=VK_COMMAND_BUFFER_LEVEL_PRIMARY; ci.commandBufferCount=1; + VkCheck(vkAllocateCommandBuffers(device,&ci,&slot.cmd),"Allocate affine commands"); + VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkCheck(vkCreateFence(device,&fi,nullptr,&slot.fence),"Create affine fence"); + } + VkSamplerCreateInfo si{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; + si.magFilter=si.minFilter=VK_FILTER_LINEAR; + si.addressModeU=si.addressModeV=si.addressModeW=VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + VkCheck(vkCreateSampler(device,&si,nullptr,&sampler),"Create sampler"); + VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + mi.codeSize=encode_nv12?sizeof(openshot_encode_nv12_spv):sizeof(openshot_affine_spv); + mi.pCode=encode_nv12?openshot_encode_nv12_spv:openshot_affine_spv; + VkShaderModule module; + VkCheck(vkCreateShaderModule(device,&mi,nullptr,&module),"Create shader module"); + VkComputePipelineCreateInfo pci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + pci.layout=layout; pci.stage.sType=VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + pci.stage.stage=VK_SHADER_STAGE_COMPUTE_BIT; pci.stage.module=module; pci.stage.pName="main"; + const auto status=vkCreateComputePipelines(device,VK_NULL_HANDLE,1,&pci,nullptr,&pipeline); + vkDestroyShaderModule(device,module,nullptr); + VkCheck(status,"Create affine pipeline"); + pipeline_creations.fetch_add(1,std::memory_order_relaxed); + }catch(...){Destroy();throw;} + } + ~Pipeline(){Destroy();} + void Destroy(){ + for(auto& slot:ring){ + Retire(slot); + for(auto* b:{&slot.data,&slot.counts,&slot.staging}) Release(*b); + if(slot.fence) vkDestroyFence(device,slot.fence,nullptr); + } + if(pipeline) vkDestroyPipeline(device,pipeline,nullptr); + if(sampler) vkDestroySampler(device,sampler,nullptr); + if(pool) vkDestroyDescriptorPool(device,pool,nullptr); + if(layout) vkDestroyPipelineLayout(device,layout,nullptr); + if(descriptors) vkDestroyDescriptorSetLayout(device,descriptors,nullptr); + if(commands) vkDestroyCommandPool(device,commands,nullptr); + av_buffer_unref(&device_ref); + } + // Wait for a slot's previous submission, then release its views. + void Retire(Slot& slot){ + if(slot.busy){ + vkWaitForFences(device,1,&slot.fence,VK_TRUE,UINT64_MAX); + vkResetFences(device,1,&slot.fence); + slot.busy=false; + } + for(auto view:slot.views) vkDestroyImageView(device,view,nullptr); + slot.views.clear(); + slot.target.reset(); + slot.inputs.clear(); + } + VkImageView View(VkImage image,VkFormat format,VkImageAspectFlags aspect){ + VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + vi.image=image; vi.viewType=VK_IMAGE_VIEW_TYPE_2D; vi.format=format; + vi.subresourceRange.aspectMask=aspect; vi.subresourceRange.levelCount=1; vi.subresourceRange.layerCount=1; + VkImageView view; + VkCheck(vkCreateImageView(device,&vi,nullptr,&view),"Create affine image view"); + return view; + } + std::shared_ptr Render(const VulkanFrameImage& input,const VulkanFrameImage* background, + int width,int height,const QTransform& matrix,double opacity,const QColor& clear,bool high_quality=false,const AVFrame* target=nullptr, + const EffectParameters* effect=nullptr,const color::SourceColor* source_color=nullptr) { + const auto* src=input.GetFrame(); + const auto* src_fc=reinterpret_cast(src->hw_frames_ctx->data); + if(src_fc->device_ctx!=device_context) throw std::runtime_error("Vulkan source device mismatch"); + const AVFrame* back=background?background->GetFrame():nullptr; + if(back){ + const auto* bc=reinterpret_cast(back->hw_frames_ctx->data); + if(bc->device_ctx!=device_context || bc->sw_format!=AV_PIX_FMT_RGBA || back->width!=width || back->height!=height) + throw std::runtime_error("Vulkan background must be matching RGBA surface"); + } + const AVFrame* mask=effect && effect->mask?effect->mask->GetFrame():nullptr; + if(mask){ + const auto* mc=reinterpret_cast(mask->hw_frames_ctx->data); + if(mc->device_ctx!=device_context || mc->sw_format!=AV_PIX_FMT_RGBA || mask->width!=width || mask->height!=height) + throw std::runtime_error("Vulkan mask must be matching RGBA surface"); + } + const bool scope=effect && effect->counts; + FramePtr output(av_frame_alloc()); + if(!output) throw std::bad_alloc(); + if(target){ + auto* target_fc=target->hw_frames_ctx?reinterpret_cast(target->hw_frames_ctx->data):nullptr; + if(!encode_nv12 || !target_fc || target->format!=AV_PIX_FMT_VULKAN || + target_fc->sw_format!=AV_PIX_FMT_NV12 || target_fc->device_ctx!=device_context || + target->width!=width || target->height!=height) + throw std::invalid_argument("Encoder target must be matching NV12 on the rendering device"); + const auto* target_vk=static_cast(target_fc->hwctx); + if(!(target_vk->usage&VK_IMAGE_USAGE_STORAGE_BIT)) + throw std::runtime_error("Encoder target cannot be written by a Vulkan shader"); + AvCheck(av_frame_ref(output.get(),target),"Retain encoder target"); + }else{ +#ifdef __APPLE__ + output.reset(AllocateMacVulkanFrame(device_ref,width,height,encode_nv12)); +#else + AVBufferRef* pool=OutputPool(device_ref,width,height,encode_nv12); + const int allocated=av_hwframe_get_buffer(pool,output.get(),0); + std::string operation="Allocate compositor output"; + if(allocated<0){ + const auto* fc=reinterpret_cast(pool->data); + const auto* vc=reinterpret_cast(fc->hwctx); + std::ostringstream detail; + detail << operation << " [" << width << 'x' << height + << ", format=" << (encode_nv12?"NV12":"RGBA") + << ", usage=0x" << std::hex << vc->usage + << ", image_flags=0x" << vc->img_flags << std::dec + << ", tiling=" << vc->tiling << ']'; + operation=detail.str(); + } + av_buffer_unref(&pool); + AvCheck(allocated,operation.c_str()); +#endif + } + AvCheck(av_frame_copy_props(output.get(),src),"Copy compositor metadata"); + // Export NV12 uses the matrix FFmpegWriter tags for this size (BT.709 for HD). + const AVColorSpace encode_matrix=color::EncodeMatrix(width,height); + output->colorspace=encode_nv12?encode_matrix:AVCOL_SPC_RGB; output->color_range=encode_nv12?AVCOL_RANGE_MPEG:AVCOL_RANGE_JPEG; + if(encode_nv12){output->color_primaries=AVCOL_PRI_BT709;output->color_trc=AVCOL_TRC_BT709;} + if(encode_nv12 && src_fc->sw_format!=AV_PIX_FMT_RGBA) throw std::runtime_error("Encoder conversion requires RGBA"); + Parameters p{}; + bool invertible=false; + const auto inverse=matrix.inverted(&invertible); + if(!invertible) opacity=0; + p.matrix_x[0]=inverse.m11();p.matrix_x[1]=inverse.m21();p.matrix_x[2]=inverse.dx(); + p.matrix_y[0]=inverse.m12();p.matrix_y[1]=inverse.m22();p.matrix_y[2]=inverse.dy(); + p.sizes[0]=width;p.sizes[1]=height;p.sizes[2]=src->width;p.sizes[3]=src->height; + p.extra[3]=std::clamp(opacity,0.0,1.0); + p.clear[3]=clear.alphaF();p.clear[0]=clear.redF()*p.clear[3];p.clear[1]=clear.greenF()*p.clear[3];p.clear[2]=clear.blueF()*p.clear[3]; + p.modes[1]=back?1:0; + p.modes[2]=high_quality?1:0; + if(effect) { + p.modes[3]=effect->mode; + std::copy_n(effect->values.begin(),4,p.effect); + // Effects render over transparent. Reuse clear and the unused matrix + // component to stay within Vulkan's guaranteed 128-byte limit. + std::copy_n(effect->values.begin()+4,4,p.clear); + p.matrix_x[3]=effect->values[8]; + } + int plane_count=0; + const AVPixelFormat sw_format=src_fc->sw_format; + // Two-plane 4:2:0/4:2:2 with MSB-aligned 10-16 bit samples (HEVC/AV1 Main10 decode). + const bool wide=sw_format==AV_PIX_FMT_P010 || sw_format==AV_PIX_FMT_P012 || sw_format==AV_PIX_FMT_P016 || + sw_format==AV_PIX_FMT_P210 || sw_format==AV_PIX_FMT_P212 || sw_format==AV_PIX_FMT_P216; + if(sw_format==AV_PIX_FMT_RGBA){p.modes[0]=input.IsPremultiplied()?2:3;plane_count=1;} + else if(sw_format==AV_PIX_FMT_NV12 || wide){p.modes[0]=1;plane_count=1;} + else if(sw_format==AV_PIX_FMT_YUV420P){p.modes[0]=0;plane_count=3;} + else throw std::runtime_error(std::string("Vulkan affine renderer does not support ")+ + (av_get_pix_fmt_name(sw_format)?av_get_pix_fmt_name(sw_format):"this format")); + const AVColorSpace matrix_space=source_color?source_color->matrix:color::DecodeMatrix(src->colorspace,src->width,src->height); + double kr=0.299,kb=0.114; + if(matrix_space==AVCOL_SPC_BT709){kr=0.2126;kb=0.0722;} + else if(matrix_space==AVCOL_SPC_BT2020_NCL || matrix_space==AVCOL_SPC_BT2020_CL){kr=0.2627;kb=0.0593;} + else if(matrix_space==AVCOL_SPC_SMPTE240M){kr=0.212;kb=0.087;} + else if(matrix_space==AVCOL_SPC_FCC){kr=0.30;kb=0.11;} + const bool full=source_color?source_color->full_range:src->color_range==AVCOL_RANGE_JPEG; + // Samples are normalized by the texture's maximum (255 or 65535). Wide + // formats hold codes in the high bits, so limited range scales by 256. + const int depth=wide?av_pix_fmt_desc_get(sw_format)->comp[0].depth:8; + const double unit=wide?65535.0:255.0; + const double maximum=wide?double((1<NeedsConversion() && p.modes[0]<2) { + color_flags=(int(source_color->transfer)<<4)|(source_color->bt2020?256:0); + p.matrix_y[3]=source_color->RelativePeak(); + } + // Retire before locking inputs: releasing the last imported surface can + // wait on that frame's synchronization mutex. + Slot& slot=ring[next_slot]; + next_slot=(next_slot+1)%kSlots; + Retire(slot); + // Each in-flight submission owns its parameters until its fence retires. + // Tables such as LUTs are copied once while a slot still holds them. + if(effect && effect->data && (!effect->data_owner || slot.data_owner!=effect->data_owner)) { + ReserveData(slot,effect->data_bytes); + std::memcpy(slot.data.mapped,effect->data,effect->data_bytes); + Sync(slot.data,true); + slot.data_owner=effect->data_owner; + } + const VkDeviceSize count_bytes=effect && effect->counts?effect->counts->size()*sizeof(uint32_t):0; + if(scope) { + Reserve(slot.counts,count_bytes,VK_BUFFER_USAGE_STORAGE_BUFFER_BIT|VK_BUFFER_USAGE_TRANSFER_SRC_BIT|VK_BUFFER_USAGE_TRANSFER_DST_BIT, + 0,VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Reserve(slot.staging,count_bytes,VK_BUFFER_USAGE_TRANSFER_DST_BIT,VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, + VK_MEMORY_PROPERTY_HOST_CACHED_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + } + // Lock in stable address order, avoiding deadlocks when frames are shared. + std::vector all{src,output.get()}; + for(const AVFrame* extra:{back,mask}) + if(extra && std::none_of(all.begin(),all.end(),[&](const AVFrame* f){return f->data[0]==extra->data[0];})) + all.push_back(extra); + std::sort(all.begin(),all.end(),[](const AVFrame* a,const AVFrame* b){return std::less()(a->data[0],b->data[0]);}); + std::vector> locks; + std::transform(all.begin(),all.end(),std::back_inserter(locks), + [](const AVFrame* f){return std::make_unique(f);}); + auto* sf=reinterpret_cast(src->data[0]); + auto* of=reinterpret_cast(output->data[0]); + const bool separate_nv12 = p.modes[0]==1 && sf->img[1]!=VK_NULL_HANDLE && sf->img[1]!=sf->img[0]; + if(separate_nv12) plane_count=2; + const auto* svc=reinterpret_cast(src_fc->hwctx); + VkCommandBuffer cmd=slot.cmd; + const VkDescriptorSet set=slot.set; + try { + // Ordinary outputs use our shared pools. An encoder's pool can + // disappear when its writer closes, before this cached pipeline. + slot.target.reset(av_frame_clone(output.get())); + if(!slot.target) throw std::bad_alloc(); + for(const AVFrame* input_frame : {src, back, mask}) { + if (!input_frame) continue; + FramePtr retained(av_frame_clone(input_frame)); + if (!retained) throw std::bad_alloc(); + slot.inputs.push_back(std::move(retained)); + } + std::array views{}; + auto make_view=[&](VkImage image,VkFormat format,VkImageAspectFlags aspect){ + auto view=View(image,format,aspect);slot.views.push_back(view);return view; + }; + if(p.modes[0]==1){ + views[0]=make_view(sf->img[0],wide?VK_FORMAT_R16_UNORM:VK_FORMAT_R8_UNORM,separate_nv12?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_0_BIT); + views[1]=views[2]=make_view(sf->img[separate_nv12?1:0],wide?VK_FORMAT_R16G16_UNORM:VK_FORMAT_R8G8_UNORM,separate_nv12?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_1_BIT); + }else if(p.modes[0]>=2){ + views[0]=views[1]=views[2]=make_view(sf->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); + }else{ + for(int i=0;i<3;++i) views[i]=make_view(sf->img[i],svc->format[i],VK_IMAGE_ASPECT_COLOR_BIT); + } + views[3]=back?make_view(reinterpret_cast(back->data[0])->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT):views[0]; + if(encode_nv12){ + const bool separate=of->img[1] && of->img[1]!=of->img[0]; + views[3]=make_view(of->img[0],VK_FORMAT_R8_UNORM,separate?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_0_BIT); + views[4]=make_view(of->img[separate?1:0],VK_FORMAT_R8G8_UNORM,separate?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_1_BIT); + }else views[4]=make_view(of->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); + views[5]=mask?make_view(reinterpret_cast(mask->data[0])->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT):views[0]; + std::array images{}; + std::array writes{}; + for(int i=0;i<6;++i){ + const int binding=i==5?6:i; + const bool storage=binding==4 || (encode_nv12 && binding==3); + images[i].sampler=storage?VK_NULL_HANDLE:sampler;images[i].imageView=views[i];images[i].imageLayout=VK_IMAGE_LAYOUT_GENERAL; + auto& write=writes[binding]; + write.sType=VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;write.dstSet=set;write.dstBinding=binding;write.descriptorCount=1; + write.descriptorType=storage?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;write.pImageInfo=&images[i]; + } + // Scopes count into GPU memory; every other mode reads its parameters. + VkDescriptorBufferInfo data_buffer{scope?slot.counts.buffer:slot.data.buffer,0,VK_WHOLE_SIZE}; + writes[5].sType=VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[5].dstSet=set;writes[5].dstBinding=5;writes[5].descriptorCount=1; + writes[5].descriptorType=VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;writes[5].pBufferInfo=&data_buffer; + // The NV12 encoder shader uses bindings 0-4 only. + if(encode_nv12) vkUpdateDescriptorSets(device,5,writes.data(),0,nullptr); + else vkUpdateDescriptorSets(device,7,writes.data(),0,nullptr); + VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};bi.flags=VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + VkCheck(vkBeginCommandBuffer(cmd,&bi),"Begin affine commands"); + std::vector semaphores; + std::vector waits,signal_values; + std::vector> changed; + std::vector barriers; + for(auto* f:all){ + auto* vf=reinterpret_cast(f->data[0]); + const int count=f==src?plane_count:(f==output.get() && encode_nv12 && of->img[1] && of->img[1]!=of->img[0]?2:1); + for(int i=0;i(vf->access[i]); + barrier.dstAccessMask=f==output.get()?VK_ACCESS_SHADER_WRITE_BIT:VK_ACCESS_SHADER_READ_BIT; + barrier.oldLayout=vf->layout[i];barrier.newLayout=VK_IMAGE_LAYOUT_GENERAL; + barrier.srcQueueFamilyIndex=vf->queue_family[i]; + barrier.dstQueueFamilyIndex=vf->queue_family[i]==VK_QUEUE_FAMILY_IGNORED?VK_QUEUE_FAMILY_IGNORED:uint32_t(queue_family); + barrier.image=vf->img[i];barrier.subresourceRange.aspectMask=VK_IMAGE_ASPECT_COLOR_BIT; + barrier.subresourceRange.levelCount=1;barrier.subresourceRange.layerCount=1; + barriers.push_back(barrier);changed.emplace_back(vf,i); + if(std::find(semaphores.begin(),semaphores.end(),vf->sem[i])==semaphores.end()){ + semaphores.push_back(vf->sem[i]);waits.push_back(vf->sem_value[i]);signal_values.push_back(vf->sem_value[i]+1); + } + } + } + auto buffer_barrier=[&](VkBuffer buffer,VkAccessFlags from,VkAccessFlags to,VkPipelineStageFlags after,VkPipelineStageFlags before){ + VkBufferMemoryBarrier barrier{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; + barrier.srcAccessMask=from;barrier.dstAccessMask=to; + barrier.srcQueueFamilyIndex=barrier.dstQueueFamilyIndex=VK_QUEUE_FAMILY_IGNORED; + barrier.buffer=buffer;barrier.size=VK_WHOLE_SIZE; + vkCmdPipelineBarrier(cmd,after,before,0,0,nullptr,1,&barrier,0,nullptr); + }; + if(scope) { + vkCmdFillBuffer(cmd,slot.counts.buffer,0,count_bytes,0); + buffer_barrier(slot.counts.buffer,VK_ACCESS_TRANSFER_WRITE_BIT,VK_ACCESS_SHADER_READ_BIT|VK_ACCESS_SHADER_WRITE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT,VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); + } + vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,0,0,nullptr,0,nullptr,barriers.size(),barriers.data()); + vkCmdBindPipeline(cmd,VK_PIPELINE_BIND_POINT_COMPUTE,pipeline); + vkCmdBindDescriptorSets(cmd,VK_PIPELINE_BIND_POINT_COMPUTE,layout,0,1,&set,0,nullptr); + p.modes[0]|=color_flags; + vkCmdPushConstants(cmd,layout,VK_SHADER_STAGE_COMPUTE_BIT,0,sizeof(p),&p); + vkCmdDispatch(cmd,(width+15)/16,(height+15)/16,1); + if(scope) { + buffer_barrier(slot.counts.buffer,VK_ACCESS_SHADER_WRITE_BIT,VK_ACCESS_TRANSFER_READ_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,VK_PIPELINE_STAGE_TRANSFER_BIT); + VkBufferCopy region{0,0,count_bytes}; + vkCmdCopyBuffer(cmd,slot.counts.buffer,slot.staging.buffer,1,®ion); + buffer_barrier(slot.staging.buffer,VK_ACCESS_TRANSFER_WRITE_BIT,VK_ACCESS_HOST_READ_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT,VK_PIPELINE_STAGE_HOST_BIT); + } + VkCheck(vkEndCommandBuffer(cmd),"End affine commands"); + VkTimelineSemaphoreSubmitInfo ti{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; + ti.waitSemaphoreValueCount=waits.size();ti.pWaitSemaphoreValues=waits.data();ti.signalSemaphoreValueCount=signal_values.size();ti.pSignalSemaphoreValues=signal_values.data(); + std::vector stages(waits.size(),VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + VkSubmitInfo si{VK_STRUCTURE_TYPE_SUBMIT_INFO};si.pNext=&ti; + si.waitSemaphoreCount=semaphores.size();si.pWaitSemaphores=semaphores.data();si.pWaitDstStageMask=stages.data(); + si.signalSemaphoreCount=semaphores.size();si.pSignalSemaphores=semaphores.data();si.commandBufferCount=1;si.pCommandBuffers=&cmd; + vkcontext->lock_queue(device_context,queue_family,0); + auto result=vkQueueSubmit(queue,1,&si,slot.fence); + vkcontext->unlock_queue(device_context,queue_family,0); + VkCheck(result,"Submit affine renderer");slot.busy=true; + for(size_t changed_index=0;changed_indexlayout[item.second]=VK_IMAGE_LAYOUT_GENERAL; + item.first->access[item.second]=item.first==of?VK_ACCESS_SHADER_WRITE_BIT:VK_ACCESS_SHADER_READ_BIT; + item.first->queue_family[item.second]=barriers[changed_index].dstQueueFamilyIndex; + item.first->sem_value[item.second]++; + } + // Publish semaphore values before releasing frame locks. Do not wait here: + // later readers wait on those semaphores. + locks.clear(); + if(scope) { + // Only the small counter buffer returns to the CPU, never the image. + Retire(slot); + Sync(slot.staging,false); + std::memcpy(effect->counts->data(),slot.staging.mapped,count_bytes); + return nullptr; + } + return std::make_shared(output.get(),!encode_nv12); + }catch(...){ + // Imported surface destructors acquire the same frame mutexes. + locks.clear(); + Retire(slot); + throw; + } + } +}; +} +static std::shared_ptr RenderImage(std::shared_ptr source, + std::shared_ptr background,int width,int height,const QTransform& matrix,double opacity,const QColor& clear,bool high_quality, + const EffectParameters* effect,const color::SourceColor* source_color=nullptr){ + auto src=std::dynamic_pointer_cast(source); + auto back=std::dynamic_pointer_cast(background); + if(!src || (background && !back)) throw std::invalid_argument("Vulkan compositor requires Vulkan surfaces"); + if(width<=0 || height<=0) throw std::invalid_argument("Invalid compositor dimensions"); + auto* fc=reinterpret_cast(src->GetFrame()->hw_frames_ctx->data); + using Key=AVHWDeviceContext*; + // Output dimensions are push constants, not pipeline state. Animated crop + // resizing must reuse the pipeline rather than compile one per frame size. + struct CachedPipeline { std::unique_ptr pipeline; uint64_t used; }; + thread_local std::map cache; + thread_local uint64_t clock=0; + const Key requested=fc->device_ctx; + auto found=cache.find(requested); + if(found==cache.end()){ + if(cache.size()>=8){ + auto oldest=std::min_element(cache.begin(),cache.end(),[](const auto& a,const auto& b){return a.second.used(fc->device_ref),0}).first; + } + found->second.used=++clock; + auto& pipeline=found->second.pipeline; + return pipeline->Render(*src,back.get(),width,height,matrix,opacity,clear,high_quality,nullptr,effect,source_color); +} +std::shared_ptr DecodeVulkanImage(std::shared_ptr source, + int width,int height,const color::SourceColor& source_color,bool high_quality) { + QTransform matrix; + matrix.scale(double(width)/source->GetWidth(),double(height)/source->GetHeight()); + return RenderImage(source,nullptr,width,height,matrix,1.0,Qt::transparent,high_quality,nullptr,&source_color); +} +std::shared_ptr RenderVulkanImage(std::shared_ptr source, + std::shared_ptr background,int width,int height,const QTransform& matrix, + double opacity,const QColor& clear,bool high_quality) { + return RenderImage(source,background,width,height,matrix,opacity,clear,high_quality,nullptr); +} +std::shared_ptr SaturateVulkanImage(std::shared_ptr source, + const std::array& saturation) { + const int width=source->GetWidth(),height=source->GetHeight(); + EffectParameters effect{1}; + std::copy(saturation.begin(),saturation.end(),effect.values.begin()); + return RenderImage(source,nullptr,width,height,QTransform(),1.0,Qt::transparent,false,&effect); +} +std::shared_ptr CropVulkanImage(std::shared_ptr source, + const QRectF& destination,const QRectF& copy,const QRect& canvas) { + QTransform matrix; + matrix.translate(destination.x()-copy.x()-canvas.x(),destination.y()-copy.y()-canvas.y()); + const std::array bounds{float(destination.left()-canvas.x()),float(destination.top()-canvas.y()), + float(destination.right()-canvas.x()),float(destination.bottom()-canvas.y())}; + EffectParameters effect{2}; + std::copy(bounds.begin(),bounds.end(),effect.values.begin()); + return RenderImage(source,nullptr,canvas.width(),canvas.height(),matrix,1.0,Qt::transparent,false,&effect); +} +std::shared_ptr GradeVulkanImage(std::shared_ptr source, + const VulkanGradeParameters& parameters) { + const int width=source->GetWidth(),height=source->GetHeight(); + EffectParameters effect{3,parameters.controls,¶meters,sizeof(parameters)}; + return RenderImage(source,nullptr,width,height,QTransform(),1.0,Qt::transparent,false,&effect); +} +std::shared_ptr LookupVulkanImage(std::shared_ptr source, + std::shared_ptr> table,int size,bool cube,const std::array& intensity) { + const size_t entries=cube?size_t(size)*size*size:size_t(size); + if(!table || size<=0 || table->size()!=6+entries*3) throw std::invalid_argument("Invalid Vulkan LUT"); + const int width=source->GetWidth(),height=source->GetHeight(); + EffectParameters effect{6,{float(size),cube?1.0f:0.0f,intensity[0],intensity[1],intensity[2]}, + table->data(),table->size()*sizeof(float),table}; + return RenderImage(source,nullptr,width,height,QTransform(),1.0,Qt::transparent,false,&effect); +} +std::shared_ptr MaskVulkanImage(std::shared_ptr source, + std::shared_ptr mask,float contrast,int brightness,bool invert,bool replace) { + const int width=source->GetWidth(),height=source->GetHeight(); + EffectParameters effect{4,{contrast,float(brightness),float(invert),float(replace)}}; + return RenderImage(source,mask,width,height,QTransform(),1.0,Qt::transparent,false,&effect); +} +// Byte-exact effects read premultiplied RGBA, as the CPU implementations do. +static std::shared_ptr PremultipliedRGBA(std::shared_ptr source) { + auto rgba=std::dynamic_pointer_cast(source); + if(!rgba) throw std::invalid_argument("Vulkan effect requires a Vulkan surface"); + const auto* context=reinterpret_cast(rgba->GetFrame()->hw_frames_ctx->data); + if(context->sw_format==AV_PIX_FMT_RGBA && rgba->IsPremultiplied()) return source; + return RenderVulkanImage(source,nullptr,source->GetWidth(),source->GetHeight(),QTransform(),1.0); +} +std::shared_ptr BlendMaskVulkanImage(std::shared_ptr original, + std::shared_ptr effected,std::shared_ptr mask,bool invert,bool drive) { + // GPU division is not correctly rounded, so share the CPU's exact factors + // (and inverses) per mask gray value, for each blend variant. + static const auto tables=[]{ + std::array>,4> result; + for(int variant=0;variant<4;++variant) { + const bool flip=variant&1,squared=variant&2; + auto table=std::make_shared>(512); + for(int gray=0;gray<256;++gray) { + float factor; + if(squared) { + // Blur and Saturation "drive amount" mode. + factor=static_cast(gray)/255.0f; + if(flip) factor=1.0f-factor; + factor=factor*factor; + } else factor=static_cast(flip?255-gray:gray)/255.0f; + (*table)[gray]=factor; + (*table)[256+gray]=1.0f-factor; + } + result[variant]=table; + } + return result; + }(); + const auto& factors=tables[(invert?1:0)|(drive?2:0)]; + auto mask_surface=std::dynamic_pointer_cast(PremultipliedRGBA(mask)); + EffectParameters effect{7,{drive?1.0f:0.0f},factors->data(),factors->size()*sizeof(float),factors}; + effect.mask=mask_surface.get(); + return RenderImage(effected,PremultipliedRGBA(original),effected->GetWidth(),effected->GetHeight(), + QTransform(),1.0,Qt::transparent,false,&effect); +} +std::vector ScopeVulkanImage(std::shared_ptr source,const QRect& region, + int waveform_columns,int waveform_bins,int vectorscope_size) { + if(region.isEmpty() || waveform_columns<=0 || waveform_bins<256 || vectorscope_size<=0) + throw std::invalid_argument("Invalid Vulkan scope layout"); + std::vector counts(4*256+4*size_t(waveform_columns)*waveform_bins+size_t(vectorscope_size)*vectorscope_size); + EffectParameters effect{8,{float(region.left()),float(region.top()),float(region.right()+1),float(region.bottom()+1), + float(waveform_columns),float(waveform_bins),float(vectorscope_size),float(region.width())}}; + effect.counts=&counts; + RenderImage(source,nullptr,source->GetWidth(),source->GetHeight(),QTransform(),1.0,Qt::transparent,false,&effect); + return counts; +} +std::shared_ptr BlurVulkanImage(std::shared_ptr source, + const QRect& area,int horizontal,int vertical,int iterations) { + const int width=source->GetWidth(),height=source->GetHeight(); + source=PremultipliedRGBA(source); + auto result=source; + for(int i=0;i nv12_pipeline; +thread_local AVHWDeviceContext* nv12_key=nullptr; +} +std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, const AVFrame* target) { + auto src=std::dynamic_pointer_cast(source); + if(!src) throw std::invalid_argument("NV12 conversion requires a Vulkan surface"); + const auto* frame=src->GetFrame(); + if(frame->width%2 || frame->height%2) throw std::invalid_argument("NV12 export requires even dimensions"); + auto* fc=reinterpret_cast(frame->hw_frames_ctx->data); + const auto requested=fc->device_ctx; + if(!nv12_pipeline || nv12_key!=requested){ + nv12_pipeline=std::make_unique(fc->device_ref,true); + nv12_key=requested; + } + return nv12_pipeline->Render(*src,nullptr,frame->width,frame->height,QTransform(),1.0,Qt::black,false,target); +} + +uint64_t VulkanPipelineCreationCount() { return pipeline_creations.load(std::memory_order_relaxed); } + +void ReleaseVulkanMemory() { + // Free CUDA-mapped NV12 frames while CUDA is loaded, not at process exit. + nv12_pipeline.reset(); + ReleaseVulkanUploads(); +#ifdef __APPLE__ + ReleaseMacVulkanPools(); +#endif + auto& shared=SharedPools(); + const std::lock_guard lock(shared.mutex); + for(auto& item:shared.pools) av_buffer_unref(&item.second.frames); + shared.pools.clear(); +} + +} diff --git a/src/VulkanCompositor.h b/src/VulkanCompositor.h new file mode 100644 index 000000000..23eeee44c --- /dev/null +++ b/src/VulkanCompositor.h @@ -0,0 +1,67 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_VULKAN_COMPOSITOR_H +#define OPENSHOT_VULKAN_COMPOSITOR_H +#include "FrameImage.h" +#include +#include +#include +#include +#include +#include +#include +namespace openshot { +namespace color { struct SourceColor; } +/// Scale a decoded YUV surface to RGBA using its color description: YUV +/// matrix and range, and for HDR or BT.2020 sources conversion to SDR BT.709 +/// matching color::SdrConverter on the CPU. +std::shared_ptr DecodeVulkanImage(std::shared_ptr source, + int width, int height, const color::SourceColor& source_color, bool high_quality = false); +/// Affine transform and source-over compositing into an independently owned GPU frame. +/// Inputs must be Vulkan surfaces from the same device; null background uses clear. +/// high_quality selects antialiased bicubic reader resampling; default matches Qt bilinear transforms. +std::shared_ptr RenderVulkanImage( + std::shared_ptr source, std::shared_ptr background, + int width, int height, const QTransform& source_to_output, double opacity, + const QColor& clear = Qt::transparent, bool high_quality = false); +/// Apply common and per-channel saturation, preserving premultiplied alpha. +std::shared_ptr SaturateVulkanImage( + std::shared_ptr source, const std::array& saturation); +/// Copy a source rectangle into a transparent output, with optional resized canvas. +std::shared_ptr CropVulkanImage(std::shared_ptr source, + const QRectF& destination, const QRectF& copy, const QRect& canvas); +/// Per-frame grading parameters; curves retain the CPU's 256-entry lookup semantics. +struct VulkanGradeParameters { + std::array controls; // temperature, tint, exposure gain, contrast, shadows, highlights, saturation, vibrance, mix + std::array wheels; // RGB bias for global, shadows, midtones, highlights + std::array,4> curves; // master, red, green, blue +}; +std::shared_ptr GradeVulkanImage(std::shared_ptr source, + const VulkanGradeParameters& parameters); +/// Apply a 1D or 3D .cube LUT as ColorMap does. The table holds the domain +/// minimum and maximum (3 floats each), then the RGB entries. Pass the same +/// table object each frame so the GPU copy is reused. +std::shared_ptr LookupVulkanImage(std::shared_ptr source, + std::shared_ptr> table, int size, bool cube, const std::array& intensity); +/// Blend an effect result over its input by a grayscale mask, as EffectBase does. +/// drive selects the squared "drive amount" response, which keeps the input alpha. +std::shared_ptr BlendMaskVulkanImage(std::shared_ptr original, + std::shared_ptr effected, std::shared_ptr mask, bool invert, bool drive = false); +/// Count FrameScope video data for a region on the GPU. Returns luma, red, +/// green and blue histograms (256 each), the same four waveforms +/// (columns x bins each), then the vectorscope (size x size). +std::vector ScopeVulkanImage(std::shared_ptr source, const QRect& region, + int waveform_columns, int waveform_bins, int vectorscope_size); +/// Apply a reader mask using the existing brightness/contrast and inversion semantics. +std::shared_ptr MaskVulkanImage(std::shared_ptr source, + std::shared_ptr mask, float contrast, int brightness, bool invert, bool replace); +/// Separable box blur with clamped region edges and CPU-compatible compositing. +std::shared_ptr BlurVulkanImage(std::shared_ptr source, + const QRect& area, int horizontal, int vertical, int iterations); +/// Diagnostic count of successful compute pipeline creations. +uint64_t VulkanPipelineCreationCount(); +/// Return pooled GPU frame memory once its frames are released, i.e. after +/// clearing caches. Later renders create new pools as needed. +void ReleaseVulkanMemory(); +} +#endif diff --git a/src/VulkanEncodeConverter.h b/src/VulkanEncodeConverter.h new file mode 100644 index 000000000..cb36612f4 --- /dev/null +++ b/src/VulkanEncodeConverter.h @@ -0,0 +1,13 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_VULKAN_ENCODE_CONVERTER_H +#define OPENSHOT_VULKAN_ENCODE_CONVERTER_H +#include "FrameImage.h" +#include +struct AVFrame; +namespace openshot { +/// Convert an RGBA Vulkan surface to encoder-ready limited-range NV12 on the GPU. +/// An optional encoder-owned Vulkan NV12 target is written directly, without a copy. +std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, const AVFrame* target = nullptr); +} +#endif diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp new file mode 100644 index 000000000..601ae22b9 --- /dev/null +++ b/src/VulkanFrameImage.cpp @@ -0,0 +1,294 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" +#include "VulkanFrameImage.h" +#include "GpuDevices.h" +#include "Logger.h" +#include "GpuDebug.h" +#include +#include +#include "VulkanCompositor.h" +#ifdef __APPLE__ +#include "MacVulkanInterop.h" +#endif +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +} +#include "VulkanLoader.h" +namespace openshot { +namespace { +void Check(int result, const char* operation) { + if (result >= 0) return; + char error[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(result, error, sizeof(error)); + throw std::runtime_error(std::string(operation) + ": " + error); +} +struct FrameDeleter { void operator()(AVFrame* f) const { av_frame_free(&f); } }; +using OwnedFrame = std::unique_ptr; +OwnedFrame AllocateFrame() { + OwnedFrame f(av_frame_alloc()); + if (!f) throw std::bad_alloc(); + return f; +} +// Readback failures are rare and hard to reproduce, so log what FFmpeg was given. +void LogReadbackFailure(const AVFrame* frame, int result) { + auto fc = reinterpret_cast(frame->hw_frames_ctx->data); + auto vfc = static_cast(fc->hwctx); + auto vk = static_cast(fc->device_ctx->hwctx); + // FFmpeg translates failed submissions to AVERROR_EXTERNAL. Confirm the + // VkResult here so callers stop using the dead device immediately. + ReportVulkanResult(vkDeviceWaitIdle(vk->act_dev)); + bool hostImport = false; + for (int i = 0; i < vk->nb_enabled_dev_extensions; ++i) + hostImport |= std::strcmp(vk->enabled_dev_extensions[i], VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) == 0; + VkPhysicalDeviceExternalMemoryHostPropertiesEXT host{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &host}; + vkGetPhysicalDeviceProperties2(vk->phys_dev, &properties); + char error[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(result, error, sizeof(error)); + const char* format = av_get_pix_fmt_name(fc->sw_format); + Logger::Instance()->Log(std::string("Vulkan readback failed: ") + error + " (" + std::to_string(result) + + "); frame " + std::to_string(frame->width) + "x" + std::to_string(frame->height) + "; pool " + + std::to_string(fc->width) + "x" + std::to_string(fc->height) + " " + (format ? format : "unknown") + + "; Vulkan format " + std::to_string(vfc->format[0]) + "; tiling " + std::to_string(vfc->tiling) + + "; usage 0x" + QString::number(vfc->usage, 16).toStdString() + "; host memory import " + + (hostImport ? "on, alignment " + std::to_string(host.minImportedHostPointerAlignment) : std::string("off")) + + "; row pitch alignment " + std::to_string(properties.properties.limits.optimalBufferCopyRowPitchAlignment) + + "; device " + properties.properties.deviceName, Logger::LevelWarning); +} +void Validate(const AVFrame* frame) { + if (!frame || frame->format != AV_PIX_FMT_VULKAN || !frame->hw_frames_ctx || + !frame->data[0] || frame->width <= 0 || frame->height <= 0) + throw std::invalid_argument("Expected an FFmpeg Vulkan hardware frame"); +} +} +namespace { std::atomic readback_count{0}, upload_failures{0}; } +uint64_t VulkanFrameReadbackCount() { return readback_count.load(std::memory_order_relaxed); } +uint64_t VulkanUploadFailureCount() { return upload_failures.load(std::memory_order_relaxed); } +void RecordVulkanUploadFailure() { + upload_failures.fetch_add(1, std::memory_order_relaxed); + gpu_metrics::Record(gpu_metrics::UploadFailure); +} +bool VulkanRenderingEnabled() { + const char* backend = std::getenv("OPENSHOT_RENDER_BACKEND"); + return !(backend && std::strcmp(backend, "cpu") == 0) && !VulkanDeviceLost() && LoadVulkan(); +} +struct VulkanFrameImage::Impl { + OwnedFrame frame; + mutable std::mutex mutex; + bool premultiplied = false; + explicit Impl(const AVFrame* source) : frame(AllocateFrame()) { + Validate(source); + Check(av_frame_ref(frame.get(), source), "Retain Vulkan frame"); + } +}; +VulkanFrameImage::VulkanFrameImage(const AVFrame* frame, bool premultiplied) : impl(new Impl(frame)) { impl->premultiplied = premultiplied; } +bool VulkanFrameImage::IsPremultiplied() const { return impl->premultiplied; } +VulkanFrameImage::~VulkanFrameImage() = default; +int VulkanFrameImage::GetWidth() const { return impl->frame->width; } +int VulkanFrameImage::GetHeight() const { return impl->frame->height; } +const AVFrame* VulkanFrameImage::GetFrame() const { return impl->frame.get(); } +int64_t VulkanFrameImage::GetBytes() const { +#ifdef __APPLE__ + if (IsMacVulkanFrame(impl->frame.get())) return MacVulkanFrameBytes(impl->frame.get()); +#endif + const auto* frame = reinterpret_cast(impl->frame->data[0]); + int64_t bytes = 0; + for (int i = 0; i < AV_NUM_DATA_POINTERS; ++i) { + if (frame->mem[i] == VK_NULL_HANDLE) continue; + bool counted = false; + for (int j = 0; j < i; ++j) counted |= frame->mem[j] == frame->mem[i]; + if (counted) continue; + // Planes may share one allocation. Count its full padded allocation once. + size_t allocation = frame->size[i]; + for (int j = i + 1; j < AV_NUM_DATA_POINTERS; ++j) + if (frame->mem[j] == frame->mem[i]) allocation = std::max(allocation, frame->size[j]); + if (allocation > static_cast(std::numeric_limits::max() - bytes)) + return std::numeric_limits::max(); + bytes += static_cast(allocation); + } + return bytes; +} +QImage VulkanFrameImage::Materialize() const { + std::lock_guard lock(impl->mutex); + auto cpu = AllocateFrame(); +#ifdef __APPLE__ + if (IsMacVulkanFrame(impl->frame.get())) + Check(ReadMacVulkanFrame(cpu.get(), impl->frame.get()), "Read back Mac Vulkan frame"); + else +#endif + { + const int result = av_hwframe_transfer_data(cpu.get(), impl->frame.get(), 0); + if (result < 0) LogReadbackFailure(impl->frame.get(), result); + Check(result, "Read back Vulkan frame"); + } + const auto readbacks = readback_count.fetch_add(1,std::memory_order_relaxed) + 1; + gpu_metrics::Record(gpu_metrics::Readback); + // GpuDebug: say why frames are copied back ("Other" is work outside the + // timeline, such as thumbnails), for the first 20 and every 100th. + if (GpuDebug() && (readbacks <= 20 || readbacks % 100 == 0)) + Logger::Instance()->Log("GPU debug readback " + std::to_string(readbacks) + ": reason " + + gpu_metrics::CurrentReason() + "; " + std::to_string(GetWidth()) + "x" + std::to_string(GetHeight()), + Logger::LevelInfo); + Check(av_frame_copy_props(cpu.get(), impl->frame.get()), "Copy Vulkan frame metadata"); + QImage result(GetWidth(), GetHeight(), QImage::Format_RGBA8888); + if (result.isNull()) throw std::bad_alloc(); + auto* scaler = sws_getContext(cpu->width, cpu->height, AVPixelFormat(cpu->format), + result.width(), result.height(), AV_PIX_FMT_RGBA, SWS_BILINEAR, nullptr, nullptr, nullptr); + if (!scaler) throw std::runtime_error("Unable to create Vulkan readback color converter"); + std::unique_ptr owned_scaler(scaler, sws_freeContext); + int space = SWS_CS_DEFAULT; + switch (cpu->colorspace) { + case AVCOL_SPC_BT709: space = SWS_CS_ITU709; break; + case AVCOL_SPC_BT2020_NCL: case AVCOL_SPC_BT2020_CL: space = SWS_CS_BT2020; break; + case AVCOL_SPC_SMPTE240M: space = SWS_CS_SMPTE240M; break; + default: break; + } + const int* coefficients = sws_getCoefficients(space); + Check(sws_setColorspaceDetails(scaler, coefficients, cpu->color_range == AVCOL_RANGE_JPEG, + coefficients, 1, 0, 1 << 16, 1 << 16), "Set readback color conversion"); + uint8_t* output[] = {result.bits(), nullptr, nullptr, nullptr}; + int strides[] = {result.bytesPerLine(), 0, 0, 0}; + Check(sws_scale(scaler, cpu->data, cpu->linesize, 0, cpu->height, output, strides), "Convert readback pixels"); + if (impl->premultiplied) return QImage(result.constBits(), result.width(), result.height(), result.bytesPerLine(), QImage::Format_RGBA8888_Premultiplied).copy(); + return result.convertToFormat(QImage::Format_RGBA8888_Premultiplied); +} +struct VulkanFrameProcessor::Impl {}; +VulkanFrameProcessor::VulkanFrameProcessor() : impl(new Impl) {} +VulkanFrameProcessor::~VulkanFrameProcessor() = default; +void VulkanFrameProcessor::Reset() {} +std::shared_ptr VulkanFrameProcessor::Process(const AVFrame* frame, int width, int height, + const color::SourceColor& color, bool high_quality) { +#ifdef __APPLE__ + OwnedFrame imported; + if (frame && frame->format == AV_PIX_FMT_VIDEOTOOLBOX) { + imported.reset(ImportVideoToolboxFrame(frame)); + frame = imported.get(); + } +#endif + Validate(frame); + if (width <= 0 || height <= 0) throw std::invalid_argument("Invalid Vulkan output dimensions"); + auto source = std::make_shared(frame); + return DecodeVulkanImage(source, width, height, color, high_quality); +} +namespace { +struct BufferDeleter { void operator()(AVBufferRef* p) const { av_buffer_unref(&p); } }; +using OwnedBuffer = std::unique_ptr; +struct UploadCache { + struct Entry { std::shared_ptr surface; int64_t bytes; uint64_t age; }; + struct Pool { OwnedBuffer frames; uint64_t age; }; + std::mutex mutex; + std::map entries; + // One frame pool per image size. A pool per upload leaked its device + // memory on NVIDIA (about 1.5 MiB per 600x600 upload) and was slow. + std::map, Pool> pools; + int64_t bytes = 0; + uint64_t clock = 0; +}; +// Never destroyed: freeing cached Vulkan frames from a static destructor at +// process exit crashes the Windows NVIDIA driver, which may already be unloading. +UploadCache& Uploads() { static auto* cache = new UploadCache; return *cache; } +AVBufferRef* UploadPool(UploadCache& cache, int width, int height) { + auto found = cache.pools.find({width, height}); + if (found != cache.pools.end()) { + found->second.age = ++cache.clock; + return found->second.frames.get(); + } + AVBufferRef* device_ptr = nullptr; + Check(CreateSharedVulkanDevice(&device_ptr), "Create image upload device"); + OwnedBuffer device(device_ptr); + OwnedBuffer pool(av_hwframe_ctx_alloc(device.get())); + if (!pool) throw std::bad_alloc(); + auto* context = reinterpret_cast(pool->data); + context->format = AV_PIX_FMT_VULKAN; + context->sw_format = AV_PIX_FMT_RGBA; + context->width = width; context->height = height; + auto* vk = reinterpret_cast(context->hwctx); + vk->usage = static_cast(VK_IMAGE_USAGE_SAMPLED_BIT | + VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); + Check(av_hwframe_ctx_init(pool.get()), "Initialize image upload pool"); + // Images still in use keep their own pool reference after it is dropped here. + if (cache.pools.size() >= 8) + cache.pools.erase(std::min_element(cache.pools.begin(), cache.pools.end(), + [](const auto& a, const auto& b) { return a.second.age < b.second.age; })); + auto* frames = pool.get(); + cache.pools.emplace(std::make_pair(width, height), UploadCache::Pool{std::move(pool), ++cache.clock}); + return frames; +} +} +std::shared_ptr UploadVulkanImage(const QImage& image) { + if (image.isNull()) throw std::invalid_argument("Cannot upload an empty image"); + auto& cache = Uploads(); + const std::lock_guard lock(cache.mutex); + const auto key = image.cacheKey(); + auto found = cache.entries.find(key); + if (found != cache.entries.end()) { + found->second.age = ++cache.clock; + return found->second.surface; + } + auto* pool = UploadPool(cache, image.width(), image.height()); + QImage rgba = image.convertToFormat(QImage::Format_RGBA8888_Premultiplied); + auto cpu = AllocateFrame(); + auto gpu = AllocateFrame(); + cpu->format = AV_PIX_FMT_RGBA; cpu->width = rgba.width(); cpu->height = rgba.height(); + cpu->data[0] = const_cast(rgba.constBits()); + cpu->linesize[0] = rgba.bytesPerLine(); + Check(av_hwframe_get_buffer(pool, gpu.get(), 0), "Allocate uploaded image"); + Check(av_hwframe_transfer_data(gpu.get(), cpu.get(), 0), "Upload CPU image"); + // FFmpeg may copy straight from the CPU pixels (host-mapped memory) and + // returns before the copy runs; wait so rgba is not freed or reused first. + auto* uploaded = reinterpret_cast(gpu->data[0]); + auto* frames = reinterpret_cast(pool->data); + auto* device = reinterpret_cast(frames->device_ctx->hwctx); + VkSemaphoreWaitInfo copied{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + copied.semaphoreCount = 1; + copied.pSemaphores = &uploaded->sem[0]; + copied.pValues = &uploaded->sem_value[0]; + const VkResult waited = vkWaitSemaphores(device->act_dev, &copied, UINT64_MAX); + ReportVulkanResult(waited); + if (waited != VK_SUCCESS) + throw std::runtime_error("Wait for CPU image upload: " + VulkanResultName(waited)); + gpu_metrics::Record(gpu_metrics::Upload); + std::shared_ptr surface = std::make_shared(gpu.get(), true); +#ifdef __APPLE__ + // Cache an IOSurface-backed image, so CPU fallback uses the synchronized + // Metal bridge instead of FFmpeg's generic MoltenVK staging-buffer readback. + surface = RenderVulkanImage(surface, nullptr, image.width(), image.height(), + QTransform(), 1.0, Qt::transparent); +#endif + const int64_t size = surface->GetBytes(); + constexpr int64_t budget = 64 * 1024 * 1024; + if (size <= budget) { + while (!cache.entries.empty() && (cache.bytes + size > budget || cache.entries.size() >= 16)) { + auto oldest = std::min_element(cache.entries.begin(), cache.entries.end(), + [](const auto& a, const auto& b) { return a.second.age < b.second.age; }); + cache.bytes -= oldest->second.bytes; + cache.entries.erase(oldest); + } + cache.entries.emplace(key, UploadCache::Entry{surface, size, ++cache.clock}); + cache.bytes += size; + } + return surface; +} +void ReleaseVulkanUploads() { + auto& cache = Uploads(); + const std::lock_guard lock(cache.mutex); + cache.entries.clear(); cache.bytes = 0; + cache.pools.clear(); +} + +} diff --git a/src/VulkanFrameImage.h b/src/VulkanFrameImage.h new file mode 100644 index 000000000..81ca2753e --- /dev/null +++ b/src/VulkanFrameImage.h @@ -0,0 +1,59 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_VULKAN_FRAME_IMAGE_H +#define OPENSHOT_VULKAN_FRAME_IMAGE_H +#include "FrameImage.h" +#include +#include +struct AVFrame; +namespace openshot { +namespace color { struct SourceColor; } +/// False when OPENSHOT_RENDER_BACKEND=cpu selects the CPU pipeline at runtime. +bool VulkanRenderingEnabled(); +/// Process-wide count of GPU images materialized as CPU pixels (diagnostics). +uint64_t VulkanFrameReadbackCount(); +/// Process-wide count of CPU images that could not be uploaded to the GPU. +uint64_t VulkanUploadFailureCount(); +#ifndef SWIG +void RecordVulkanUploadFailure(); +#endif +/// A retained FFmpeg Vulkan image. CPU pixels are created only by Materialize(). +class VulkanFrameImage final : public FrameImage { +public: + explicit VulkanFrameImage(const AVFrame* frame, bool premultiplied = false); + ~VulkanFrameImage() override; + VulkanFrameImage(const VulkanFrameImage&) = delete; + VulkanFrameImage& operator=(const VulkanFrameImage&) = delete; + int GetWidth() const override; + int GetHeight() const override; + int64_t GetBytes() const override; + QImage Materialize() const override; + /// Borrowed immutable frame; retain a reference before asynchronous use. + const AVFrame* GetFrame() const; + bool IsPremultiplied() const; +private: + struct Impl; + std::unique_ptr impl; +}; +/// Upload CPU pixels once per QImage cache key. Bounded cache; returned surfaces +/// retain their own GPU storage. Mutable QImage access invalidates its cache key. +std::shared_ptr UploadVulkanImage(const QImage& image); +void ReleaseVulkanUploads(); +/// Reuses the scaling pipeline across frames. Outputs own independent storage. +class VulkanFrameProcessor { +public: + VulkanFrameProcessor(); + ~VulkanFrameProcessor(); + VulkanFrameProcessor(const VulkanFrameProcessor&) = delete; + VulkanFrameProcessor& operator=(const VulkanFrameProcessor&) = delete; + /// Scale and convert a decoded frame to RGBA. color describes the frame + /// (see color::DescribeFrame); HDR and BT.2020 sources become SDR BT.709. + std::shared_ptr Process(const AVFrame* frame, int width, int height, + const color::SourceColor& color, bool high_quality = false); + void Reset(); +private: + struct Impl; + std::unique_ptr impl; +}; +} +#endif diff --git a/src/VulkanLoader.cpp b/src/VulkanLoader.cpp new file mode 100644 index 000000000..dcba4d08e --- /dev/null +++ b/src/VulkanLoader.cpp @@ -0,0 +1,54 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "VulkanLoader.h" +#include "Logger.h" +#include +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#else +#include +#endif + +#define OPENSHOT_VULKAN_DEFINE(name) PFN_##name name = nullptr; +OPENSHOT_VULKAN_FUNCTIONS(OPENSHOT_VULKAN_DEFINE) +#undef OPENSHOT_VULKAN_DEFINE + +namespace openshot { +namespace { +bool Load() { + // Never unloaded: drivers keep threads and state for the process lifetime. +#if defined(_WIN32) + HMODULE library = LoadLibraryExA("vulkan-1.dll", nullptr, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + const auto symbol = [library](const char* name) { return reinterpret_cast(GetProcAddress(library, name)); }; +#elif defined(__APPLE__) + // MoltenVK ships inside the app and is linked directly + void* library = RTLD_DEFAULT; + const auto symbol = [](const char* name) { return dlsym(RTLD_DEFAULT, name); }; +#else + void* library = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL); + const auto symbol = [library](const char* name) { return dlsym(library, name); }; +#endif + if (!library) { + Logger::Instance()->Log("Vulkan library not found; rendering on the CPU", Logger::LevelWarning); + return false; + } + bool complete = true; +#define OPENSHOT_VULKAN_LOAD(name) \ + name = reinterpret_cast(symbol(#name)); \ + if (!name) { \ + Logger::Instance()->Log("Vulkan library lacks " #name "; rendering on the CPU", Logger::LevelWarning); \ + complete = false; \ + } + OPENSHOT_VULKAN_FUNCTIONS(OPENSHOT_VULKAN_LOAD) +#undef OPENSHOT_VULKAN_LOAD + return complete; +} +} // namespace + +bool LoadVulkan() { + static const bool loaded = Load(); + return loaded; +} +} // namespace openshot diff --git a/src/VulkanLoader.h b/src/VulkanLoader.h new file mode 100644 index 000000000..50b1ef957 --- /dev/null +++ b/src/VulkanLoader.h @@ -0,0 +1,47 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// Loads the Vulkan functions OpenShot uses at runtime, so libopenshot starts on +// systems without a Vulkan loader or driver (they render on the CPU instead). +// Build with VK_NO_PROTOTYPES; each pointer below has the same name as the +// Vulkan function, so calling code is unchanged. +#pragma once +#include + +#define OPENSHOT_VULKAN_FUNCTIONS(X) \ + X(vkAllocateCommandBuffers) X(vkAllocateDescriptorSets) X(vkAllocateMemory) \ + X(vkBeginCommandBuffer) X(vkBindBufferMemory) X(vkCmdBindDescriptorSets) \ + X(vkCmdBindPipeline) X(vkCmdCopyBuffer) X(vkCmdCopyImageToBuffer) X(vkCmdDispatch) \ + X(vkCmdFillBuffer) X(vkCmdPipelineBarrier) X(vkCmdPushConstants) X(vkCreateBuffer) \ + X(vkCreateCommandPool) X(vkCreateComputePipelines) X(vkCreateDescriptorPool) \ + X(vkCreateDescriptorSetLayout) X(vkCreateFence) X(vkCreateImage) X(vkCreateImageView) \ + X(vkCreateInstance) X(vkCreatePipelineLayout) X(vkCreateSampler) X(vkCreateSemaphore) \ + X(vkCreateShaderModule) X(vkDestroyBuffer) X(vkDestroyCommandPool) \ + X(vkDestroyDescriptorPool) X(vkDestroyDescriptorSetLayout) X(vkDestroyFence) \ + X(vkDestroyImage) X(vkDestroyImageView) X(vkDestroyInstance) X(vkDestroyPipeline) \ + X(vkDestroyPipelineLayout) X(vkDestroySampler) X(vkDestroySemaphore) \ + X(vkDestroyShaderModule) X(vkDeviceWaitIdle) X(vkEndCommandBuffer) \ + X(vkEnumerateDeviceExtensionProperties) X(vkEnumerateInstanceVersion) \ + X(vkEnumeratePhysicalDevices) X(vkFlushMappedMemoryRanges) X(vkFreeMemory) \ + X(vkGetBufferMemoryRequirements) X(vkGetBufferMemoryRequirements2) \ + X(vkGetDeviceProcAddr) X(vkGetDeviceQueue) X(vkGetPhysicalDeviceMemoryProperties) \ + X(vkGetPhysicalDeviceProperties) X(vkGetPhysicalDeviceProperties2) \ + X(vkGetPhysicalDeviceQueueFamilyProperties) X(vkInvalidateMappedMemoryRanges) \ + X(vkMapMemory) X(vkQueueSubmit) X(vkResetFences) X(vkUnmapMemory) \ + X(vkUpdateDescriptorSets) X(vkWaitForFences) X(vkWaitSemaphores) + +// Hidden, so other libraries in the process never bind to these pointers +#if defined(_WIN32) +#define OPENSHOT_VULKAN_HIDDEN +#else +#define OPENSHOT_VULKAN_HIDDEN __attribute__((visibility("hidden"))) +#endif +#define OPENSHOT_VULKAN_DECLARE(name) extern OPENSHOT_VULKAN_HIDDEN PFN_##name name; +OPENSHOT_VULKAN_FUNCTIONS(OPENSHOT_VULKAN_DECLARE) +#undef OPENSHOT_VULKAN_DECLARE + +namespace openshot { +// True when the Vulkan library and every function above were found. Cheap +// after the first call; nothing may call Vulkan when it returns false. +bool LoadVulkan(); +} diff --git a/src/effects/Blur.cpp b/src/effects/Blur.cpp index 43f3ce8b3..cce4531f7 100644 --- a/src/effects/Blur.cpp +++ b/src/effects/Blur.cpp @@ -19,6 +19,11 @@ #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -81,9 +86,6 @@ void Blur::init_effect_details() // modified openshot::Frame object std::shared_ptr Blur::GetFrame(std::shared_ptr frame, int64_t frame_number) { - // Get the frame's image - std::shared_ptr frame_image = frame->GetImage(); - // Get the current blur radius int horizontal_radius_value = horizontal_radius.GetValue(frame_number); int vertical_radius_value = vertical_radius.GetValue(frame_number); @@ -91,31 +93,47 @@ std::shared_ptr Blur::GetFrame(std::shared_ptr int iteration_value = iterations.GetInt(frame_number); (void) sigma_value; - int w = frame_image->width(); - int h = frame_image->height(); + int w = frame->GetWidth(); + int h = frame->GetHeight(); if (w <= 0 || h <= 0 || iteration_value <= 0) return frame; // Define area we're working on in terms of a QRect with QMargins applied. - QRect area(QPoint(0, 0), frame_image->size()); + QRect area(0,0,w,h); area = area.marginsRemoved({ int(clamp_margin(left.GetValue(frame_number)) * w), int(clamp_margin(top.GetValue(frame_number)) * h), int(clamp_margin(right.GetValue(frame_number)) * w), int(clamp_margin(bottom.GetValue(frame_number)) * h) }); - area = area.intersected(QRect(QPoint(0, 0), frame_image->size())); + area = area.intersected(QRect(0,0,w,h)); if (area.isEmpty()) return frame; + const int horizontal_area_radius=std::min(std::max(0,horizontal_radius_value),area.width()-1); + const int vertical_area_radius=std::min(std::max(0,vertical_radius_value),area.height()-1); + if (!horizontal_area_radius && !vertical_area_radius) return frame; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled()) { + try { + auto surface=frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface=UploadVulkanImage(*frame->GetImage()); + if (surface) { + frame->AddDeferredImage(BlurVulkanImage(surface,area,horizontal_area_radius,vertical_area_radius,iteration_value)); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan blur CPU fallback: ")+error.what()); + } + } +#endif + auto frame_image=frame->GetImage(); // Grab two copies of the image pixel data QImage image_copy = frame_image->copy(area); std::shared_ptr blur_image = std::make_shared(image_copy); std::shared_ptr frame_image_2 = std::make_shared(image_copy); const int area_w = blur_image->width(); const int area_h = blur_image->height(); - const int horizontal_area_radius = std::min(std::max(0, horizontal_radius_value), std::max(0, area_w - 1)); - const int vertical_area_radius = std::min(std::max(0, vertical_radius_value), std::max(0, area_h - 1)); bool blurred = false; // Loop through each iteration diff --git a/src/effects/Blur.h b/src/effects/Blur.h index d9ac15b50..d08218fbe 100644 --- a/src/effects/Blur.h +++ b/src/effects/Blur.h @@ -39,6 +39,10 @@ namespace openshot */ class Blur : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + bool MaskDrivesAmount(int64_t frame_number) const override { return UseCustomMaskBlend(frame_number); } + private: /// Init effect settings void init_effect_details(); diff --git a/src/effects/ColorGrade.cpp b/src/effects/ColorGrade.cpp index 51e1f5ec0..42757b23e 100644 --- a/src/effects/ColorGrade.cpp +++ b/src/effects/ColorGrade.cpp @@ -20,6 +20,12 @@ using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif + namespace { constexpr float kInv255 = 1.0f / 255.0f; @@ -219,11 +225,16 @@ void ColorGrade::sync_lut_effect() { lut_dirty = false; } -std::shared_ptr ColorGrade::GetFrame(std::shared_ptr frame, int64_t frame_number) { - std::shared_ptr frame_image = frame->GetImage(); - if (!frame_image) +std::shared_ptr ColorGrade::apply_lut(std::shared_ptr frame, int64_t frame_number) { + if (lut_path.empty() || lut_intensity.GetValue(frame_number) <= 0.0) return frame; + sync_lut_effect(); + // Let the LUT upload clip images itself, as ColorGrade does. + lut_effect.ParentClip(ParentClip()); + return lut_effect.GetFrame(frame, frame_number); +} +std::shared_ptr ColorGrade::GetFrame(std::shared_ptr frame, int64_t frame_number) { const float temperature_value = std::max(-1.0f, std::min(1.0f, static_cast(temperature.GetValue(frame_number)))); const float tint_value = std::max(-1.0f, std::min(1.0f, static_cast(tint.GetValue(frame_number)))); const float exposure_value = std::max(-4.0f, std::min(4.0f, static_cast(exposure.GetValue(frame_number)))); @@ -244,11 +255,7 @@ std::shared_ptr ColorGrade::GetFrame(std::shared_ptr 0.0) { - sync_lut_effect(); - return lut_effect.GetFrame(frame, frame_number); - } - return frame; + return apply_lut(frame, frame_number); } const float exposure_gain = std::pow(2.0f, exposure_value); const float contrast_factor = std::max(0.0f, 1.0f + contrast_value); @@ -261,6 +268,32 @@ std::shared_ptr ColorGrade::GetFrame(std::shared_ptrGetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); + if (surface) { + const VulkanGradeParameters parameters{ + {temperature_value,tint_value,exposure_gain,contrast_factor,shadows_value,highlights_value, + saturation_value,vibrance_value,mix_value}, + {global_wheel.red_delta,global_wheel.green_delta,global_wheel.blue_delta, + shadows_wheel.red_delta,shadows_wheel.green_delta,shadows_wheel.blue_delta, + midtones_wheel.red_delta,midtones_wheel.green_delta,midtones_wheel.blue_delta, + highlights_wheel.red_delta,highlights_wheel.green_delta,highlights_wheel.blue_delta}, + {curve_all_lut,curve_red_lut,curve_green_lut,curve_blue_lut}}; + frame->AddDeferredImage(GradeVulkanImage(surface,parameters)); + return apply_lut(frame, frame_number); + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan grading CPU fallback: ") + error.what()); + } + } +#endif + auto frame_image = frame->GetImage(); + if (!frame_image) return frame; + static const std::array inv_alpha = [] { std::array lut{}; lut[0] = 0.0f; @@ -384,12 +417,7 @@ std::shared_ptr ColorGrade::GetFrame(std::shared_ptr 0.0) { - sync_lut_effect(); - frame = lut_effect.GetFrame(frame, frame_number); - } - - return frame; + return apply_lut(frame, frame_number); } std::string ColorGrade::Json() const { diff --git a/src/effects/ColorGrade.h b/src/effects/ColorGrade.h index 0649ad0d8..2ee6a6dce 100644 --- a/src/effects/ColorGrade.h +++ b/src/effects/ColorGrade.h @@ -72,6 +72,9 @@ namespace openshot */ class ColorGrade : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + private: ColorMap lut_effect; std::string lut_path; @@ -79,6 +82,8 @@ namespace openshot void init_effect_details(); void sync_lut_effect(); + /// Apply the optional LUT through ColorMap, on Vulkan when the frame is. + std::shared_ptr apply_lut(std::shared_ptr frame, int64_t frame_number); static float Clamp01(float value); diff --git a/src/effects/ColorMap.cpp b/src/effects/ColorMap.cpp index ef735f427..69e23984d 100644 --- a/src/effects/ColorMap.cpp +++ b/src/effects/ColorMap.cpp @@ -15,6 +15,11 @@ #include #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -22,6 +27,7 @@ void ColorMap::load_cube_file() { if (lut_path.empty()) { lut_data.clear(); + gpu_table.reset(); lut_size = 0; lut_type = LUTType::None; needs_refresh = false; @@ -126,7 +132,12 @@ void ColorMap::load_cube_file() lut_type = parsed_is_3d ? LUTType::LUT3D : LUTType::LUT1D; lut_domain_min = parsed_domain_min; lut_domain_max = parsed_domain_max; + auto table = std::make_shared>(lut_domain_min.begin(), lut_domain_min.end()); + table->insert(table->end(), lut_domain_max.begin(), lut_domain_max.end()); + table->insert(table->end(), lut_data.begin(), lut_data.end()); + gpu_table = table; } else { + gpu_table.reset(); lut_data.clear(); lut_size = 0; lut_type = LUTType::None; @@ -186,15 +197,31 @@ ColorMap::GetFrame(std::shared_ptr frame, int64_t frame_number) if (lut_data.empty() || lut_size <= 0 || lut_type == LUTType::None) return frame; - auto image = frame->GetImage(); - int w = image->width(), h = image->height(); - unsigned char *pixels = image->bits(); - float overall = float(intensity.GetValue(frame_number)); float tR = float(intensity_r.GetValue(frame_number)) * overall; float tG = float(intensity_g.GetValue(frame_number)) * overall; float tB = float(intensity_b.GetValue(frame_number)) * overall; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled() && gpu_table) { + try { + auto surface = frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); + if (surface) { + frame->AddDeferredImage(LookupVulkanImage(surface, gpu_table, lut_size, + lut_type == LUTType::LUT3D, {tR, tG, tB})); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan LUT CPU fallback: ") + error.what()); + } + } +#endif + + auto image = frame->GetImage(); + int w = image->width(), h = image->height(); + unsigned char *pixels = image->bits(); + const bool use3d = (lut_type == LUTType::LUT3D); const bool use1d = (lut_type == LUTType::LUT1D); const int lut_dim = lut_size; diff --git a/src/effects/ColorMap.h b/src/effects/ColorMap.h index 61525fb93..7203c6f55 100644 --- a/src/effects/ColorMap.h +++ b/src/effects/ColorMap.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,9 @@ namespace openshot */ class ColorMap : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + private: enum class LUTType { None, LUT1D, LUT3D }; @@ -44,6 +48,7 @@ namespace openshot bool needs_refresh; ///< Reload LUT on next frame std::array lut_domain_min; ///< Input domain minimum per channel std::array lut_domain_max; ///< Input domain maximum per channel + std::shared_ptr> gpu_table; ///< Domain and entries, shared with Vulkan /// Populate info fields (class_name, name, description) void init_effect_details(); diff --git a/src/effects/Crop.cpp b/src/effects/Crop.cpp index cb5e1f102..3248844cf 100644 --- a/src/effects/Crop.cpp +++ b/src/effects/Crop.cpp @@ -19,6 +19,11 @@ #include #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -53,9 +58,6 @@ void Crop::init_effect_details() // modified openshot::Frame object std::shared_ptr Crop::GetFrame(std::shared_ptr frame, int64_t frame_number) { - // Get the frame's image - std::shared_ptr frame_image = frame->GetImage(); - // Get current keyframe values double left_value = left.GetValue(frame_number); double top_value = top.GetValue(frame_number); @@ -66,7 +68,7 @@ std::shared_ptr Crop::GetFrame(std::shared_ptr double x_shift = x.GetValue(frame_number); double y_shift = y.GetValue(frame_number); - QSize sz = frame_image->size(); + QSize sz(frame->GetWidth(), frame->GetHeight()); // Compute destination rectangle to paint into QRectF paint_r( @@ -96,6 +98,24 @@ std::shared_ptr Crop::GetFrame(std::shared_ptr copy_r.setBottom(sz.height()); } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled()) { + const QRect canvas = resize ? paint_r.toRect() : QRect(QPoint(0,0),sz); + if (canvas.width()>0 && canvas.height()>0 && paint_r.width()>0 && paint_r.height()>0) { + try { + auto surface = frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); + if (surface) { + frame->AddDeferredImage(CropVulkanImage(surface,paint_r,copy_r,canvas)); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan crop CPU fallback: ") + error.what()); + } + } + } +#endif + auto frame_image = frame->GetImage(); QImage cropped(sz, QImage::Format_RGBA8888_Premultiplied); cropped.fill(Qt::transparent); diff --git a/src/effects/Crop.h b/src/effects/Crop.h index 9d03a8dc9..aadd9919e 100644 --- a/src/effects/Crop.h +++ b/src/effects/Crop.h @@ -36,6 +36,9 @@ namespace openshot */ class Crop : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + private: /// Init effect settings void init_effect_details(); diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index 51f15685a..7b25dd6df 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -17,6 +17,11 @@ #include "ReaderBase.h" #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -55,6 +60,26 @@ void Mask::init_effect_details() // This method is required for all derived classes of EffectBase, and returns a // modified openshot::Frame object std::shared_ptr Mask::GetFrame(std::shared_ptr frame, int64_t frame_number) { + // An absent reader/generated mask is a no-op, including on deferred frames. + if (!MaskReader() && MaskSourceId().empty()) return frame; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled() && MaskSourceId().empty()) { + try { + auto mask=ResolveReaderMaskSurface(frame->GetWidth(),frame->GetHeight(),frame_number); + if (!mask) return frame; + auto surface=frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface=UploadVulkanImage(*frame->GetImage()); + if (surface) { + const float factor=20.0f/std::max(0.5f,20.0f-float(contrast.GetValue(frame_number))); + frame->AddDeferredImage(MaskVulkanImage(surface,mask,factor, + int(255*brightness.GetValue(frame_number)),mask_invert,replace_image)); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan mask CPU fallback: ")+error.what()); + } + } +#endif // Get frame image first std::shared_ptr frame_image = frame->GetImage(); if (!frame_image || frame_image->isNull()) diff --git a/src/effects/Saturation.cpp b/src/effects/Saturation.cpp index 39bd6914a..03b2d3eb4 100644 --- a/src/effects/Saturation.cpp +++ b/src/effects/Saturation.cpp @@ -14,6 +14,11 @@ #include "Exceptions.h" #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -123,6 +128,22 @@ void Saturation::init_effect_details() // modified openshot::Frame object std::shared_ptr Saturation::GetFrame(std::shared_ptr frame, int64_t frame_number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled()) { + try { + auto surface = frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); + if (surface) { + frame->AddDeferredImage(SaturateVulkanImage(surface, { + float(saturation.GetValue(frame_number)), float(saturation_R.GetValue(frame_number)), + float(saturation_G.GetValue(frame_number)), float(saturation_B.GetValue(frame_number))})); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan saturation CPU fallback: ") + error.what()); + } + } +#endif // Get the frame's image std::shared_ptr frame_image = frame->GetImage(); diff --git a/src/effects/Saturation.h b/src/effects/Saturation.h index cea64cdf8..bcbf11317 100644 --- a/src/effects/Saturation.h +++ b/src/effects/Saturation.h @@ -45,6 +45,10 @@ namespace openshot */ class Saturation : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + bool MaskDrivesAmount(int64_t frame_number) const override { return UseCustomMaskBlend(frame_number); } + private: /// Init effect settings void init_effect_details(); diff --git a/src/vulkan/affine.comp b/src/vulkan/affine.comp new file mode 100644 index 000000000..3d54ca364 --- /dev/null +++ b/src/vulkan/affine.comp @@ -0,0 +1,297 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#version 450 +layout(local_size_x=16, local_size_y=16) in; +layout(binding=0) uniform sampler2D plane_y; +layout(binding=1) uniform sampler2D plane_u; +layout(binding=2) uniform sampler2D plane_v; +layout(binding=3) uniform sampler2D background_tex; +layout(binding=4, rgba8) uniform writeonly image2D output_img; +// Effect parameters and tables (float bits), or scope counters. +layout(std430,binding=5) buffer Data { uint data[]; }; +layout(binding=6) uniform sampler2D mask_tex; +shared uint scope_histograms[1024]; +layout(push_constant) uniform Parameters { + vec4 matrix_x; + vec4 matrix_y; + vec4 color_coeffs; // Y multiplier, V->R, U->G, V->G + vec4 extra; // U->B, Y offset, chroma offset, opacity + vec4 clear; + ivec4 sizes; // output width/height, source width/height + ivec4 modes; // x: source format (bits 0-3: 0 planar, 1 NV12/P01x, 2 premult RGBA, 3 straight RGBA) + // plus transfer (bits 4-7) and BT.2020 gamut (bit 8), see to_sdr(); y: background enabled + vec4 effect; // saturation controls, or crop bounds in output pixels +} p; +// Mitchell/Netravali family B=0, C=0.6 matches libswscale's default +// SWS_BICUBIC kernel. Expand the footprint on minification to suppress aliasing. +float cubic(float value) { + float x=abs(value); + if(x<1.0) return ((1.4*x-2.4)*x)*x+1.0; + if(x<2.0) return ((-0.6*x+3.0)*x-4.8)*x+2.4; + return 0.0; +} +vec4 sample_source(sampler2D image, vec2 uv) { + if(p.modes.w==2 || p.modes.w==4 || p.modes.w==5) return texelFetch(image,ivec2(floor(uv*vec2(textureSize(image,0)))),0); + if(p.modes.z==0) return texture(image,uv); + ivec2 size=textureSize(image,0); + vec2 pos=uv*vec2(size)-0.5; + vec2 footprint=max(vec2(1.0),vec2(length(p.matrix_x.xy),length(p.matrix_y.xy)) + *vec2(size)/vec2(p.sizes.zw)); + ivec2 first=ivec2(ceil(pos-2.0*footprint)); + ivec2 last=ivec2(floor(pos+2.0*footprint)); + vec4 result=vec4(0); + float total=0.0; + for(int y=first.y;y<=last.y;++y) { + float wy=cubic((float(y)-pos.y)/footprint.y); + if(abs(wy)<0.000001) continue; + for(int x=first.x;x<=last.x;++x) { + float weight=wy*cubic((float(x)-pos.x)/footprint.x); + if(abs(weight)<0.000001) continue; + result+=texelFetch(image,clamp(ivec2(x,y),ivec2(0),size-1),0)*weight; + total+=weight; + } + } + return result/total; +} +float value(int index) { return uintBitsToFloat(data[index]); } +vec3 wheel_bias(int wheel) { + int start=9+wheel*3; + return vec3(value(start),value(start+1),value(start+2)); +} +float grade_curve(int curve,float value) { + int index=clamp(int(value*255.0+0.5),0,255); + return uintBitsToFloat(data[21+curve*256+index]); +} +// HDR/wide gamut to SDR BT.709; mirrors color::SdrConverter (ColorManagement.cpp). +// Linear values are relative to the BT.2408 reference white (203 nits). +#define TONE_KNEE 0.8 +#define REFERENCE_WHITE 203.0 +const mat3 BT2020_TO_709=mat3( // column-major + 1.6604910,-0.1245505,-0.0181508, + -0.5876411, 1.1328999,-0.1005789, + -0.0728499,-0.0083494, 1.1187297); +vec3 pq_to_linear(vec3 e) { + const float m1=2610.0/16384.0,m2=2523.0/4096.0*128.0; + const float c1=3424.0/4096.0,c2=2413.0/4096.0*32.0,c3=2392.0/4096.0*32.0; + vec3 v=pow(clamp(e,0.0,1.0),vec3(1.0/m2)); + return 10000.0/REFERENCE_WHITE*pow(max(v-c1,0.0)/(c2-c3*v),vec3(1.0/m1)); +} +vec3 hlg_to_scene(vec3 e) { + const float a=0.17883277,b=0.28466892,c=0.55991073; + e=clamp(e,0.0,1.0); + return mix(e*e/3.0,(exp((e-c)/a)+b)/12.0,greaterThan(e,vec3(0.5))); +} +float tone_map(float value,float peak) { + if(value<=TONE_KNEE || peak<=1.0) return min(value,1.0); + float span=1.0-TONE_KNEE,t=(value-TONE_KNEE)/span; + float w=max((peak-TONE_KNEE)/span,1.0); + return min(TONE_KNEE+span*t*(1.0+t/(w*w))/(1.0+t),1.0); +} +vec3 to_sdr(vec3 signal) { + int transfer=(p.modes.x>>4)&15; + bool bt2020=(p.modes.x&256)!=0; + vec3 rgb; + if(transfer==1) rgb=pq_to_linear(signal); + else if(transfer==2) { + // HLG OOTF for the nominal 1000 nit display (system gamma 1.2). + vec3 scene=hlg_to_scene(signal); + float luma=dot(scene,bt2020?vec3(0.2627,0.6780,0.0593):vec3(0.2126,0.7152,0.0722)); + rgb=scene*(1000.0/REFERENCE_WHITE)*pow(clamp(luma,0.0,1.0),0.2); + } else rgb=pow(clamp(signal,0.0,1.0),vec3(2.4)); + if(bt2020) rgb=BT2020_TO_709*rgb; + rgb=max(rgb,0.0); + float brightest=max(rgb.r,max(rgb.g,rgb.b)); + if(brightest>TONE_KNEE) rgb*=tone_map(brightest,p.matrix_y.w)/brightest; + return pow(min(rgb,1.0),vec3(1.0/2.4)); +} +// Premultiplied RGBA from any supported source format. +vec4 load_source(vec2 uv) { + if((p.modes.x&15)>=2) { + vec4 front=clamp(sample_source(plane_y,uv),0.0,1.0); + if((p.modes.x&15)==3) front.rgb*=front.a; + else front.rgb=min(front.rgb,vec3(front.a)); + return front; + } + float y=(sample_source(plane_y,uv).r-p.extra.y)*p.color_coeffs.x; + vec2 chroma=(p.modes.x&15)==1 ? sample_source(plane_u,uv).rg : vec2(sample_source(plane_u,uv).r,sample_source(plane_v,uv).r); + chroma-=p.extra.z; + vec3 rgb=clamp(vec3(y+p.color_coeffs.y*chroma.y, + y+p.color_coeffs.z*chroma.x+p.color_coeffs.w*chroma.y, + y+p.extra.x*chroma.x),0,1); + if((p.modes.x>>4)!=0) rgb=to_sdr(rgb); + return vec4(rgb,1); +} +// ColorMap table: domain min/max, then RGB entries. +vec3 lut_entry(int index) { int i=6+index*3; return vec3(value(i),value(i+1),value(i+2)); } +float lut_1d(float v,int channel,int size) { + if(size<=1) return value(6+channel); + precise float scaled=v*float(size-1); + int i0=int(floor(scaled)),i1=min(i0+1,size-1); + precise float t=scaled-float(i0); + int last=size*3-1; + return value(6+clamp(i0*3+channel,0,last))*(1.0-t)+value(6+clamp(i1*3+channel,0,last))*t; +} +vec3 lut_3d(vec3 v,int n) { + precise vec3 f=v*float(n-1); + ivec3 i0=ivec3(floor(f)),i1=min(i0+1,ivec3(n-1)); + precise vec3 d=f-vec3(i0); + #define LUT(r,g,b) lut_entry(((b*n+g)*n+r)) + precise vec3 c00=LUT(i0.r,i0.g,i0.b)*(1.0-d.r)+LUT(i1.r,i0.g,i0.b)*d.r; + precise vec3 c01=LUT(i0.r,i0.g,i1.b)*(1.0-d.r)+LUT(i1.r,i0.g,i1.b)*d.r; + precise vec3 c10=LUT(i0.r,i1.g,i0.b)*(1.0-d.r)+LUT(i1.r,i1.g,i0.b)*d.r; + precise vec3 c11=LUT(i0.r,i1.g,i1.b)*(1.0-d.r)+LUT(i1.r,i1.g,i1.b)*d.r; + #undef LUT + precise vec3 c0=c00*(1.0-d.g)+c10*d.g; + precise vec3 c1=c01*(1.0-d.g)+c11*d.g; + return c0*(1.0-d.b)+c1*d.b; +} +// FrameScope analysis: shared histograms per workgroup, global waveforms and +// vectorscope. Matches FrameScope::analyze_video's byte math. +void scope() { + for(uint i=gl_LocalInvocationIndex;i<1024u;i+=256u) scope_histograms[i]=0u; + barrier(); + ivec2 pixel=ivec2(gl_GlobalInvocationID.xy); + ivec4 region=ivec4(p.effect); + if(all(greaterThanEqual(pixel,region.xy)) && all(lessThan(pixel,region.zw))) { + ivec4 c=ivec4(round(load_source((vec2(pixel)+0.5)/vec2(p.sizes.zw))*255.0)); + if(c.a>0) { + const float inv255=1.0/255.0; + precise vec3 f; + ivec3 index; + if(c.a==255) { f=vec3(c.rgb)*inv255; index=c.rgb; } + else { + f=min(vec3(1.0),(vec3(c.rgb)*(255.0/float(c.a)))*inv255); + index=ivec3(f*255.0+0.5); + } + precise float luma=0.299*f.r+0.587*f.g+0.114*f.b; + int luma_index=int(luma*255.0+0.5); + int columns=int(p.clear.x),bins=int(p.clear.y),size=int(p.clear.z); + int column=clamp(((pixel.x-region.x)*columns)/max(1,int(p.clear.w)),0,columns-1); + uint plane=uint(columns*bins),wave=1024u+uint(column*bins); + precise float u=-0.14713*f.r-0.28886*f.g+0.43600*f.b; + precise float v=0.61500*f.r-0.51499*f.g-0.10001*f.b; + float middle=float(size-1)*0.5; + int vx=clamp(int(middle+(u/0.43600)*middle+0.5),0,size-1); + int vy=clamp(int(middle-(v/0.61500)*middle+0.5),0,size-1); + atomicAdd(scope_histograms[luma_index],1u); + atomicAdd(scope_histograms[256+index.r],1u); + atomicAdd(scope_histograms[512+index.g],1u); + atomicAdd(scope_histograms[768+index.b],1u); + atomicAdd(data[wave+uint(luma_index)],1u); + atomicAdd(data[wave+plane+uint(index.r)],1u); + atomicAdd(data[wave+2u*plane+uint(index.g)],1u); + atomicAdd(data[wave+3u*plane+uint(index.b)],1u); + atomicAdd(data[1024u+4u*plane+uint(vy*size+vx)],1u); + } + } + barrier(); + for(uint i=gl_LocalInvocationIndex;i<1024u;i+=256u) + if(scope_histograms[i]!=0u) atomicAdd(data[i],scope_histograms[i]); +} +void main() { + // Scopes count instead of writing an image; every invocation reaches the barriers. + if(p.modes.w==8) { scope(); return; } + ivec2 pixel=ivec2(gl_GlobalInvocationID.xy); + if(any(greaterThanEqual(pixel,p.sizes.xy))) return; + vec2 center=vec2(pixel)+0.5; + vec2 src=vec2(dot(p.matrix_x.xyz,vec3(center,1)),dot(p.matrix_y.xyz,vec3(center,1))); + vec4 front=vec4(0); + if(all(greaterThanEqual(src,vec2(0))) && all(lessThan(src,vec2(p.sizes.zw)))) { + front=load_source(src/vec2(p.sizes.zw)); + if(p.modes.w==1 && front.a>0.0) { + // Match Saturation's integer rounding and fixed-point brightness. + int alpha=int(round(front.a*255.0)); + ivec3 rgb=ivec3(round(front.rgb*255.0)); + if(alpha<255) rgb=ivec3(vec3(rgb)*(255.0/float(alpha))); + int weighted=(77*rgb.r*rgb.r+150*rgb.g*rgb.g+29*rgb.b*rgb.b+128)>>8; + float brightness=sqrt(float(weighted)); + rgb=clamp(ivec3(vec3(brightness)+(vec3(rgb)-brightness)*p.effect.x),ivec3(0),ivec3(255)); + vec3 channel=vec3(rgb)*sqrt(vec3(0.299,0.587,0.114)); + ivec3 red=ivec3(vec3(channel.r)+(vec3(rgb.r,0,0)-channel.r)*p.effect.y); + ivec3 green=ivec3(vec3(channel.g)+(vec3(0,rgb.g,0)-channel.g)*p.effect.z); + ivec3 blue=ivec3(vec3(channel.b)+(vec3(0,0,rgb.b)-channel.b)*p.effect.w); + rgb=clamp(red+green+blue,ivec3(0),ivec3(255)); + if(alpha<255) rgb=ivec3(vec3(rgb)*(float(alpha)/255.0)); + front.rgb=vec3(rgb)/255.0; + } + if(p.modes.w==3 && front.a>0.0) { + vec3 original=front.rgb/front.a; + vec3 rgb=original; + rgb.r=clamp(rgb.r+p.effect.x*0.125,0.0,1.0); + rgb.b=clamp(rgb.b-p.effect.x*0.125,0.0,1.0); + rgb=clamp(rgb+vec3(0.05,-0.1,0.05)*p.effect.y,0.0,1.0); + rgb=clamp(rgb*p.effect.z,0.0,1.0); + rgb=clamp((rgb-0.5)*p.effect.w+0.5,0.0,1.0); + float luma=dot(rgb,vec3(0.299,0.587,0.114)); + rgb=clamp(rgb+0.35*(p.clear.x*(1.0-luma)*(1.0-luma)+p.clear.y*luma*luma),0.0,1.0); + luma=dot(rgb,vec3(0.299,0.587,0.114)); + float centered=abs(luma-0.5)*2.0; + rgb=clamp(rgb+wheel_bias(0)+wheel_bias(1)*(1.0-luma)*(1.0-luma) + +wheel_bias(2)*clamp(1.0-centered*centered,0.0,1.0)+wheel_bias(3)*luma*luma,0.0,1.0); + luma=dot(rgb,vec3(0.299,0.587,0.114)); + float colorfulness=max(rgb.r,max(rgb.g,rgb.b))-min(rgb.r,min(rgb.g,rgb.b)); + float factor=p.clear.z*max(0.0,1.0+p.clear.w*(1.0-colorfulness)); + rgb=clamp(luma+(rgb-luma)*factor,0.0,1.0); + rgb=vec3(grade_curve(0,rgb.r),grade_curve(0,rgb.g),grade_curve(0,rgb.b)); + rgb=vec3(grade_curve(1,rgb.r),grade_curve(2,rgb.g),grade_curve(3,rgb.b)); + rgb=clamp(mix(original,rgb,p.matrix_x.w),0.0,1.0); + front.rgb=floor(rgb*front.a*255.0+0.5)/255.0; + } + if(p.modes.w==6 && front.a>0.0) { + // ColorMap: straight color from premultiplied bytes, truncating writes. + ivec4 bytes=ivec4(round(front*255.0)); + float alpha=float(bytes.a)/255.0; + precise vec3 color=(vec3(bytes.rgb)/alpha)*(1.0/255.0); + vec3 low=vec3(value(0),value(1),value(2)),range=vec3(value(3),value(4),value(5))-low; + vec3 domain; + for(int i=0;i<3;++i) domain[i]=clamp(range[i]<=0.0?color[i]:(color[i]-low[i])/range[i],0.0,1.0); + int size=int(p.effect.x); + vec3 mapped=p.effect.y!=0.0?lut_3d(domain,size): + vec3(lut_1d(domain.r,0,size),lut_1d(domain.g,1,size),lut_1d(domain.b,2,size)); + vec3 amount=vec3(p.effect.zw,p.clear.x); + precise vec3 blended=(mapped*amount+color*(1.0-amount))*alpha; + front.rgb=vec3(clamp(ivec3(blended*255.0),ivec3(0),ivec3(255)))/255.0; + } + front*=p.extra.w; + } + if(p.modes.w==2 && (any(lessThan(center,p.effect.xy)) || any(greaterThanEqual(center,p.effect.zw)))) + front=vec4(0); + vec4 back=p.modes.y==1 ? texture(background_tex,center/vec2(p.sizes.xy)) : p.clear; + if(p.modes.w==5) { + ivec2 first=ivec2(p.effect.zw),last=ivec2(p.clear.xy); + if(all(greaterThanEqual(pixel,first)) && all(lessThanEqual(pixel,last))) { + int radius=int(p.effect.x); + ivec2 direction=p.effect.y==0.0?ivec2(1,0):ivec2(0,1); + ivec4 total=ivec4(0); + for(int offset=-radius;offset<=radius;++offset) + total+=ivec4(round(texelFetch(plane_y,clamp(pixel+direction*offset,first,last),0)*255.0)); + front=floor(vec4(total)/float(2*radius+1)+0.5)/255.0; + } + } + if(p.modes.w==4) { + ivec4 mask=ivec4(round(back*255.0)); + int gray=(mask.r*11+mask.g*16+mask.b*5)>>5; + // Preserve CPU rounding before truncation: do not fuse multiply/add. + precise float adjusted_value=p.effect.x*(float(gray)+p.effect.y-128.0)+128.0; + int adjusted=clamp(int(adjusted_value),0,255); + int alpha=clamp(mask.a-adjusted,0,255); + if(p.effect.w!=0.0) front=vec4(float(alpha)/255.0); + else { + if(p.effect.z!=0.0) alpha=255-alpha; + front=floor(round(front*255.0)*float(alpha)/255.0)/255.0; + } + } + if(p.modes.w==7) { + // EffectBase::BlendWithMask: mix input (background) and result by mask + // gray. The table holds the factor, then its inverse, per gray value. + ivec4 effected=ivec4(round(front*255.0)),original=ivec4(round(back*255.0)); + ivec4 mask=ivec4(round(texelFetch(mask_tex,pixel,0)*255.0)); + int gray=(mask.r*11+mask.g*16+mask.b*5)/32; + precise vec4 mixed=vec4(original)*value(256+gray)+vec4(effected)*value(gray); + front=floor(mixed)/255.0; + // Drive-amount masks keep the input's alpha. + if(p.effect.x!=0.0) front.a=float(original.a)/255.0; + } + if(p.modes.w>=3) back=vec4(0); + imageStore(output_img,pixel,front+back*(1-front.a)); +} diff --git a/src/vulkan/encode_nv12.comp b/src/vulkan/encode_nv12.comp new file mode 100644 index 000000000..b323a78cd --- /dev/null +++ b/src/vulkan/encode_nv12.comp @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#version 450 +layout(local_size_x=16,local_size_y=16) in; +layout(binding=0) uniform sampler2D source_image; +layout(binding=3,r8) uniform writeonly image2D output_y; +layout(binding=4,rg8) uniform writeonly image2D output_uv; +layout(push_constant) uniform Parameters { + vec4 matrix_x, matrix_y, coeffs, extra, clear; + ivec4 sizes, modes; +} p; +// Limited-range output with the writer's matrix (coeffs.xy = Kr, Kb), as +// FFmpegWriter's swscale path encodes. +vec3 yuv(vec3 rgb) { + float kr=p.coeffs.x,kb=p.coeffs.y; + float y=dot(rgb,vec3(kr,1.0-kr-kb,kb)); + return vec3(y*219.0/255.0+16.0/255.0, + (rgb.b-y)/(2.0*(1.0-kb))*224.0/255.0+128.0/255.0, + (rgb.r-y)/(2.0*(1.0-kr))*224.0/255.0+128.0/255.0); +} +void main() { + ivec2 at=ivec2(gl_GlobalInvocationID.xy); + if(any(greaterThanEqual(at,p.sizes.xy))) return; + vec3 value=yuv(texelFetch(source_image,at,0).rgb); + imageStore(output_y,at,vec4(value.x,0,0,1)); + if((at.x%2==0)&&(at.y%2==0)) { + vec3 rgb=vec3(0); + for(int y=0;y<2;++y) for(int x=0;x<2;++x) + rgb+=texelFetch(source_image,min(at+ivec2(x,y),p.sizes.xy-1),0).rgb; + vec3 chroma=yuv(rgb*0.25); + imageStore(output_uv,at/2,vec4(chroma.yz,0,1)); + } +} diff --git a/tests/AudioWaveformer.cpp b/tests/AudioWaveformer.cpp index 2ceac1ee4..de40a71c2 100644 --- a/tests/AudioWaveformer.cpp +++ b/tests/AudioWaveformer.cpp @@ -38,6 +38,49 @@ std::shared_ptr MakeWaveformFrame(int frame_number, int sample_rate, cons frame->AddAudio(true, 0, 0, samples.data(), static_cast(samples.size()), 1.0f); return frame; } + +// Write a stereo PCM WAV, where each channel's samples come from a generator(channel, sample_index) +template +void WriteStereoWav(const std::string& path, int sample_rate, double seconds, Generator generator) +{ + std::remove(path.c_str()); + FFmpegWriter writer(path); + writer.SetAudioOptions(true, "pcm_s16le", sample_rate, 2, LAYOUT_STEREO, sample_rate * 2 * 16); + writer.Open(); + + const int64_t total = static_cast(std::llround(seconds * sample_rate)); + const int chunk = sample_rate / 10; + int frame_number = 1; + for (int64_t start = 0; start < total; start += chunk) { + const int count = static_cast(std::min(chunk, total - start)); + auto frame = std::make_shared(frame_number++, count, 2); + frame->SampleRate(sample_rate); + frame->ChannelsLayout(LAYOUT_STEREO); + for (int channel = 0; channel < 2; channel++) { + std::vector samples(count); + for (int i = 0; i < count; i++) { + samples[i] = generator(channel, start + i); + } + frame->AddAudio(true, channel, 0, samples.data(), count, 1.0f); + } + writer.WriteFrame(frame); + } + writer.Close(); +} + +int ArgMax(const std::vector& values) +{ + return static_cast(std::distance(values.begin(), std::max_element(values.begin(), values.end()))); +} + +Keyframe LinearKeyframe(std::initializer_list> points) +{ + Keyframe keyframe; + for (const auto& point : points) { + keyframe.AddPoint(point.first, point.second, LINEAR); + } + return keyframe; +} } TEST_CASE( "Extract waveform data piano.wav", "[libopenshot][audiowaveformer]" ) @@ -507,3 +550,230 @@ TEST_CASE( "AudioWaveformData struct methods", "[libopenshot][audiowaveformer]" CHECK(vectors[0].size() == 0); CHECK(vectors[0].size() == 0); } + +TEST_CASE( "Waveform chunks do not drift when sample rate is not a multiple of the density", "[libopenshot][audiowaveformer][timing]" ) +{ + // 44100 / 200 = 220.5 samples per chunk. Integer chunking used 220 samples, which put + // this click 11 samples late at 25 seconds (and 16 seconds late at the end of 2 hours). + const std::string path = "AudioWaveformer-drift-44100.wav"; + const int sample_rate = 44100; + const double click_seconds = 25.0; + const int64_t click_start = static_cast(click_seconds * sample_rate); + WriteStereoWav(path, sample_rate, 30.0, [&](int, int64_t index) { + return (index >= click_start && index < click_start + 200) ? 0.9f : 0.001f; + }); + + FFmpegReader reader(path); + reader.Open(); + AudioWaveformer waveformer(&reader); + const int samples_per_second = 200; + AudioWaveformData waveform = waveformer.ExtractSamples(-1, samples_per_second, false); + + REQUIRE(waveform.max_samples.size() == static_cast(std::ceil(reader.info.duration * samples_per_second))); + CHECK(ArgMax(waveform.max_samples) == static_cast(click_seconds * samples_per_second)); + + // The end of the media is covered (no truncated tail from accumulated rounding) + CHECK(waveform.max_samples.back() > 0.0f); + CHECK(waveform.rms_samples.back() > 0.0f); + + reader.Close(); + std::remove(path.c_str()); +} + +TEST_CASE( "Single-pass channel extraction matches per-channel extraction", "[libopenshot][audiowaveformer][channels]" ) +{ + const std::string path = "AudioWaveformer-channels.wav"; + const int sample_rate = 48000; + WriteStereoWav(path, sample_rate, 3.0, [&](int channel, int64_t index) { + const double amplitude = channel == 0 ? 0.8 : 0.2; + return static_cast(amplitude * std::sin(2.0 * M_PI * 440.0 * index / sample_rate)); + }); + + FFmpegReader reader(path); + reader.Open(); + REQUIRE(reader.info.channels == 2); + AudioWaveformer waveformer(&reader); + const int samples_per_second = 200; + + AudioWaveformData all = waveformer.ExtractChannelSamples(samples_per_second, false); + AudioWaveformData mixed = waveformer.ExtractSamples(-1, samples_per_second, false); + REQUIRE(all.channel_count() == 2); + REQUIRE(all.channel_rms_samples.size() == 2); + REQUIRE(all.max_samples.size() == mixed.max_samples.size()); + for (size_t i = 0; i < mixed.max_samples.size(); i++) { + CHECK(all.max_samples[i] == Approx(mixed.max_samples[i]).margin(1e-6)); + CHECK(all.rms_samples[i] == Approx(mixed.rms_samples[i]).margin(1e-6)); + } + + for (int channel = 0; channel < 2; channel++) { + AudioWaveformData single = waveformer.ExtractSamples(channel, samples_per_second, false); + const auto& channel_max = all.channel_max_samples[channel]; + const auto& channel_rms = all.channel_rms_samples[channel]; + REQUIRE(channel_max.size() == single.max_samples.size()); + REQUIRE(channel_rms.size() == single.rms_samples.size()); + for (size_t i = 0; i < single.max_samples.size(); i++) { + CHECK(channel_max[i] == Approx(single.max_samples[i]).margin(1e-6)); + CHECK(channel_rms[i] == Approx(single.rms_samples[i]).margin(1e-6)); + } + std::vector> vectors = all.channel_vectors(channel); + CHECK(vectors[0] == channel_max); + CHECK(vectors[1] == channel_rms); + } + + // Channels keep their own levels: left peaks near 0.8, right near 0.2, mix follows the loudest + const size_t middle = all.max_samples.size() / 2; + CHECK(all.channel_max_samples[0][middle] == Approx(0.8f).margin(0.01f)); + CHECK(all.channel_max_samples[1][middle] == Approx(0.2f).margin(0.01f)); + CHECK(all.max_samples[middle] == Approx(0.8f).margin(0.01f)); + CHECK(all.channel_rms_samples[0][middle] == Approx(0.8f / std::sqrt(2.0f)).margin(0.01f)); + CHECK(all.rms_samples[middle] == Approx(std::sqrt((0.32f + 0.02f) / 2.0f)).margin(0.01f)); + + // Out-of-range channel requests return empty envelopes + CHECK(all.channel_vectors(2)[0].empty()); + CHECK(all.channel_vectors(-1)[1].empty()); + + // Normalizing scales every envelope by the loudest mixed peak + AudioWaveformData normalized = waveformer.ExtractChannelSamples(samples_per_second, true); + const float scale = 1.0f / *std::max_element(all.max_samples.begin(), all.max_samples.end()); + CHECK(*std::max_element(normalized.max_samples.begin(), normalized.max_samples.end()) == Approx(1.0f).margin(1e-5)); + CHECK(normalized.channel_max_samples[1][middle] == Approx(all.channel_max_samples[1][middle] * scale).margin(1e-5)); + + reader.Close(); + std::remove(path.c_str()); +} + +TEST_CASE( "Single-pass channel extraction of media without audio is empty", "[libopenshot][audiowaveformer][channels]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "front.png"; + FFmpegReader reader(path.str()); + AudioWaveformer waveformer(&reader); + AudioWaveformData waveform = waveformer.ExtractChannelSamples(200, false); + CHECK(waveform.max_samples.empty()); + CHECK(waveform.channel_count() == 0); + reader.Close(); +} + +TEST_CASE( "Single-pass channel extraction of a video file", "[libopenshot][audiowaveformer][channels]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader reader(path.str()); + reader.Open(); + AudioWaveformer waveformer(&reader); + const int samples_per_second = 20; + AudioWaveformData waveform = waveformer.ExtractChannelSamples(samples_per_second, false); + const size_t expected_total = static_cast(std::ceil(reader.info.duration * samples_per_second)); + CHECK(waveform.max_samples.size() == expected_total); + REQUIRE(waveform.channel_count() == reader.info.channels); + for (int channel = 0; channel < waveform.channel_count(); channel++) { + CHECK(waveform.channel_max_samples[channel].size() == expected_total); + CHECK(waveform.channel_rms_samples[channel].size() == expected_total); + } + // Live reader keeps decoding video for preview + CHECK(reader.info.has_video); + reader.Close(); +} + +TEST_CASE( "Audio-only reader cache stays bounded on long sequential reads", "[libopenshot][ffmpegreader][audiowaveformer][cache]" ) +{ + // An audio-only cache budget sized from width x height never evicted, so every frame of + // the file stayed cached and each insert rescanned the whole cache (quadratic reads). + const std::string path = "AudioWaveformer-cache.wav"; + WriteStereoWav(path, 48000, 60.0, [](int, int64_t index) { + return static_cast(0.5 * std::sin(index * 0.01)); + }); + + FFmpegReader reader(path); + reader.Open(); + REQUIRE_FALSE(reader.info.has_video); + REQUIRE(reader.info.video_length > 1000); + for (int64_t frame = 1; frame <= reader.info.video_length; frame++) { + reader.GetFrame(frame); + } + CHECK(reader.GetCache()->Count() < 300); + CHECK(reader.GetCache()->Count() > 0); + reader.Close(); + std::remove(path.c_str()); +} + +TEST_CASE( "ApplyKeyframes maps time and volume curves onto envelopes", "[libopenshot][audiowaveformer][keyframes]" ) +{ + AudioWaveformer waveformer(nullptr); + const Fraction fps(30, 1); + const int samples_per_second = 30; // one envelope sample per frame keeps the math readable + std::vector ramp(60); + std::vector ramp_rms(60); + for (size_t i = 0; i < ramp.size(); i++) { + ramp[i] = static_cast(i) / 100.0f; + ramp_rms[i] = ramp[i] / 2.0f; + } + + SECTION("no keyframes keeps the envelope") { + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, nullptr, nullptr, fps, samples_per_second); + CHECK(out.max_samples == ramp); + CHECK(out.rms_samples == ramp_rms); + } + + SECTION("identity time curve and unity volume keep the envelope") { + Keyframe time = LinearKeyframe({{1, 1}, {60, 60}}); + Keyframe volume(1.0); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, &time, &volume, fps, samples_per_second); + REQUIRE(out.max_samples.size() == ramp.size()); + for (size_t i = 0; i < ramp.size(); i++) { + CHECK(out.max_samples[i] == Approx(ramp[i]).margin(1e-6)); + CHECK(out.rms_samples[i] == Approx(ramp_rms[i]).margin(1e-6)); + } + } + + SECTION("single-point time curve and empty volume curve are ignored") { + Keyframe time(1.0); + Keyframe volume; + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, &time, &volume, fps, samples_per_second); + CHECK(out.max_samples == ramp); + } + + SECTION("volume curve scales max and RMS") { + Keyframe volume = LinearKeyframe({{1, 0.0}, {60, 1.0}}); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, nullptr, &volume, fps, samples_per_second); + REQUIRE(out.max_samples.size() == ramp.size()); + CHECK(out.max_samples.front() == Approx(0.0f).margin(1e-6)); + CHECK(out.max_samples[30] == Approx(ramp[30] * volume.GetValue(31)).margin(1e-5)); + CHECK(out.rms_samples[30] == Approx(ramp_rms[30] * volume.GetValue(31)).margin(1e-5)); + CHECK(out.max_samples.back() == Approx(ramp.back()).margin(1e-5)); + } + + SECTION("reversed time curve reverses the envelope") { + Keyframe time = LinearKeyframe({{1, 60}, {60, 1}}); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, &time, nullptr, fps, samples_per_second); + REQUIRE(out.max_samples.size() == ramp.size()); + CHECK(out.max_samples.front() == Approx(ramp.back()).margin(1e-6)); + CHECK(out.max_samples.back() == Approx(ramp.front()).margin(1e-6)); + } + + SECTION("slowed time curve doubles the length and interpolates") { + Keyframe time = LinearKeyframe({{1, 1}, {120, 60}}); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, &time, nullptr, fps, samples_per_second); + REQUIRE(out.max_samples.size() == 120); + CHECK(out.max_samples.front() == Approx(ramp.front()).margin(1e-6)); + CHECK(out.max_samples[119] == Approx(ramp.back()).margin(1e-6)); + for (size_t i = 1; i < out.max_samples.size(); i++) { + CHECK(out.max_samples[i] >= out.max_samples[i - 1]); + } + } + + SECTION("short or empty RMS envelopes do not read out of bounds") { + std::vector short_rms(10, 0.5f); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, short_rms, nullptr, nullptr, fps, samples_per_second); + REQUIRE(out.rms_samples.size() == ramp.size()); + CHECK(out.rms_samples[5] == Approx(0.5f)); + CHECK(out.rms_samples[50] == Approx(0.0f)); + AudioWaveformData empty = waveformer.ApplyKeyframes(std::vector(), ramp_rms, nullptr, nullptr, fps, samples_per_second); + CHECK(empty.max_samples.empty()); + } + + SECTION("invalid density or fps returns empty data") { + CHECK(waveformer.ApplyKeyframes(ramp, ramp_rms, nullptr, nullptr, fps, 0).max_samples.empty()); + CHECK(waveformer.ApplyKeyframes(ramp, ramp_rms, nullptr, nullptr, Fraction(0, 1), samples_per_second).max_samples.empty()); + } +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d668d92a0..c909e5f55 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -34,6 +34,7 @@ target_link_libraries(openshot-benchmark openshot) ### TEST SOURCE FILES ### set(OPENSHOT_TESTS + GpuMetrics AdaptivePreroll AudioDeviceManager AudioRecorder @@ -47,6 +48,7 @@ set(OPENSHOT_TESTS Caption Clip Color + ColorManagement Coordinate DummyReader FFmpegCapabilities @@ -64,6 +66,7 @@ set(OPENSHOT_TESTS QtImageReader QtTextReader ReaderBase + RendererBase NativeArm64ProcessOracle Settings ScreenCaptureReader @@ -97,6 +100,10 @@ set(OPENSHOT_TESTS WaveEffect ) +if(ENABLE_GPU_FRAME) + list(APPEND OPENSHOT_TESTS VulkanCompositor VulkanPresentation VulkanDeviceLost) +endif() + # The SIGPIPE crash-handler test is POSIX-only. Do not create an empty # Catch2 executable on Windows, where discovery registers a NOT_BUILT test. if(NOT WIN32) @@ -185,7 +192,19 @@ foreach(tname ${OPENSHOT_TESTS}) SKIP_REGULAR_EXPRESSION "Skipping VAAPI test:" ) endif() - if(tname STREQUAL "QtTextReader") + if(tname MATCHES "^Vulkan") + # Same Vulkan headers as the library (newer than some system packages) + target_include_directories(openshot-${tname}-test PRIVATE ${Vulkan_INCLUDE_DIR}) + if(APPLE) + # Tests build Core Video surfaces to exercise VideoToolbox interop. + target_link_libraries(openshot-${tname}-test "-framework CoreVideo" "-framework CoreFoundation") + endif() + # Report missing GPU, display or interop support as skipped, not passed. + list(APPEND test_properties + SKIP_REGULAR_EXPRESSION "Skipping Vulkan test:" + ) + endif() + if(tname STREQUAL "QtTextReader" OR tname STREQUAL "RendererBase") # Pixel assertions need a font engine; Qt's minimal plugin has no font database. list(APPEND test_properties ENVIRONMENT "QT_QPA_PLATFORM=offscreen") elseif(tname STREQUAL "Caption" OR tname STREQUAL "Timer") diff --git a/tests/CacheMemory.cpp b/tests/CacheMemory.cpp index 6338cac88..62774c406 100644 --- a/tests/CacheMemory.cpp +++ b/tests/CacheMemory.cpp @@ -21,6 +21,41 @@ using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +class CacheTestSurface : public FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 16; } + int GetHeight() const override { return 16; } + int64_t GetBytes() const override { return 1024; } + QImage Materialize() const override { + ++reads; + QImage image(16, 16, QImage::Format_RGBA8888_Premultiplied); + image.fill(Qt::red); + return image; + } +}; +} +TEST_CASE("Hardware cache honors byte budget below the CPU frame floor", "[cache][gpu]") { + CacheMemory cache; + auto surface = std::make_shared(); + for (int i = 1; i <= 5; ++i) { + auto frame = std::make_shared(); + frame->number = i; + frame->AddDeferredImage(surface); + cache.SetMaxBytes(frame->GetBytes() * 2); + cache.Add(frame); + } + CHECK(cache.Count() == 2); + CHECK(cache.GetBytes() <= cache.GetMaxBytes()); + CHECK(surface->reads == 0); + CHECK_FALSE(cache.GetFrame(1)); + REQUIRE(cache.GetFrame(5)); + CHECK(cache.GetFrame(5)->GetImage()->pixelColor(0, 0) == QColor(Qt::red)); +} +#endif + TEST_CASE( "default constructor", "[libopenshot][cachememory]" ) { // Create cache object diff --git a/tests/Clip.cpp b/tests/Clip.cpp index 413cd2fbb..48e7e8403 100644 --- a/tests/Clip.cpp +++ b/tests/Clip.cpp @@ -46,6 +46,58 @@ using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +class AudioOnlyGpuTestReader : public DummyReader { +public: + AudioOnlyGpuTestReader() : DummyReader(Fraction(24, 1), 16, 16, 48000, 2, 5.0) { + info.has_video = false; + info.has_audio = true; + } + std::shared_ptr GetFrame(int64_t n) override { + auto frame = std::make_shared(n, 16, 16, "#00000000", 2000, 2); + frame->SampleRate(48000); + frame->ChannelsLayout(LAYOUT_STEREO); + frame->AddAudioSilence(2000); + return frame; + } +}; +class ClipTestSurface : public FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 16; } + int GetHeight() const override { return 16; } + int64_t GetBytes() const override { return 1024; } + QImage Materialize() const override { + ++reads; + QImage image(16, 16, QImage::Format_RGBA8888_Premultiplied); + image.fill(Qt::red); + return image; + } +}; +} +TEST_CASE("Audio-only clip preserves deferred background", "[clip][gpu]") { + AudioOnlyGpuTestReader reader; + Clip clip(&reader); + clip.Open(); + auto surface = std::make_shared(); + auto background = std::make_shared(); + background->AddDeferredImage(surface); + auto result = clip.GetFrame(background, 1); + CHECK(surface->reads == 0); + CHECK(background->GetDeferredImage() == surface); + CHECK(result->GetDeferredImage() == surface); + reader.info.has_video = true; + clip.has_video = Keyframe(0.0); + TimelineInfoStruct options{}; + auto audio = clip.GetFrame(background, 2, &options); + CHECK(surface->reads == 0); + CHECK(background->GetDeferredImage() == surface); + CHECK(audio->GetAudioSamplesCount() == 2000); + clip.Close(); +} +#endif + namespace { struct ClipTransformCase { ScaleType scale; diff --git a/tests/ColorManagement.cpp b/tests/ColorManagement.cpp new file mode 100644 index 000000000..23d7ea0e7 --- /dev/null +++ b/tests/ColorManagement.cpp @@ -0,0 +1,225 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "openshot_catch.h" + +#include "ColorManagement.h" +#include "FFmpegReader.h" +#include "FFmpegWriter.h" +#include "Frame.h" +#include "Settings.h" + +extern "C" { +#include +#include +#include +} + +using namespace openshot; + +namespace { + +// Patches of examples/color-chart-*.mp4 (see tests/generate-color-charts.py): +// SDR code values, then 600 and 1000 nit highlights. 4x4 grid of 160x96 patches. +struct Patch { const char* name; std::array code; double nits; }; +const Patch kPatches[] = { + {"reference white", {255, 255, 255}, 0}, {"18% grey", {118, 118, 118}, 0}, + {"light skin", {224, 172, 140}, 0}, {"dark skin", {141, 85, 62}, 0}, + {"sky blue", {98, 155, 210}, 0}, {"foliage", {87, 122, 59}, 0}, + {"red", {220, 30, 30}, 0}, {"green", {40, 200, 40}, 0}, {"blue", {30, 40, 220}, 0}, + {"yellow", {240, 220, 40}, 0}, {"near black", {20, 20, 20}, 0}, + {"600 nit highlight", {}, 600}, {"1000 nit highlight", {}, 1000}, +}; + +// The SDR value OpenShot should produce: BT.2408 placement inverted, with +// the shared tone curve for HDR charts (relative_peak > 1). +std::array Expected(const Patch& patch, float relative_peak) { + float rgb[3]; + for (int c = 0; c < 3; ++c) + rgb[c] = patch.nits > 0 ? float(patch.nits / color::kReferenceWhiteNits) + : float(std::pow(patch.code[c] / 255.0, 2.4)); + const float brightest = std::max(rgb[0], std::max(rgb[1], rgb[2])); + const float scale = relative_peak > 1.0f ? color::ToneMap(brightest, relative_peak) / brightest + : std::min(brightest, 1.0f) / brightest; + std::array result; + for (int c = 0; c < 3; ++c) + result[c] = int(std::lround(255.0 * std::pow(std::min(rgb[c] * scale, 1.0f), 1.0 / 2.4))); + return result; +} + +std::array PatchColor(std::shared_ptr frame, int index) { + const int x0 = (index % 4) * 160 + 70, y0 = (index / 4) * 96 + 42; + long sum[3] = {0, 0, 0}; + for (int y = y0; y < y0 + 12; ++y) { + const unsigned char* row = frame->GetPixels(y); + for (int x = x0; x < x0 + 20; ++x) + for (int c = 0; c < 3; ++c) sum[c] += row[x * 4 + c]; + } + return {int(sum[0] / 240), int(sum[1] / 240), int(sum[2] / 240)}; +} + +void CheckChart(const std::string& file, float relative_peak, int tolerance) { + FFmpegReader reader(TEST_MEDIA_PATH + file); + reader.Open(); + auto frame = reader.GetFrame(2); + REQUIRE(frame->GetWidth() == 640); + REQUIRE(frame->GetHeight() == 384); + for (int i = 0; i < int(std::size(kPatches)); ++i) { + const auto expected = Expected(kPatches[i], relative_peak); + const auto actual = PatchColor(frame, i); + INFO(file << " " << kPatches[i].name << " expected (" << expected[0] << "," << expected[1] << "," + << expected[2] << ") got (" << actual[0] << "," << actual[1] << "," << actual[2] << ")"); + for (int c = 0; c < 3; ++c) CHECK(std::abs(actual[c] - expected[c]) <= tolerance); + } + reader.Close(); +} + +// Decode and convert on the CPU. Vulkan builds ignore HARDWARE_DECODER and +// select the CPU pipeline with OPENSHOT_RENDER_BACKEND=cpu. +struct SoftwareDecode { + int previous = Settings::Instance()->HARDWARE_DECODER; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + QByteArray backend; + bool had_backend = false; +#endif + SoftwareDecode() { + Settings::Instance()->HARDWARE_DECODER = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + had_backend = qEnvironmentVariableIsSet("OPENSHOT_RENDER_BACKEND"); + backend = qgetenv("OPENSHOT_RENDER_BACKEND"); + qputenv("OPENSHOT_RENDER_BACKEND", "cpu"); +#endif + } + ~SoftwareDecode() { + Settings::Instance()->HARDWARE_DECODER = previous; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (had_backend) qputenv("OPENSHOT_RENDER_BACKEND", backend); + else qunsetenv("OPENSHOT_RENDER_BACKEND"); +#endif + } +}; + +} // namespace + +TEST_CASE("YUV matrices for decoding and encoding", "[libopenshot][color]") { + CHECK(color::DecodeMatrix(AVCOL_SPC_BT709, 640, 480) == AVCOL_SPC_BT709); + CHECK(color::DecodeMatrix(AVCOL_SPC_BT2020_NCL, 3840, 2160) == AVCOL_SPC_BT2020_NCL); + // Untagged files keep BT.601, as earlier OpenShot exports were written. + CHECK(color::DecodeMatrix(AVCOL_SPC_UNSPECIFIED, 1920, 1080) == AVCOL_SPC_SMPTE170M); + CHECK(color::EncodeMatrix(1920, 1080) == AVCOL_SPC_BT709); + CHECK(color::EncodeMatrix(1280, 720) == AVCOL_SPC_BT709); + CHECK(color::EncodeMatrix(720, 576) == AVCOL_SPC_SMPTE170M); + CHECK(color::EncodeMatrix(640, 480) == AVCOL_SPC_SMPTE170M); +} + +TEST_CASE("Frames describe HDR transfer and gamut", "[libopenshot][color]") { + std::unique_ptr frame(av_frame_alloc(), [](AVFrame* f) { av_frame_free(&f); }); + REQUIRE(frame); + CHECK_FALSE(color::DescribeFrame(frame.get(), 1920, 1080).NeedsConversion()); + + frame->color_trc = AVCOL_TRC_ARIB_STD_B67; + frame->color_primaries = AVCOL_PRI_BT2020; + frame->colorspace = AVCOL_SPC_BT2020_NCL; + auto hlg = color::DescribeFrame(frame.get(), 1920, 1080); + CHECK(hlg.transfer == color::Transfer::HLG); + CHECK(hlg.bt2020); + CHECK(hlg.matrix == AVCOL_SPC_BT2020_NCL); + CHECK(hlg.RelativePeak() == Approx(1000.0 / 203.0)); + + frame->color_trc = AVCOL_TRC_SMPTE2084; + auto pq = color::DescribeFrame(frame.get(), 1920, 1080, 4000.0); + CHECK(pq.transfer == color::Transfer::PQ); + CHECK(pq.RelativePeak() == Approx(4000.0 / 203.0)); + + // BT.2020 SDR still needs gamut conversion; unspecified primaries follow the matrix. + frame->color_trc = AVCOL_TRC_BT2020_10; + frame->color_primaries = AVCOL_PRI_UNSPECIFIED; + auto wide = color::DescribeFrame(frame.get(), 1920, 1080); + CHECK(wide.transfer == color::Transfer::Sdr); + CHECK(wide.bt2020); + CHECK(wide.NeedsConversion()); +} + +TEST_CASE("Tone curve keeps SDR levels and compresses highlights", "[libopenshot][color]") { + const float peak = 1000.0f / 203.0f; + CHECK(color::ToneMap(0.5f, peak) == 0.5f); + CHECK(color::ToneMap(color::kToneKnee, peak) == Approx(color::kToneKnee)); + CHECK(color::ToneMap(peak, peak) == Approx(1.0f)); + CHECK(color::ToneMap(2.0f * peak, peak) == 1.0f); + CHECK(color::ToneMap(3.0f, 1.0f) == 1.0f); + float previous = 0.0f; + for (float value = 0.0f; value <= peak; value += 0.01f) { + const float mapped = color::ToneMap(value, peak); + CHECK(mapped >= previous); + previous = mapped; + } + // Reference white stays bright, leaving headroom for highlights. + CHECK(color::ToneMap(1.0f, peak) > 0.85f); + CHECK(color::ToneMap(1.0f, peak) < 1.0f); +} + +TEST_CASE("CPU decoding converts SDR, HLG and PQ charts to SDR BT.709", "[libopenshot][color][ffmpegreader]") { + SoftwareDecode software; + const float hdr_peak = float(color::kDefaultPeakNits / color::kReferenceWhiteNits); + CheckChart("color-chart-sdr-bt709.mp4", 1.0f, 4); + CheckChart("color-chart-hlg.mp4", hdr_peak, 4); + CheckChart("color-chart-pq.mp4", hdr_peak, 4); +} + +TEST_CASE("Exports are tagged and round trip their colors", "[libopenshot][color][ffmpegwriter]") { + if (!avcodec_find_encoder_by_name("libx264")) { + WARN("libx264 unavailable; skipping"); + return; + } + SoftwareDecode software; + struct Size { int width, height; AVColorSpace matrix; }; + for (const Size size : {Size{1280, 720, AVCOL_SPC_BT709}, Size{640, 480, AVCOL_SPC_SMPTE170M}}) { + QTemporaryDir directory; + REQUIRE(directory.isValid()); + const std::string path = directory.filePath("color.mp4").toStdString(); + const char* colors[] = {"#dc1e1e", "#28c828", "#1e28dc", "#e0ac8c"}; + { + FFmpegWriter writer(path); + writer.SetVideoOptions(true, "libx264", Fraction(30, 1), size.width, size.height, + Fraction(1, 1), false, false, 20000000); + writer.Open(); + for (int number = 1; number <= 4; ++number) + writer.WriteFrame(std::make_shared(number, size.width, size.height, colors[number - 1])); + writer.Close(); + } + + AVFormatContext* input = nullptr; + REQUIRE(avformat_open_input(&input, path.c_str(), nullptr, nullptr) == 0); + REQUIRE(avformat_find_stream_info(input, nullptr) >= 0); + const int index = av_find_best_stream(input, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + REQUIRE(index >= 0); + const AVCodecParameters* parameters = input->streams[index]->codecpar; + CHECK(parameters->color_space == size.matrix); + CHECK(parameters->color_primaries == AVCOL_PRI_BT709); + CHECK(parameters->color_trc == AVCOL_TRC_BT709); + CHECK(parameters->color_range == AVCOL_RANGE_MPEG); + avformat_close_input(&input); + + FFmpegReader reader(path); + reader.Open(); + for (int number = 1; number <= 4; ++number) { + const QColor expected(colors[number - 1]); + const QColor actual = QColor::fromRgba(reader.GetFrame(number)->GetImage()->pixel(size.width / 2, size.height / 2)); + INFO(size.width << "x" << size.height << " " << colors[number - 1] << " got " << actual.name().toStdString()); + CHECK(std::abs(actual.red() - expected.red()) <= 4); + CHECK(std::abs(actual.green() - expected.green()) <= 4); + CHECK(std::abs(actual.blue() - expected.blue()) <= 4); + } + reader.Close(); + } +} diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index a91e73a16..c02bc49cf 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -17,6 +17,11 @@ #include #include #include +// Windows CRT equivalents for these tests (all calls overwrite). +#ifdef _WIN32 +#define setenv(name, value, overwrite) _putenv_s(name, value) +#define unsetenv(name) _putenv_s(name, "") +#endif #include #include #include @@ -95,6 +100,28 @@ struct HardwareDecoderSettingsGuard { } }; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +struct RenderBackendGuard { + std::string previous; + bool had_previous = false; + + explicit RenderBackendGuard(const char* backend) { + if (const char* value = std::getenv("OPENSHOT_RENDER_BACKEND")) { + previous = value; + had_previous = true; + } + setenv("OPENSHOT_RENDER_BACKEND", backend, 1); + } + + ~RenderBackendGuard() { + if (had_previous) + setenv("OPENSHOT_RENDER_BACKEND", previous.c_str(), 1); + else + unsetenv("OPENSHOT_RENDER_BACKEND"); + } +}; +#endif + struct TemporaryFileGuard { std::string path; @@ -916,6 +943,10 @@ TEST_CASE( "HardwareDecodeSuccessful_IsFalse_WhenHardwareDecodeIsDisabled", "[li HardwareDecoderSettingsGuard guard; Settings::Instance()->HARDWARE_DECODER = 0; Settings::Instance()->HW_DE_DEVICE_SET = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // GPU builds select Vulkan decoding by render backend, not by decoder setting. + RenderBackendGuard backend("cpu"); +#endif std::stringstream path; path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; @@ -926,6 +957,9 @@ TEST_CASE( "HardwareDecodeSuccessful_IsFalse_WhenHardwareDecodeIsDisabled", "[li auto frame = r.GetFrame(1); REQUIRE(frame->has_image_data); CHECK_FALSE(r.HardwareDecodeSuccessful()); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + CHECK_FALSE(frame->GetDeferredImage()); +#endif r.Close(); } diff --git a/tests/Frame.cpp b/tests/Frame.cpp index e352293f6..c64e8a35b 100644 --- a/tests/Frame.cpp +++ b/tests/Frame.cpp @@ -277,3 +277,79 @@ TEST_CASE( "Convert_Image", "[libopenshot][opencv][frame]" ) CHECK(cvimage.channels() == 3); } #endif + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +class TestFrameImage : public FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 8; } + int GetHeight() const override { return 6; } + int64_t GetBytes() const override { return 512; } + QImage Materialize() const override { + ++reads; + QImage result(8, 6, QImage::Format_RGBA8888_Premultiplied); + result.fill(Qt::red); + return result; + } +}; +} + +TEST_CASE("Deferred frame images preserve ownership and materialize lazily", "[Frame]") { + auto surface = std::make_shared(); + Frame frame; + const auto baseline_bytes = frame.GetBytes(); + frame.AddDeferredImage(surface); + CHECK(frame.GetWidth() == 8); + CHECK(frame.GetHeight() == 6); + CHECK(frame.has_image_data); + CHECK(frame.GetBytes() == baseline_bytes + 512); + Frame copy(frame); + CHECK(copy.GetDeferredImage() == surface); + CHECK(surface->reads == 0); + auto pixels = frame.GetImage(); + CHECK(surface->reads == 1); + CHECK_FALSE(frame.GetDeferredImage()); + CHECK(frame.GetImage() == pixels); + CHECK(surface->reads == 1); + pixels->fill(Qt::blue); + CHECK(copy.GetImage()->pixelColor(0, 0) == QColor(Qt::red)); + CHECK(surface->reads == 2); +} + +TEST_CASE("Replacing deferred frame images does not read discarded pixels", "[Frame]") { + auto surface = std::make_shared(); + Frame frame; + frame.AddDeferredImage(surface); + frame.AddColor(Qt::green); + CHECK_FALSE(frame.GetDeferredImage()); + CHECK(surface->reads == 0); + frame.AddDeferredImage(surface); + auto image = std::make_shared(4, 4, QImage::Format_RGBA8888_Premultiplied); + image->fill(Qt::blue); + frame.AddImage(image); + CHECK_FALSE(frame.GetDeferredImage()); + CHECK(surface->reads == 0); + frame.AddDeferredImage(surface); + Frame empty; + frame = empty; + CHECK_FALSE(frame.GetDeferredImage()); + CHECK(surface->reads == 0); + CHECK(frame.GetImage()->pixelColor(0, 0) == QColor(Qt::black)); +} + +TEST_CASE("Deferred frame pixel access and field writes materialize correctly", "[Frame]") { + auto surface = std::make_shared(); + Frame frame; + frame.AddDeferredImage(surface); + CHECK(frame.GetPixels()[0] == 255); + CHECK(surface->reads == 1); + frame.AddDeferredImage(surface); + auto field = std::make_shared(8, 6, QImage::Format_RGBA8888_Premultiplied); + field->fill(Qt::blue); + frame.AddImage(field, true); + CHECK(surface->reads == 2); + CHECK(frame.GetImage()->pixelColor(0, 0) == QColor(Qt::red)); + CHECK(frame.GetImage()->pixelColor(0, 1) == QColor(Qt::blue)); +} +#endif diff --git a/tests/FrameMapper.cpp b/tests/FrameMapper.cpp index 781203efb..3d87ed614 100644 --- a/tests/FrameMapper.cpp +++ b/tests/FrameMapper.cpp @@ -27,6 +27,47 @@ using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +class MapperSurface : public FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 16; } + int GetHeight() const override { return 16; } + int64_t GetBytes() const override { return 1024; } + QImage Materialize() const override { + ++reads; + QImage result(16, 16, QImage::Format_RGBA8888_Premultiplied); + result.fill(Qt::red); + return result; + } +}; +class SurfaceReader : public DummyReader { +public: + std::shared_ptr surface = std::make_shared(); + SurfaceReader() : DummyReader(Fraction(24, 1), 16, 16, 48000, 2, 5.0) {} + std::shared_ptr GetFrame(int64_t number) override { + auto frame = DummyReader::GetFrame(number); + frame->AddDeferredImage(surface); + return frame; + } +}; +} + +TEST_CASE("Progressive frame mapping retains deferred image storage", "[libopenshot][framemapper]") { + SurfaceReader reader; + FrameMapper mapper(&reader, Fraction(30, 1), PULLDOWN_NONE, 48000, 2, LAYOUT_STEREO); + mapper.Open(); + for (int i = 1; i <= 8; ++i) { + auto frame = mapper.GetFrame(i); + REQUIRE(frame->GetDeferredImage() == reader.surface); + CHECK(frame->number == i); + } + CHECK(reader.surface->reads == 0); + mapper.Close(); +} +#endif + TEST_CASE( "NoOp_GetMappedFrame", "[libopenshot][framemapper]" ) { // Create a reader diff --git a/tests/GpuMetrics.cpp b/tests/GpuMetrics.cpp new file mode 100644 index 000000000..10d28ad8d --- /dev/null +++ b/tests/GpuMetrics.cpp @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "openshot_catch.h" +#include "GpuMetrics.h" +#include "Json.h" +#include +using namespace openshot; +using namespace openshot::gpu_metrics; +namespace { +Json::Value snapshot(bool exporting = false) { + Json::Reader reader; + Json::Value value; + REQUIRE(reader.parse(GpuMetricsSnapshot(exporting), value)); + return value; +} +} +TEST_CASE("GPU metrics isolate work scopes and exports", "[GpuMetrics]") { + ResetGpuMetrics(); ResetGpuMetrics(true); + Record(DecodeCpu); // Unrelated reader/thumbnail work is not playback. + { Scope scope(false); Record(DecodeGpu); Record(RenderGpu); } + { Scope scope(true); Record(EncodeGpu); Record(Upload); } + REQUIRE(snapshot()["decode_cpu"].asUInt64() == 0); + REQUIRE(snapshot()["decode_gpu"].asUInt64() == 1); + REQUIRE(snapshot()["encode_gpu"].asUInt64() == 0); + REQUIRE(snapshot(true)["encode_gpu"].asUInt64() == 1); + REQUIRE(snapshot()["copy_free"].asUInt64() == 1); +} +TEST_CASE("GPU metrics count nested work once and attribute readbacks", "[GpuMetrics]") { + ResetGpuMetrics(); + { Scope outer(false); Record(RenderGpu); + { Scope nested(false); Reason reason("Effect"); Record(Readback); } + } + auto value = snapshot(); + REQUIRE(value["gpu_requests"].asUInt64() == 1); + REQUIRE(value["copy_free"].asUInt64() == 0); + REQUIRE(value["readback_reasons"]["Effect"].asUInt64() == 1); +} +TEST_CASE("GPU metric reset excludes old in-flight work", "[GpuMetrics]") { + ResetGpuMetrics(); + { Scope old(false); Record(RenderGpu); ResetGpuMetrics(); Record(Readback); } + REQUIRE(snapshot()["gpu_requests"].asUInt64() == 0); + REQUIRE(snapshot()["readbacks"].asUInt64() == 0); + { Scope fresh(false); Record(RenderGpu); } + REQUIRE(snapshot()["copy_free"].asUInt64() == 1); +} +TEST_CASE("GPU counters support parallel requests", "[GpuMetrics]") { + ResetGpuMetrics(); + auto work = [] { for (int i = 0; i < 1000; ++i) { Scope scope(false); Record(DecodeGpu); } }; + std::thread first(work), second(work); first.join(); second.join(); + REQUIRE(snapshot()["decode_gpu"].asUInt64() == 2000); + REQUIRE(snapshot()["gpu_requests"].asUInt64() == 2000); +} + +TEST_CASE("Failed GPU work is not a copy-free success", "[GpuMetrics]") { + ResetGpuMetrics(); + try { Scope scope(false); Record(DecodeGpu); throw 1; } catch (int) {} + REQUIRE(snapshot()["gpu_requests"].asUInt64() == 0); +} + +TEST_CASE("Export residency includes cached frames read back by a CPU encoder", "[GpuMetrics]") { + ResetGpuMetrics(true); + // The cache worker completes GPU work in a separate request. + { Scope render(true); Record(RenderGpu); } + { Scope writer(true); RecordExportFrame(); Reason reason("Encoder"); + Record(Readback); Record(EncodeCpu); } + auto value = snapshot(true); + REQUIRE(value["copy_free"].asUInt64() == 1); // Operation-only metric. + REQUIRE(value["export_frames"].asUInt64() == 1); + REQUIRE(value["export_copy_free"].asUInt64() == 0); + REQUIRE(value["readback_reasons"]["Encoder"].asUInt64() == 1); +} + +TEST_CASE("Export residency counts GPU writes and excludes failed requests and old sessions", "[GpuMetrics]") { + ResetGpuMetrics(true); + { Scope writer(true); RecordExportFrame(); Record(EncodeGpu); } + try { Scope writer(true); RecordExportFrame(); Record(EncodeGpu); throw 1; } catch (int) {} + REQUIRE(snapshot(true)["export_frames"].asUInt64() == 1); + REQUIRE(snapshot(true)["export_copy_free"].asUInt64() == 1); + { Scope old(true); RecordExportFrame(); Record(EncodeGpu); ResetGpuMetrics(true); } + REQUIRE(snapshot(true)["export_frames"].asUInt64() == 0); +} diff --git a/tests/QtImageReader.cpp b/tests/QtImageReader.cpp index 0c8647342..d41192b63 100644 --- a/tests/QtImageReader.cpp +++ b/tests/QtImageReader.cpp @@ -13,6 +13,7 @@ #include "openshot_catch.h" #include +#include #include #include "QtImageReader.h" @@ -156,3 +157,29 @@ TEST_CASE( "Max_Decode_Size_SVG_QtImageReader", "[libopenshot][qtimagereader]" ) r.Close(); } + +TEST_CASE("Artwork resolution follows maximum animation scale and preview size", "[libopenshot][qtimagereader]") { + QTemporaryDir directory; + const auto path=directory.filePath("large.png"); + QImage image(1024,1024,QImage::Format_RGBA8888_Premultiplied); + image.fill(Qt::red); + REQUIRE(image.save(path)); + QtImageReader reader(path.toStdString()); + Clip clip(&reader); + clip.scale_x.AddPoint(1,1.0,LINEAR); + clip.scale_x.AddPoint(10,2.0,LINEAR); + clip.scale_y=clip.scale_x; + Timeline timeline(128,128,Fraction(24,1),48000,2,LAYOUT_STEREO); + timeline.AddClip(&clip); + timeline.Open(); + clip.Reader()->Open(); + auto first=reader.GetFrame(1)->GetImage(); + CHECK(first->size()==QSize(256,256)); + CHECK(reader.GetFrame(8)->GetImage()->cacheKey()==first->cacheKey()); + clip.scale_x.AddPoint(10,4.0,LINEAR); + clip.scale_y=clip.scale_x; + CHECK(reader.GetFrame(1)->GetWidth()==512); + timeline.preview_width=timeline.preview_height=64; + CHECK(reader.GetFrame(1)->GetWidth()==256); + timeline.Close(); +} diff --git a/tests/RendererBase.cpp b/tests/RendererBase.cpp new file mode 100644 index 000000000..b353335e0 --- /dev/null +++ b/tests/RendererBase.cpp @@ -0,0 +1,102 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "openshot_catch.h" +#include "RendererBase.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "Qt/VideoRenderer.h" +#include +#include +#include +#endif + +namespace { +class ImageRenderer : public openshot::RendererBase { +public: + std::shared_ptr received; + void OverrideWidget(uintptr_t) override {} +protected: + void render(std::shared_ptr image) override { received = image; } +}; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +class RetainedRenderer : public ImageRenderer { +public: + std::shared_ptr frame; +protected: + void renderFrame(const std::shared_ptr& value) override { frame = value; } +}; +class DeferredImage : public openshot::FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 2; } + int GetHeight() const override { return 2; } + int64_t GetBytes() const override { return 16; } + QImage Materialize() const override { + ++reads; + QImage result(2, 2, QImage::Format_RGBA8888_Premultiplied); + result.fill(Qt::red); + return result; + } +}; +class NativePresenter : public QWidget, public FramePresenter { +public: + std::shared_ptr received; + void PresentFrame(const std::shared_ptr& frame) override { received = frame; } +}; +#endif +} + +TEST_CASE("Image renderer preserves CPU presentation", "[renderer]") { + ImageRenderer renderer; + renderer.paint(nullptr); + CHECK_FALSE(renderer.received); + auto frame = std::make_shared(); + frame->AddColor(2, 2, "#ff0000"); + renderer.paint(frame); + REQUIRE(renderer.received); + CHECK(renderer.received->pixelColor(0, 0) == QColor(Qt::red)); +} + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +TEST_CASE("Frame renderer retains surfaces until CPU fallback is requested", "[renderer]") { + auto surface = std::make_shared(); + auto frame = std::make_shared(); + frame->AddDeferredImage(surface); + RetainedRenderer retained; + retained.paint(frame); + CHECK(retained.frame == frame); + CHECK(surface->reads == 0); + CHECK_FALSE(retained.received); + ImageRenderer fallback; + fallback.paint(frame); + CHECK(surface->reads == 1); + REQUIRE(fallback.received); + CHECK(fallback.received->pixelColor(0, 0) == QColor(Qt::red)); +} + +TEST_CASE("Qt presenter queues owned frames without materializing pixels", "[renderer]") { + if (qEnvironmentVariableIsEmpty("DISPLAY") && qEnvironmentVariableIsEmpty("QT_QPA_PLATFORM")) + qputenv("QT_QPA_PLATFORM", "offscreen"); + int argc = 1; + char name[] = "renderer-test"; + char* argv[] = {name, nullptr}; + QApplication app(argc, argv); + NativePresenter widget; + VideoRenderer renderer; + renderer.OverrideWidget(reinterpret_cast(&widget)); + auto surface = std::make_shared(); + auto frame = std::make_shared(); + frame->AddDeferredImage(surface); + std::weak_ptr lifetime = frame; + renderer.paint(frame); + frame.reset(); + CHECK_FALSE(lifetime.expired()); + CHECK_FALSE(widget.received); + // Platform plugins may need several passes to deliver the queued frame. + QDeadlineTimer deadline(1000); + while (!widget.received && !deadline.hasExpired()) + QCoreApplication::processEvents(QEventLoop::AllEvents, 10); + REQUIRE(widget.received); + CHECK(surface->reads == 0); + CHECK(widget.received->GetDeferredImage() == surface); +} +#endif diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp new file mode 100644 index 000000000..2d5b530ee --- /dev/null +++ b/tests/VulkanCompositor.cpp @@ -0,0 +1,1396 @@ +#ifdef __APPLE__ +#include "MacVulkanInterop.h" +#include +#endif +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "openshot_catch.h" +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "VulkanEncodeConverter.h" +#include "ColorManagement.h" +#include "Settings.h" +#include +#include "GpuDevices.h" +#include "GpuMetrics.h" +#include "FrameScope.h" +#include "FFmpegReader.h" +#include "FFmpegWriter.h" +#include +#include +#include +#include "Clip.h" +#include "effects/Saturation.h" +#include "effects/Crop.h" +#include "effects/ColorGrade.h" +#include "effects/ColorMap.h" +#include +#include +#include +#include "effects/Mask.h" +#include "effects/Blur.h" +#include "QtImageReader.h" +#include "Timeline.h" +#include "DummyReader.h" +#include +#include +#include +extern "C" { +#include +#include +#include +#include +} +namespace { +struct BufferDelete { void operator()(AVBufferRef* p) const { av_buffer_unref(&p); } }; +struct FrameDelete { void operator()(AVFrame* p) const { av_frame_free(&p); } }; +using Buffer=std::unique_ptr; +using Frame=std::unique_ptr; +// CTest reports this message as skipped; see tests/CMakeLists.txt. +bool VulkanAvailable() { + if(!openshot::VulkanRenderingEnabled()) { + WARN("Skipping Vulkan test: CPU mode, or no Vulkan library"); + return false; + } + AVBufferRef* device=nullptr; + if(openshot::CreateSharedVulkanDevice(&device)<0) { + WARN("Skipping Vulkan test: no Vulkan device is available"); + return false; + } + av_buffer_unref(&device); + return true; +} +class PatternReader : public openshot::DummyReader { + std::shared_ptr surface; + QImage pixels; +public: + PatternReader(std::shared_ptr image, bool gpu) + : DummyReader(openshot::Fraction(24,1),image->GetWidth(),image->GetHeight(),48000,2,5.0), + surface(gpu?image:nullptr),pixels(gpu?QImage():image->Materialize()) {} + std::shared_ptr GetFrame(int64_t n) override { + auto frame=std::make_shared(n,info.width,info.height,"#00000000"); + if(surface) frame->AddDeferredImage(surface); + else frame->AddImage(std::make_shared(pixels)); + return frame; + } +}; +} +TEST_CASE("Vulkan GPU listing reports the card OpenShot renders with", "[vulkan][hardware]") { + if(!VulkanAvailable()) return; + const auto active=openshot::ActiveGpuDevice(); + REQUIRE(!active.name.empty()); + CHECK(active.active); + CHECK(active.type!="software"); + CHECK(active.uuid.size()==32); + int active_count=0; + for(const auto& gpu : openshot::ListGpuDevices()) { + CHECK(!gpu.name.empty()); + if(gpu.active) { ++active_count; CHECK(gpu.uuid==active.uuid); } + } + CHECK(active_count==1); + // Every reported encoder exists in this FFmpeg build; the list is cached. + const auto encoders=openshot::GpuEncoders(); + for(const auto& name : encoders) CHECK(avcodec_find_encoder_by_name(name.c_str())); + CHECK(openshot::GpuEncoders()==encoders); +} +TEST_CASE("AV1 prefers Vulkan decoding over the default software decoder", "[vulkan][hardware][reader]") { + if (!VulkanAvailable()) return; + if (!openshot::VulkanAv1DecodeSupported()) { + WARN("Skipping Vulkan test: no Vulkan AV1 decode extension"); + return; + } + openshot::FFmpegReader reader(TEST_MEDIA_PATH "test_video_sync.mp4"); + reader.Open(); + auto frame = reader.GetFrame(90); + REQUIRE(frame->GetDeferredImage()); + CHECK(reader.HardwareDecodeSuccessful()); + auto image = frame->GetImage(); + REQUIRE(image); + CHECK(image->pixelColor(930, 820).red() > 250); + reader.Close(); +} + +TEST_CASE("VA-API AV1 export accepts a requested bitrate", "[vulkan][hardware][writer]") { + if(!VulkanAvailable()) return; + const auto encoders=openshot::GpuEncoders(); + if(std::find(encoders.begin(),encoders.end(),"av1_vaapi")==encoders.end()){ + WARN("Skipping Vulkan test: no VA-API AV1 encoder"); + return; + } + QTemporaryDir directory; + REQUIRE(directory.isValid()); + const auto path=directory.filePath("av1.mp4").toStdString(); + openshot::FFmpegWriter writer(path); + writer.SetVideoOptions(true,"av1_vaapi",openshot::Fraction(24,1),256,256,openshot::Fraction(1,1),false,false,1000000); + REQUIRE_NOTHROW(writer.Open()); + for(int n=1;n<=3;++n) writer.WriteFrame(std::make_shared(n,256,256,"red")); + writer.Close(); + openshot::FFmpegReader reader(path); + reader.Open(); + const auto* decoder=avcodec_find_decoder_by_name(reader.info.vcodec.c_str()); + REQUIRE(decoder); + CHECK(decoder->id==AV_CODEC_ID_AV1); + auto image=reader.GetFrame(1)->GetImage(); + REQUIRE(image); + if (openshot::VulkanAv1DecodeSupported()) + CHECK(reader.HardwareDecodeSuccessful()); + CHECK(image->pixelColor(128,128).red()>200); + CHECK(image->pixelColor(128,128).blue()<30); + reader.Close(); +} +TEST_CASE("Native Vulkan export keeps decoded and composited frames on the GPU", "[vulkan][hardware][writer]") { + if(!VulkanAvailable()) return; + const auto encoders=openshot::GpuEncoders(); + if(std::find(encoders.begin(),encoders.end(),"h264_vulkan")==encoders.end()){ + WARN("Skipping Vulkan test: no native Vulkan H.264 encoder"); + return; + } + QTemporaryDir directory; + REQUIRE(directory.isValid()); + // Repeat exports to exercise encoder surface reuse and owner cleanup. + for(int pass=0;pass<2;++pass){ + const int width=pass?640:256,height=pass?360:256; + openshot::FFmpegReader base_reader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + openshot::FFmpegReader overlay_reader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + openshot::Clip base(&base_reader),overlay(&overlay_reader); + openshot::Timeline timeline(width,height,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.SetMaxSize(width,height); + base.Layer(0);overlay.Layer(1); + overlay.scale_x=openshot::Keyframe(0.7);overlay.rotation=openshot::Keyframe(12.0); + overlay.alpha=openshot::Keyframe(0.6); + timeline.AddClip(&base);timeline.AddClip(&overlay);timeline.Open(); + const auto path=directory.filePath(QString("native-%1.mp4").arg(pass)).toStdString(); + openshot::FFmpegWriter writer(path); + writer.SetVideoOptions(true,"h264_vulkan",openshot::Fraction(24,1),width,height,openshot::Fraction(1,1),false,false,1000000); + writer.Open(); + REQUIRE(writer.info.vcodec=="h264_vulkan"); // CPU fallback must not pass this test. + std::shared_ptr expected_frame; + const auto readbacks=openshot::VulkanFrameReadbackCount(); + for(int n=1;n<=24;++n){ + auto frame=timeline.GetFrame(n); + REQUIRE(frame->GetDeferredImage()); + writer.WriteFrame(frame); + REQUIRE(frame->GetDeferredImage()); + if (n == 12) expected_frame = frame; + } + writer.Close(); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + CHECK(base_reader.HardwareDecodeSuccessful()); + CHECK(overlay_reader.HardwareDecodeSuccessful()); + timeline.Close(); + // Decode the finished bitstream on the CPU only after measuring export. + openshot::FFmpegReader reader(path); + reader.Open(); + CHECK(reader.info.width==width);CHECK(reader.info.height==height); + CHECK(reader.info.video_length==24); + auto image=reader.GetFrame(12)->GetImage(); + REQUIRE(image);CHECK(image->width()==width);CHECK(image->height()==height); + reader.Close(); + } +} +TEST_CASE("NVENC export accepts CPU and Vulkan frames without changing the requested encoder", "[vulkan][hardware][writer][nvenc]") { + if (!VulkanAvailable()) return; + const auto encoders = openshot::GpuEncoders(); + if (std::find(encoders.begin(), encoders.end(), "h264_nvenc") == encoders.end()) { + WARN("Skipping Vulkan test: no usable NVENC encoder"); + return; + } + QTemporaryDir directory; + REQUIRE(directory.isValid()); + QImage pixels(256, 256, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < pixels.height(); ++y) + for (int x = 0; x < pixels.width(); ++x) + pixels.setPixelColor(x, y, y < 128 ? Qt::red : Qt::blue); + const auto uploaded = openshot::UploadVulkanImage(pixels); + REQUIRE(uploaded); + for (const std::string codec : {"h264_nvenc", "hevc_nvenc"}) { + if (std::find(encoders.begin(), encoders.end(), codec) == encoders.end()) continue; + for (bool gpu : {false, true}) { + INFO("encoder=" << codec << " GPU input=" << gpu); + const auto path = directory.filePath(QString::fromStdString(codec + (gpu ? "-gpu.mp4" : "-cpu.mp4"))).toStdString(); + openshot::FFmpegWriter writer(path); + writer.SetVideoOptions(true, codec, openshot::Fraction(24, 1), 256, 256, + openshot::Fraction(1, 1), false, false, 2000000); + writer.Open(); + REQUIRE(writer.info.vcodec == codec); + const auto before = openshot::VulkanFrameReadbackCount(); + for (int n = 1; n <= 8; ++n) { + auto frame = std::make_shared(n, 256, 256, "black"); + if (gpu) frame->AddDeferredImage(uploaded); + else frame->AddImage(std::make_shared(pixels)); + writer.WriteFrame(frame); + } + writer.Close(); + if (gpu) CHECK(openshot::VulkanFrameReadbackCount() == before); + openshot::FFmpegReader reader(path); + reader.Open(); + CHECK(reader.info.video_length == 8); + const auto image = reader.GetFrame(4)->GetImage(); + REQUIRE(image); + CHECK(image->pixelColor(128, 64).red() > 220); + CHECK(image->pixelColor(128, 64).blue() < 25); + CHECK(image->pixelColor(128, 192).blue() > 220); + CHECK(image->pixelColor(128, 192).red() < 25); + reader.Close(); + } + } +} +TEST_CASE("Vulkan export accepts CPU frames and keeps writers independent", "[vulkan][writer][fallback]") { + // Run this with OPENSHOT_RENDER_BACKEND=cpu, an invalid Vulkan device, or + // an older FFmpeg too: the requested codec must fall back before the header. + if(!avcodec_find_encoder_by_name("libx264")){ + WARN("Skipping Vulkan test: CPU fallback test requires libx264"); + return; + } + QTemporaryDir directory; + REQUIRE(directory.isValid()); + const auto gpu_path=directory.filePath("gpu-or-fallback.mp4").toStdString(); + const auto cpu_path=directory.filePath("cpu.mp4").toStdString(); + openshot::FFmpegWriter gpu(gpu_path),cpu(cpu_path); + gpu.SetVideoOptions(true,"h264_vulkan",openshot::Fraction(24,1),256,256,openshot::Fraction(1,1),false,false,1000000); + cpu.SetVideoOptions(true,"libx264",openshot::Fraction(24,1),256,256,openshot::Fraction(1,1),false,false,1000000); + gpu.Open();cpu.Open(); + if(!openshot::VulkanRenderingEnabled()) CHECK(gpu.info.vcodec!="h264_vulkan"); + for(int n=1;n<=3;++n){ + gpu.WriteFrame(std::make_shared(n,256,256,"red")); + cpu.WriteFrame(std::make_shared(n,256,256,"blue")); + } + gpu.Close();cpu.Close(); + for(const auto& item:std::vector>{{gpu_path,Qt::red},{cpu_path,Qt::blue}}){ + openshot::FFmpegReader reader(item.first); + reader.Open(); + CHECK(reader.info.video_length==3); + auto image=reader.GetFrame(2)->GetImage(); + REQUIRE(image); + const auto actual=image->pixelColor(128,128); + CHECK(std::abs(actual.red()-item.second.red())<25); + CHECK(std::abs(actual.blue()-item.second.blue())<25); + reader.Close(); + } +} +TEST_CASE("Vulkan export recovers from an incompatible GPU surface", "[vulkan][hardware][fallback]") { + if(!VulkanAvailable()) return; + const auto encoders=openshot::GpuEncoders(); + if(std::find(encoders.begin(),encoders.end(),"h264_vulkan")==encoders.end()){ + WARN("Skipping Vulkan test: no native Vulkan H.264 encoder"); + return; + } + // A different logical device cannot write into the encoder's pool. The + // original frame is still readable, so this failure can use CPU pixels. + AVBufferRef* other=nullptr; + const auto name=openshot::ActiveGpuDevice().name; + REQUIRE(av_hwdevice_ctx_create(&other,AV_HWDEVICE_TYPE_VULKAN,name.c_str(),nullptr,0)>=0); + Buffer device(other),pool(av_hwframe_ctx_alloc(other)); + REQUIRE(pool); + auto* context=reinterpret_cast(pool->data); + context->format=AV_PIX_FMT_VULKAN;context->sw_format=AV_PIX_FMT_RGBA; + context->width=256;context->height=256; + REQUIRE(av_hwframe_ctx_init(pool.get())>=0); + Frame cpu(av_frame_alloc()),gpu(av_frame_alloc()); + cpu->format=AV_PIX_FMT_RGBA;cpu->width=256;cpu->height=256; + REQUIRE(av_frame_get_buffer(cpu.get(),0)>=0); + for(int y=0;y<256;++y)for(int x=0;x<256;++x){ + auto* pixel=cpu->data[0]+y*cpu->linesize[0]+x*4; + pixel[0]=255;pixel[1]=0;pixel[2]=0;pixel[3]=255; + } + REQUIRE(av_hwframe_get_buffer(pool.get(),gpu.get(),0)>=0); + REQUIRE(av_hwframe_transfer_data(gpu.get(),cpu.get(),0)>=0); + QTemporaryDir directory; + REQUIRE(directory.isValid()); + const auto path=directory.filePath("fallback.mp4").toStdString(); + openshot::FFmpegWriter writer(path); + writer.SetVideoOptions(true,"h264_vulkan",openshot::Fraction(24,1),256,256,openshot::Fraction(1,1),false,false,1000000); + writer.Open(); + const auto before=openshot::VulkanFrameReadbackCount(); + for(int n=1;n<=3;++n){ + auto frame=std::make_shared(n,256,256,"black"); + frame->AddDeferredImage(std::make_shared(gpu.get())); + REQUIRE_NOTHROW(writer.WriteFrame(frame)); + CHECK_FALSE(frame->GetDeferredImage()); + } + writer.Close(); + CHECK(openshot::VulkanFrameReadbackCount()==before+3); + openshot::FFmpegReader reader(path); + reader.Open(); + CHECK(reader.info.video_length==3); + const auto image=reader.GetFrame(2)->GetImage(); + REQUIRE(image);CHECK(image->pixelColor(128,128).red()>220); + CHECK(image->pixelColor(128,128).blue()<25); + reader.Close(); +} +TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan][hardware]") { + if(!VulkanAvailable()) return; + AVBufferRef* device_ptr=nullptr; + REQUIRE(openshot::CreateSharedVulkanDevice(&device_ptr)>=0); + Buffer device(device_ptr); + Buffer pool(av_hwframe_ctx_alloc(device.get())); + REQUIRE(pool); + auto* context=reinterpret_cast(pool->data); + context->format=AV_PIX_FMT_VULKAN;context->sw_format=AV_PIX_FMT_RGBA; + context->width=16;context->height=16; + REQUIRE(av_hwframe_ctx_init(pool.get())>=0); + Frame cpu(av_frame_alloc()),gpu(av_frame_alloc()); + REQUIRE(cpu);REQUIRE(gpu); + cpu->format=AV_PIX_FMT_RGBA;cpu->width=16;cpu->height=16; + REQUIRE(av_frame_get_buffer(cpu.get(),0)>=0); + for(int y=0;y<16;++y)for(int x=0;x<16;++x){ + auto* p=cpu->data[0]+y*cpu->linesize[0]+x*4; + p[0]=255;p[1]=0;p[2]=0;p[3]=255; + } + REQUIRE(av_hwframe_get_buffer(pool.get(),gpu.get(),0)>=0); + REQUIRE(av_hwframe_transfer_data(gpu.get(),cpu.get(),0)>=0); + auto source=std::make_shared(gpu.get()); + CHECK(source->GetBytes() >= 16 * 16 * 4); + QTransform transform;transform.translate(8,8); + auto first=openshot::RenderVulkanImage(source,nullptr,32,32,transform,0.5,Qt::blue); + // A second render must not overwrite the first cached frame's GPU storage. + auto second=openshot::RenderVulkanImage(source,nullptr,32,32,QTransform(),1.0,Qt::green); + gpu.reset();pool.reset();device.reset();source.reset(); + CHECK(first->GetBytes() >= 32 * 32 * 4); + QImage image=first->Materialize(); + // FFmpeg may import the upload memory asynchronously; retain it until completion. + cpu.reset(); + const auto blended=image.pixelColor(12,12); + CHECK(blended.red()>=126);CHECK(blended.red()<=129); + CHECK(blended.green()==0); + CHECK(blended.blue()>=126);CHECK(blended.blue()<=129); + CHECK(blended.alpha()==255); + CHECK(image.pixelColor(0,0)==QColor(Qt::blue)); + CHECK(second->Materialize().pixelColor(0,0)==QColor(Qt::red)); + // Export conversion remains on the GPU and produces limited-range NV12 (BT.601 at this SD size). + auto encoded = openshot::ConvertVulkanToNV12(second); + auto encoded_surface = std::dynamic_pointer_cast(encoded); + REQUIRE(encoded_surface); + // NVENC still receives separate Y/UV Vulkan images for CUDA interop. + const auto* planes=reinterpret_cast(encoded_surface->GetFrame()->data[0]); + REQUIRE(planes->img[0]!=VK_NULL_HANDLE); + CHECK(planes->img[1]!=VK_NULL_HANDLE); + CHECK(planes->img[0]!=planes->img[1]); + Frame downloaded(av_frame_alloc()); + REQUIRE(downloaded); +#ifdef __APPLE__ + REQUIRE(openshot::ReadMacVulkanFrame(downloaded.get(), encoded_surface->GetFrame()) >= 0); +#else + REQUIRE(av_hwframe_transfer_data(downloaded.get(), encoded_surface->GetFrame(), 0) >= 0); +#endif + REQUIRE(downloaded->format == AV_PIX_FMT_NV12); + const int y = downloaded->data[0][4 * downloaded->linesize[0] + 4]; + const int u = downloaded->data[1][2 * downloaded->linesize[1] + 4]; + const int v = downloaded->data[1][2 * downloaded->linesize[1] + 5]; + CHECK(y >= 80); CHECK(y <= 83); + CHECK(u >= 89); CHECK(u <= 92); + CHECK(v >= 238); CHECK(v <= 241); + const auto converted_red = encoded->Materialize().pixelColor(4, 4); + CHECK(converted_red.red() >= 252); + CHECK(converted_red.green() <= 3); + CHECK(converted_red.blue() <= 3); + // Exercise the actual Clip transform path against its QPainter fallback. + // Compare flat interior pixels; edge resampling intentionally differs. + for(auto mode : {openshot::SCALE_FIT,openshot::SCALE_CROP,openshot::SCALE_STRETCH,openshot::SCALE_NONE}) { + for(int gravity=openshot::GRAVITY_TOP_LEFT;gravity<=openshot::GRAVITY_BOTTOM_RIGHT;++gravity) { + PatternReader cpu_reader(second,false),gpu_reader(second,true); + openshot::Clip cpu_clip(&cpu_reader),gpu_clip(&gpu_reader); + for(auto* clip : {&cpu_clip,&gpu_clip}) { + clip->scale=mode; + clip->gravity=static_cast(gravity); + clip->scale_x=openshot::Keyframe(0.7);clip->scale_y=openshot::Keyframe(0.6); + clip->shear_x=openshot::Keyframe(0.12);clip->shear_y=openshot::Keyframe(-0.08); + clip->rotation=openshot::Keyframe(13.0);clip->alpha=openshot::Keyframe(0.6); + clip->Open(); + } + auto cpu_background=std::make_shared(1,80,60,"#00000000"); + auto gpu_background=std::make_shared(1,80,60,"#00000000"); + auto cpu_frame=cpu_clip.GetFrame(cpu_background,1); + auto gpu_frame=gpu_clip.GetFrame(gpu_background,1); + REQUIRE(gpu_frame->GetDeferredImage()); + auto expected=cpu_frame->GetImage();auto actual=gpu_frame->GetImage(); + REQUIRE(expected->size()==actual->size()); + int compared=0,max_error=0; + for(int y=2;yheight()-2;++y)for(int x=2;xwidth()-2;++x) { + const auto pixel=expected->pixel(x,y); + if(qAlpha(pixel)==0) continue; + bool interior=true; + for(int dy=-2;dy<=2;++dy)for(int dx=-2;dx<=2;++dx) + interior &= expected->pixel(x+dx,y+dy)==pixel; + if(!interior) continue; + const auto* a=actual->constScanLine(y)+4*x; + const auto* b=expected->constScanLine(y)+4*x; + for(int channel=0;channel<4;++channel)max_error=std::max(max_error,std::abs(int(a[channel])-int(b[channel]))); + ++compared; + } + INFO("scale=" << mode << " gravity=" << gravity << " compared=" << compared << " max_error=" << max_error); + CHECK(compared>8);CHECK(max_error<=3); + cpu_clip.Close();gpu_clip.Close(); + } + } + // Premultiplied output is not multiplied twice when materialized. + auto transparent=openshot::RenderVulkanImage(second,nullptr,32,32,QTransform(),0.5); + auto rgba=transparent->Materialize(); + const auto red=rgba.pixelColor(0,0); + CHECK(red.red()>=253); + CHECK(red.alpha()>=126);CHECK(red.alpha()<=129); +} + +TEST_CASE("Vulkan sequential decoding preserves pending audio video frames", "[vulkan][hardware][reader]") { + if(!VulkanAvailable()) return; + // OPENSHOT_VULKAN_TEST_VIDEO may select a larger (e.g. 4K) audio/video clip + // of at least 120 frames; the bundled trailer covers the same cache paths. + std::string video=TEST_MEDIA_PATH "sintel_trailer-720p.mp4"; + if(const char* custom=std::getenv("OPENSHOT_VULKAN_TEST_VIDEO"); custom && *custom) video=custom; + // Full-resolution RGBA frames exceed a small working-cache byte cap before + // their audio arrives. Evicting those pending frames silently produced black + // images and incorrect audio; compare sequential reads against direct seeks. + std::map> reference; + { + openshot::FFmpegReader reader(video); + reader.Open(); + for(int n : {12,120}) { + auto frame=reader.GetFrame(n); + REQUIRE(frame->GetDeferredImage()); + reference[n]=frame->GetImage(); + } + reader.Close(); + } + openshot::FFmpegReader reader(video); + reader.Open(); + for(int n=1;n<=120;++n) { + auto frame=reader.GetFrame(n); + REQUIRE(frame->GetDeferredImage()); + if(reference.count(n)) { + auto actual=frame->GetImage(); + const auto expected=reference[n]; + REQUIRE(actual->size()==expected->size()); + REQUIRE(actual->format()==expected->format()); + uint64_t error=0; + for(int y=0;yheight();++y) { + const auto* a=actual->constScanLine(y); + const auto* b=expected->constScanLine(y); + for(int x=0;xwidth()*4;++x) error+=std::abs(int(a[x])-int(b[x])); + } + const double mae=double(error)/(actual->width()*actual->height()*4.0); + INFO("frame=" << n << " sequential/direct MAE=" << mae); + CHECK(mae<=3.0); + } + } + reader.Close(); +} + +TEST_CASE("Vulkan high quality reader resizing uses bicubic antialiasing", "[vulkan][hardware][quality]") { + if(!VulkanAvailable()) return; + AVBufferRef* raw_device=nullptr; + REQUIRE(openshot::CreateSharedVulkanDevice(&raw_device)>=0); + Buffer device(raw_device),pool(av_hwframe_ctx_alloc(raw_device)); + REQUIRE(pool); + auto* context=reinterpret_cast(pool->data); + context->format=AV_PIX_FMT_VULKAN;context->sw_format=AV_PIX_FMT_RGBA; + context->width=context->height=32; + REQUIRE(av_hwframe_ctx_init(pool.get())>=0); + Frame cpu(av_frame_alloc()),gpu(av_frame_alloc()); + REQUIRE(cpu);REQUIRE(gpu); + cpu->format=AV_PIX_FMT_RGBA;cpu->width=cpu->height=32; + REQUIRE(av_frame_get_buffer(cpu.get(),0)>=0); + for(int y=0;y<32;++y)for(int x=0;x<32;++x){ + auto* pixel=cpu->data[0]+y*cpu->linesize[0]+x*4; + pixel[0]=32+192*((x/4+y/4)%2);pixel[1]=y*8;pixel[2]=x*8;pixel[3]=255; + } + REQUIRE(av_hwframe_get_buffer(pool.get(),gpu.get(),0)>=0); + REQUIRE(av_hwframe_transfer_data(gpu.get(),cpu.get(),0)>=0); + openshot::VulkanFrameProcessor processor; + for(const auto size : {QSize(12,10),QSize(61,47),QSize(32,32)}) { + QImage reference(size,QImage::Format_RGBA8888); + auto* scaler=sws_getContext(32,32,AV_PIX_FMT_RGBA,size.width(),size.height(),AV_PIX_FMT_RGBA, + SWS_BICUBIC,nullptr,nullptr,nullptr); + REQUIRE(scaler); + std::unique_ptr owner(scaler,sws_freeContext); + uint8_t* output[]={reference.bits(),nullptr,nullptr,nullptr}; + int strides[]={reference.bytesPerLine(),0,0,0}; + REQUIRE(sws_scale(scaler,cpu->data,cpu->linesize,0,32,output,strides)==size.height()); + auto high=processor.Process(gpu.get(),size.width(),size.height(),openshot::color::SourceColor(),true); + REQUIRE(std::dynamic_pointer_cast(high)); + auto high_image=high->Materialize(); + auto low_image=processor.Process(gpu.get(),size.width(),size.height(),openshot::color::SourceColor(),false)->Materialize(); + double high_error=0,low_error=0; + for(int y=0;y expected_frame; + const auto readbacks=openshot::VulkanFrameReadbackCount(); + for(int n=1;n<=24;++n){ + auto frame=timeline.GetFrame(n); + REQUIRE(frame->GetDeferredImage()); + writer.WriteFrame(frame); + REQUIRE(frame->GetDeferredImage()); + if (n == 12) expected_frame = frame; + } + writer.Close(); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + CHECK(base_reader.HardwareDecodeSuccessful()); + CHECK(overlay_reader.HardwareDecodeSuccessful()); + timeline.Close(); + openshot::FFmpegReader verify(path); + verify.Open(); + CHECK(verify.info.width==width); CHECK(verify.info.height==height); + CHECK(verify.info.video_length==24); + auto image=verify.GetFrame(12)->GetImage(); + REQUIRE(image); CHECK(image->width()==width); CHECK(image->height()==height); + REQUIRE(expected_frame); + const auto expected = expected_frame->GetImage(); + REQUIRE(expected); + double error = 0, brightness = 0; + int samples = 0; + for (int y = 8; y < height; y += 16) { + for (int x = 8; x < width; x += 16) { + const auto a = expected->pixelColor(x, y), b = image->pixelColor(x, y); + error += std::abs(a.red()-b.red()) + std::abs(a.green()-b.green()) + std::abs(a.blue()-b.blue()); + brightness += a.red() + a.green() + a.blue(); + samples += 3; + } + } + CHECK(brightness / samples > 10); + CHECK(error / samples < 20); + verify.Close(); + } +} +#endif + +TEST_CASE("CPU image uploads cache pixels and preserve alpha ownership", "[vulkan][hardware][upload]") { + if (!VulkanAvailable()) return; + openshot::ReleaseVulkanUploads(); + QImage image(16, 16, QImage::Format_ARGB32_Premultiplied); + image.fill(QColor(200, 60, 20, 128)); + openshot::ResetGpuMetrics(); + std::shared_ptr first, second; + { + openshot::gpu_metrics::Scope scope(false); + first = openshot::UploadVulkanImage(image); +#ifdef __APPLE__ + auto mac_surface = std::dynamic_pointer_cast(first); + REQUIRE(mac_surface); + CHECK(openshot::IsMacVulkanFrame(mac_surface->GetFrame())); +#endif + CHECK(openshot::UploadVulkanImage(image) == first); + image.fill(QColor(0, 255, 0, 255)); + second = openshot::UploadVulkanImage(image); + CHECK(second != first); + } + Json::Reader reader; + Json::Value metrics; + REQUIRE(reader.parse(openshot::GpuMetricsSnapshot(), metrics)); + CHECK(metrics["uploads"].asUInt64() == 2); + QTransform identity; + auto rendered = openshot::RenderVulkanImage(first, nullptr, 16, 16, identity, 1.0, Qt::transparent); + auto pixel = rendered->Materialize().pixelColor(8,8); + CHECK(std::abs(pixel.red() - 200) <= 2); + CHECK(std::abs(pixel.green() - 60) <= 2); + CHECK(std::abs(pixel.alpha() - 128) <= 1); + openshot::ReleaseVulkanUploads(); + const auto second_pixel = second->Materialize().pixelColor(8,8); + INFO("Uploaded opaque RGBA: " << second_pixel.red() << "," << second_pixel.green() + << "," << second_pixel.blue() << "," << second_pixel.alpha()); + CHECK(second_pixel == QColor(Qt::green)); + CHECK(openshot::UploadVulkanImage(image) != second); +} + +TEST_CASE("PNG clips upload once and keep animated transforms on Vulkan", "[vulkan][hardware][upload]") { + if (!VulkanAvailable()) return; + QTemporaryDir temporary; + REQUIRE(temporary.isValid()); + const auto path = temporary.filePath("artwork.png"); + QImage bitmap(64, 64, QImage::Format_ARGB32_Premultiplied); + bitmap.fill(QColor(0, 0, 255, 128)); + REQUIRE(bitmap.save(path)); + openshot::Clip clip(path.toStdString()); + clip.scale_x.AddPoint(1,0.6,openshot::LINEAR); + clip.scale_x.AddPoint(8,0.8,openshot::LINEAR); + clip.scale_y = openshot::Keyframe(0.6); + clip.rotation.AddPoint(1,10,openshot::LINEAR); + clip.rotation.AddPoint(8,20,openshot::LINEAR); + openshot::Saturation saturation; + saturation.saturation.AddPoint(1,1.0,openshot::LINEAR); + saturation.saturation.AddPoint(8,1.5,openshot::LINEAR); + openshot::Crop crop; + crop.left.AddPoint(1,0.0,openshot::LINEAR); + crop.left.AddPoint(8,0.125,openshot::LINEAR); + clip.AddEffect(&saturation); + clip.AddEffect(&crop); + openshot::Timeline timeline(128,128,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.AddClip(&clip); + timeline.Open(); + openshot::ResetGpuMetrics(); + std::shared_ptr result; + for (int i = 1; i <= 8; ++i) { + result = timeline.GetFrame(i); + REQUIRE(result->GetDeferredImage()); + } + Json::Reader reader; + Json::Value metrics; + REQUIRE(reader.parse(openshot::GpuMetricsSnapshot(), metrics)); + CHECK(metrics["uploads"].asUInt64() == 1); + CHECK(metrics["readbacks"].asUInt64() == 0); + CHECK(metrics["render_cpu"].asUInt64() == 0); + CHECK(metrics["render_gpu"].asUInt64() > 0); + CHECK(metrics["image_sources"].asUInt64() == 8); + CHECK(metrics["decode_cpu"].asUInt64() == 0); + CHECK(metrics["copy_free"].asUInt64() == 8); + openshot::Frame copy(*result); + const auto pixel = copy.GetImage()->pixelColor(64,64); + CHECK(std::abs(pixel.blue() - 128) <= 2); + CHECK(pixel.red() == 0); + CHECK(pixel.alpha() == 255); + timeline.Close(); +} + +TEST_CASE("FFmpeg still images and cover art upload once", "[vulkan][hardware][upload]") { + if (!VulkanAvailable()) return; + QTemporaryDir temporary; + REQUIRE(temporary.isValid()); + const auto path = temporary.filePath("cover.jpg"); + QImage bitmap(64, 64, QImage::Format_RGB32); + bitmap.fill(QColor(255, 0, 0)); + REQUIRE(bitmap.save(path)); + // FFmpegReader fills each frame of a still image from the previous one, as + // it does for an audio file's cover art; that must not upload every frame. + openshot::FFmpegReader reader(path.toStdString()); + REQUIRE(reader.info.has_single_image); + openshot::Clip clip(&reader); + openshot::Timeline timeline(128,128,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.AddClip(&clip); + timeline.Open(); + openshot::ResetGpuMetrics(); + for (int i = 1; i <= 24; ++i) + REQUIRE(timeline.GetFrame(i)->GetDeferredImage()); + Json::Reader parser; + Json::Value metrics; + REQUIRE(parser.parse(openshot::GpuMetricsSnapshot(), metrics)); + CHECK(metrics["uploads"].asUInt64() <= 1); + CHECK(metrics["upload_failures"].asUInt64() == 0); + timeline.Close(); +} + +TEST_CASE("Saved GPU preferences reach the native runtime environment", "[vulkan][settings]") { + struct Restore { + std::string backend, device; + ~Restore() { openshot::ConfigureGpuEnvironment(backend, device); } + } restore{std::getenv("OPENSHOT_RENDER_BACKEND") ? std::getenv("OPENSHOT_RENDER_BACKEND") : "", + std::getenv("OPENSHOT_VULKAN_DEVICE") ? std::getenv("OPENSHOT_VULKAN_DEVICE") : ""}; + openshot::ConfigureGpuEnvironment("cpu", "Example GPU"); + REQUIRE(std::getenv("OPENSHOT_RENDER_BACKEND")); + REQUIRE(std::getenv("OPENSHOT_VULKAN_DEVICE")); + CHECK(std::string(std::getenv("OPENSHOT_RENDER_BACKEND")) == "cpu"); + CHECK(std::string(std::getenv("OPENSHOT_VULKAN_DEVICE")) == "Example GPU"); + CHECK_FALSE(openshot::VulkanRenderingEnabled()); + openshot::ConfigureGpuEnvironment("vulkan", ""); + CHECK(std::string(std::getenv("OPENSHOT_RENDER_BACKEND")) == "vulkan"); + CHECK(std::getenv("OPENSHOT_VULKAN_DEVICE") == nullptr); +} + +namespace { +void ProcessCpuEffect(openshot::EffectBase& effect, std::shared_ptr frame, int number) { + struct Restore { + std::string backend, device; + ~Restore() { openshot::ConfigureGpuEnvironment(backend,device); } + } restore{std::getenv("OPENSHOT_RENDER_BACKEND") ? std::getenv("OPENSHOT_RENDER_BACKEND") : "", + std::getenv("OPENSHOT_VULKAN_DEVICE") ? std::getenv("OPENSHOT_VULKAN_DEVICE") : ""}; + openshot::ConfigureGpuEnvironment("cpu",restore.device); + effect.ProcessFrame(frame,number); +} +void CheckVulkanEffect(openshot::EffectBase& effect, const QImage& pixels, + std::shared_ptr source, int number, int tolerance) { + auto cpu=std::make_shared(number,pixels.width(),pixels.height(),"transparent"); + auto gpu=std::make_shared(number,pixels.width(),pixels.height(),"transparent"); + cpu->AddImage(std::make_shared(pixels.copy())); + gpu->AddDeferredImage(source); + ProcessCpuEffect(effect,cpu,number); + const auto readbacks=openshot::VulkanFrameReadbackCount(); + effect.ProcessFrame(gpu,number); + REQUIRE(gpu->GetDeferredImage()); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + auto expected=cpu->GetImage(),actual=gpu->GetImage(); + REQUIRE(actual->size()==expected->size()); + int max_error=0; + for (int y=0;yheight();++y) + for (int x=0;xwidth()*4;++x) + max_error=std::max(max_error,std::abs(int(expected->constScanLine(y)[x])-int(actual->constScanLine(y)[x]))); + INFO("frame=" << number << " max byte error=" << max_error); + CHECK(max_error<=tolerance); +} + +} + +TEST_CASE("Vulkan saturation matches CPU for animated channels and alpha", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QImage pixels(32, 16, QImage::Format_RGBA8888_Premultiplied); + for (int y=0; y> frames; + for(int width=16;width<48;++width) { + frames.push_back(openshot::RenderVulkanImage(source,nullptr,width,32,QTransform(),1.0)); + REQUIRE(frames.back()->GetWidth()==width); + } + CHECK(openshot::VulkanPipelineCreationCount()-created<=1); + // Old frames remain valid after both submission slots and retained pools turn over. + CHECK(frames.front()->Materialize().pixelColor(8,16)==QColor(Qt::red)); + CHECK(frames.back()->Materialize().pixelColor(8,16)==QColor(Qt::red)); +} + +TEST_CASE("Queued grading frames retain their own wheel parameters", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QImage pixels(16,16,QImage::Format_RGBA8888_Premultiplied); + pixels.fill(QColor(100,80,60)); + auto source=openshot::UploadVulkanImage(pixels); + openshot::VulkanGradeParameters parameters{}; + parameters.controls={0,0,1,1,0,0,1,0,1}; + for(auto& curve:parameters.curves) + for(int i=0;i<256;++i) curve[i]=float(i)/255.0f; + std::vector> frames; + for(int i=0;i<12;++i) { + parameters.wheels[0]=float(i)/32.0f; + frames.push_back(openshot::GradeVulkanImage(source,parameters)); + } + for(int i=0;i<12;++i) { + const auto color=frames[i]->Materialize().pixelColor(8,8); + CHECK(std::abs(color.red()-int(100+255.0*i/32.0+0.5))<=1); + CHECK(color.green()==80); + CHECK(color.blue()==60); + } +} + +TEST_CASE("Scope analysis preserves the cached GPU preview surface", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QImage pixels(16,16,QImage::Format_RGBA8888_Premultiplied); + pixels.fill(Qt::red); + auto source=openshot::UploadVulkanImage(pixels); + auto frame=std::make_shared(1,16,16,"transparent"); + frame->AddDeferredImage(source); + openshot::FrameScope scope; + scope.SetFrame(frame); + CHECK(scope.HasVideo()); + CHECK(frame->GetDeferredImage()==source); +} + +namespace { +QImage EffectPixels(int width=32,int height=16) { + QImage pixels(width,height,QImage::Format_RGBA8888_Premultiplied); + for (int y=0; y> effects{{&crop,blend_tolerance},{&grade,1},{&saturation,2},{&lut,1}, + {&driven_saturation,2},{&driven_blur,1}}; + for (const auto& [effect,tolerance] : effects) { + effect->MaskReader(new openshot::QtImageReader(path.toStdString())); + for (bool invert : {false,true}) { + effect->mask_invert=invert; + INFO(effect->info.class_name << " invert=" << invert); + CheckVulkanEffect(*effect,pixels,source,1,tolerance); + } + } +} + +TEST_CASE("Vulkan scopes count on the GPU like the CPU", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QImage pixels=EffectPixels(48,24); + pixels.setPixelColor(1,1,QColor(255,255,255)); + pixels.setPixelColor(2,1,QColor(0,0,0)); + pixels.setPixelColor(5,1,QColor(255,10,250)); + auto source=openshot::UploadVulkanImage(pixels); + for (bool region : {false,true}) { + for (int columns : {20,64}) { + openshot::FrameScope cpu,gpu; + for (auto* scope : {&cpu,&gpu}) { + scope->SetWaveformColumns(columns); + scope->SetVectorscopeSize(64); + if (region) scope->SetVideoRegionNormalized(0.2f,0.1f,0.5f,0.7f); + } + auto cpu_frame=std::make_shared(1,pixels.width(),pixels.height(),"transparent"); + cpu_frame->AddImage(std::make_shared(pixels.copy())); + auto gpu_frame=std::make_shared(1,pixels.width(),pixels.height(),"transparent"); + gpu_frame->AddDeferredImage(source); + cpu.SetFrame(cpu_frame); + const auto readbacks=openshot::VulkanFrameReadbackCount(); + gpu.SetFrame(gpu_frame); + INFO("region=" << region << " columns=" << columns); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + CHECK(gpu_frame->GetDeferredImage()==source); + REQUIRE(gpu.HasVideo()); + CHECK(gpu.GetVideoHistogramLumaBins()==cpu.GetVideoHistogramLumaBins()); + CHECK(gpu.GetVideoHistogramRedBins()==cpu.GetVideoHistogramRedBins()); + CHECK(gpu.GetVideoHistogramGreenBins()==cpu.GetVideoHistogramGreenBins()); + CHECK(gpu.GetVideoHistogramBlueBins()==cpu.GetVideoHistogramBlueBins()); + CHECK(gpu.GetVideoWaveformLumaBins()==cpu.GetVideoWaveformLumaBins()); + CHECK(gpu.GetVideoWaveformRedBins()==cpu.GetVideoWaveformRedBins()); + CHECK(gpu.GetVideoWaveformGreenBins()==cpu.GetVideoWaveformGreenBins()); + CHECK(gpu.GetVideoWaveformBlueBins()==cpu.GetVideoWaveformBlueBins()); + CHECK(gpu.GetVideoVectorscopeBins()==cpu.GetVideoVectorscopeBins()); + CHECK(gpu.GetVideoClippedShadows()==cpu.GetVideoClippedShadows()); + CHECK(gpu.GetVideoClippedHighlights()==cpu.GetVideoClippedHighlights()); + CHECK(gpu.GetVideoClippedRed()==cpu.GetVideoClippedRed()); + CHECK(gpu.GetVideoClippedGreen()==cpu.GetVideoClippedGreen()); + CHECK(gpu.GetVideoClippedBlue()==cpu.GetVideoClippedBlue()); + CHECK(std::abs(gpu.GetVideoAverageLuma()-cpu.GetVideoAverageLuma())<=0.5/255.0+1e-6); + } + } +} + +TEST_CASE("Masked clip grading with a LUT and scopes stays on Vulkan", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QTemporaryDir directory; + const auto artwork=directory.filePath("artwork.png"),mask=directory.filePath("mask.png"); + REQUIRE(EffectPixels(64,64).save(artwork)); + QImage gradient(16,16,QImage::Format_RGBA8888); + for (int y=0; y<16; ++y) + for (int x=0; x<16; ++x) + gradient.setPixelColor(x,y,QColor(x*17,x*17,x*17)); + REQUIRE(gradient.save(mask)); + openshot::Clip clip(artwork.toStdString()); + openshot::ColorGrade grade; + Json::Value settings; + settings["lut_path"]=WriteCube(directory,true,5); + settings["lut_intensity"]=openshot::Keyframe(0.6).JsonValue(); + settings["exposure"]=openshot::Keyframe(0.4).JsonValue(); + grade.SetJsonValue(settings); + grade.MaskReader(new openshot::QtImageReader(mask.toStdString())); + clip.AddEffect(&grade); + openshot::Timeline timeline(128,128,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.AddClip(&clip); + timeline.Open(); + openshot::ResetGpuMetrics(); + for (int i=1; i<=4; ++i) { + auto frame=timeline.GetFrame(i); + REQUIRE(frame->GetDeferredImage()); + openshot::FrameScope scope; + scope.SetVideoRegionNormalized(0.25f,0.25f,0.5f,0.5f); + scope.SetFrame(frame); + REQUIRE(scope.HasVideo()); + const auto bins=scope.GetVideoHistogramLumaBins(); + const uint64_t counted=std::accumulate(bins.begin(),bins.end(),uint64_t{0}); + CHECK(counted>0); + } + // A neutral grade applies only its LUT, still on the GPU. + grade.exposure=openshot::Keyframe(0.0); + for (int i=5; i<=6; ++i) { + auto frame=timeline.GetFrame(i); + REQUIRE(frame->GetDeferredImage()); + } + Json::Reader reader; + Json::Value metrics; + REQUIRE(reader.parse(openshot::GpuMetricsSnapshot(), metrics)); + // The clip and the still mask each upload once; nothing returns to the CPU. + CHECK(metrics["uploads"].asUInt64()==2); + CHECK(metrics["readbacks"].asUInt64()==0); + CHECK(metrics["render_cpu"].asUInt64()==0); + clip.RemoveEffect(&grade); + timeline.Close(); +} + +TEST_CASE("Video masks stay on Vulkan for Mask and effect masks", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + // Scale a 720p decoded mask down to the frame on the GPU, for the Mask + // effect's wipe and for the common effect mask. + const QImage pixels=EffectPixels(320,180); + auto source=openshot::UploadVulkanImage(pixels); + openshot::Mask wipe; + wipe.MaskReader(new openshot::FFmpegReader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4")); + openshot::ColorGrade grade; + grade.exposure=openshot::Keyframe(1.0); + grade.MaskReader(new openshot::FFmpegReader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4")); + // These frames have detailed artwork; the trailer opens on black. + const std::vector numbers{300,700}; + { + openshot::FFmpegReader probe(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + probe.Open(); + for (int number : numbers) { + auto frame=probe.GetFrame(number); + REQUIRE(frame->GetDeferredImage()); + const auto image=frame->GetImage(); + int lowest=255,highest=0; + for (int y=0; yheight(); y+=8) + for (int x=0; xwidth(); x+=8) { + const int gray=qGray(image->pixel(x,y)); + lowest=std::min(lowest,gray); highest=std::max(highest,gray); + } + REQUIRE(highest-lowest>100); + } + } + for (auto* effect : std::initializer_list{&wipe,&grade}) { + for (int number : numbers) { + // GPU first: the CPU run reads the reader's cached mask frames back. + auto gpu=std::make_shared(number,pixels.width(),pixels.height(),"transparent"); + gpu->AddDeferredImage(source); + const auto readbacks=openshot::VulkanFrameReadbackCount(); + effect->ProcessFrame(gpu,number); + INFO(effect->info.class_name << " frame=" << number); + REQUIRE(gpu->GetDeferredImage()); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + auto cpu=std::make_shared(number,pixels.width(),pixels.height(),"transparent"); + cpu->AddImage(std::make_shared(pixels.copy())); + ProcessCpuEffect(*effect,cpu,number); + // The GPU scales the mask with its own filter, so compare on average. + auto expected=cpu->GetImage(),actual=gpu->GetImage(); + double total=0.0; + for (int y=0; yheight(); ++y) + for (int x=0; xwidth()*4; ++x) + total+=std::abs(int(expected->constScanLine(y)[x])-int(actual->constScanLine(y)[x])); + const double mean=total/(actual->width()*actual->height()*4.0); + double changed=0.0; + for (int y=0; yheight(); ++y) + for (int x=0; xwidth()*4; ++x) + changed+=std::abs(int(pixels.constScanLine(y)[x])-int(actual->constScanLine(y)[x])); + changed/=actual->width()*actual->height()*4.0; + INFO("mean byte error=" << mean << " mean change=" << changed); + CHECK(changed>0.5); + CHECK(mean<=2.0); + } + } +} + +TEST_CASE("Vulkan decoding converts SDR, HLG and PQ charts like the CPU path", "[vulkan][hardware][color]") { + if(!VulkanAvailable()) return; + struct DecoderSetting { + int previous=openshot::Settings::Instance()->HARDWARE_DECODER; + ~DecoderSetting(){openshot::Settings::Instance()->HARDWARE_DECODER=previous;} + } restore; + // Mean of a flat patch interior; see tests/generate-color-charts.py. + auto patch=[](std::shared_ptr frame,int index){ + const int x0=(index%4)*160+70,y0=(index/4)*96+42; + std::array sum{}; + for(int y=y0;yGetPixels(y)[x*4+c]; + return std::array{int(sum[0]/240),int(sum[1]/240),int(sum[2]/240)}; + }; + for(const char* chart : {"color-chart-sdr-bt709.mp4","color-chart-hlg.mp4","color-chart-pq.mp4"}) { + const std::string path=std::string(TEST_MEDIA_PATH)+chart; + // The CPU pipeline is selected per reader when it opens. + qputenv("OPENSHOT_RENDER_BACKEND","cpu"); + openshot::FFmpegReader cpu_reader(path); + cpu_reader.Open(); + qunsetenv("OPENSHOT_RENDER_BACKEND"); + auto cpu=cpu_reader.GetFrame(2); + REQUIRE_FALSE(cpu->GetDeferredImage()); + openshot::Settings::Instance()->HARDWARE_DECODER=1; + openshot::FFmpegReader gpu_reader(path); + gpu_reader.Open(); + auto gpu=gpu_reader.GetFrame(2); + // 10-bit P010 surfaces stay on the GPU instead of falling back to software decode. +#ifdef __APPLE__ + // VideoToolbox decodes H.264 and HEVC Main10 on every supported Mac. Intel + // Macs refuse very small H.264 frames (320x192), so the charts are 640x384. + REQUIRE(gpu->GetDeferredImage()); +#else + if(!gpu->GetDeferredImage()) { + WARN("Skipping Vulkan test: this device does not decode " << chart << " on the GPU"); + continue; + } +#endif + for(int i=0;i<13;++i) { + const auto expected=patch(cpu,i),actual=patch(gpu,i); + INFO(chart << " patch " << i << " cpu (" << expected[0] << "," << expected[1] << "," << expected[2] + << ") gpu (" << actual[0] << "," << actual[1] << "," << actual[2] << ")"); + for(int c=0;c<3;++c) CHECK(std::abs(actual[c]-expected[c])<=3); + } + } +} + +TEST_CASE("Vulkan export encodes and tags HD with BT.709", "[vulkan][hardware][color]") { + if(!VulkanAvailable()) return; + QImage red(1280,720,QImage::Format_RGBA8888_Premultiplied); + red.fill(QColor(255,0,0)); + auto encoded=std::dynamic_pointer_cast( + openshot::ConvertVulkanToNV12(openshot::UploadVulkanImage(red))); + REQUIRE(encoded); + const AVFrame* nv12=encoded->GetFrame(); + CHECK(nv12->colorspace==AVCOL_SPC_BT709); + CHECK(nv12->color_primaries==AVCOL_PRI_BT709); + CHECK(nv12->color_trc==AVCOL_TRC_BT709); + CHECK(nv12->color_range==AVCOL_RANGE_MPEG); + Frame downloaded(av_frame_alloc()); + REQUIRE(downloaded); +#ifdef __APPLE__ + REQUIRE(openshot::ReadMacVulkanFrame(downloaded.get(), nv12) >= 0); +#else + REQUIRE(av_hwframe_transfer_data(downloaded.get(), nv12, 0) >= 0); +#endif + // BT.709 limited range red: Y 63, Cb 102, Cr 240 (BT.601 would be 81, 90, 240). + const int y=downloaded->data[0][100*downloaded->linesize[0]+100]; + const int u=downloaded->data[1][50*downloaded->linesize[1]+100]; + const int v=downloaded->data[1][50*downloaded->linesize[1]+101]; + CHECK(y>=62); CHECK(y<=64); + CHECK(u>=101); CHECK(u<=103); + CHECK(v>=239); CHECK(v<=241); +} + +#ifdef __APPLE__ +namespace { +// A Core Video 4:2:0 biplanar buffer filled with one Y'CbCr value per half +// (left, right), as VideoToolbox returns decoded frames. 10-bit formats keep +// samples in the high bits of 16-bit words, like P010. +struct PixelBuffer { + CVPixelBufferRef pixel=nullptr; + PixelBuffer(OSType type,int width,int height,std::array left,std::array right,int bits) { + const void* keys[]={kCVPixelBufferIOSurfacePropertiesKey,kCVPixelBufferMetalCompatibilityKey}; + CFDictionaryRef empty=CFDictionaryCreate(kCFAllocatorDefault,nullptr,nullptr,0, + &kCFTypeDictionaryKeyCallBacks,&kCFTypeDictionaryValueCallBacks); + const void* values[]={empty,kCFBooleanTrue}; + CFDictionaryRef attributes=CFDictionaryCreate(kCFAllocatorDefault,keys,values,2, + &kCFTypeDictionaryKeyCallBacks,&kCFTypeDictionaryValueCallBacks); + const CVReturn created=CVPixelBufferCreate(kCFAllocatorDefault,width,height,type,attributes,&pixel); + CFRelease(attributes);CFRelease(empty); + REQUIRE(created==kCVReturnSuccess); + REQUIRE(CVPixelBufferLockBaseAddress(pixel,0)==kCVReturnSuccess); + const int shift=bits>8?16-bits:0; + auto store=[&](uint8_t* row,int x,int value){ + if(bits>8) reinterpret_cast(row)[x]=uint16_t(value<(CVPixelBufferGetBaseAddressOfPlane(pixel,plane)); + const size_t stride=CVPixelBufferGetBytesPerRowOfPlane(pixel,plane); + const int plane_width=int(CVPixelBufferGetWidthOfPlane(pixel,plane)); + const int plane_height=int(CVPixelBufferGetHeightOfPlane(pixel,plane)); + for(int y=0;yformat=AV_PIX_FMT_VIDEOTOOLBOX; + frame->width=int(CVPixelBufferGetWidth(pixel));frame->height=int(CVPixelBufferGetHeight(pixel)); + frame->data[3]=reinterpret_cast(pixel); + frame->colorspace=matrix;frame->color_trc=trc;frame->color_primaries=primaries; + frame->color_range=AVCOL_RANGE_MPEG; + return frame; + } +}; +} + +TEST_CASE("Mac VideoToolbox surfaces map to shareable FFmpeg layouts", "[vulkan][hardware][mac][color]") { + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange)==AV_PIX_FMT_NV12); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)==AV_PIX_FMT_NV12); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange)==AV_PIX_FMT_P010); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange)==AV_PIX_FMT_P010); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange)==AV_PIX_FMT_P210); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_32BGRA)==AV_PIX_FMT_NONE); +} + +TEST_CASE("Mac 10-bit VideoToolbox surfaces convert on the GPU", "[vulkan][hardware][mac][color]") { + if(!VulkanAvailable()) return; + // Left half: BT.709 limited-range red (Y'CbCr 250, 409, 960 at 10 bits). + // Right half: 75% grey, the HLG reference white (Y' 721, neutral chroma). + const std::array red{250,409,960},grey{721,512,512}; + for(const OSType type : {kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange,kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange}) { + INFO("Core Video format " << type); + PixelBuffer buffer(type,64,32,red,grey,10); + openshot::VulkanFrameProcessor processor; + + // SDR BT.709: exercises 16-bit plane mapping and 10-bit scaling. + auto sdr=buffer.Wrap(AVCOL_SPC_BT709,AVCOL_TRC_BT709,AVCOL_PRI_BT709); + const auto sdr_color=openshot::color::DescribeFrame(sdr.get(),64,32); + auto surface=processor.Process(sdr.get(),64,32,sdr_color); + REQUIRE(std::dynamic_pointer_cast(surface)); + QImage image=surface->Materialize(); + const QColor left=image.pixelColor(8,16),right=image.pixelColor(56,16); + INFO("SDR left " << left.name().toStdString() << " right " << right.name().toStdString()); + CHECK(left.red()>=252);CHECK(left.green()<=3);CHECK(left.blue()<=3); + // 75% grey without HDR conversion: (721-64)/876 = 0.75 -> 191. + CHECK(std::abs(right.red()-191)<=2);CHECK(std::abs(right.green()-191)<=2);CHECK(std::abs(right.blue()-191)<=2); + + // HLG BT.2020: reference white maps to the tone curve's SDR white. + auto hlg=buffer.Wrap(AVCOL_SPC_BT2020_NCL,AVCOL_TRC_ARIB_STD_B67,AVCOL_PRI_BT2020); + const auto hlg_color=openshot::color::DescribeFrame(hlg.get(),64,32); + REQUIRE(hlg_color.NeedsConversion()); + image=processor.Process(hlg.get(),64,32,hlg_color)->Materialize(); + const int white=int(std::lround(255.0*std::pow(openshot::color::ToneMap(1.0f,hlg_color.RelativePeak()),1.0/2.4))); + const QColor hlg_grey=image.pixelColor(56,16); + INFO("HLG grey " << hlg_grey.name().toStdString() << " expected " << white); + CHECK(std::abs(hlg_grey.red()-white)<=3);CHECK(std::abs(hlg_grey.green()-white)<=3);CHECK(std::abs(hlg_grey.blue()-white)<=3); + + // CPU readback of the shared surface keeps the 10-bit samples. + Frame imported(openshot::ImportVideoToolboxFrame(sdr.get())); + REQUIRE(imported); + Frame readback(av_frame_alloc()); + REQUIRE(readback); + REQUIRE(openshot::ReadMacVulkanFrame(readback.get(),imported.get())>=0); + CHECK(readback->format==(type==kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange?AV_PIX_FMT_P010:AV_PIX_FMT_P210)); + CHECK(reinterpret_cast(readback->data[0]+16*readback->linesize[0])[8]==uint16_t(250<<6)); + CHECK(reinterpret_cast(readback->data[0]+16*readback->linesize[0])[56]==uint16_t(721<<6)); + } +} +#endif diff --git a/tests/VulkanDeviceLost.cpp b/tests/VulkanDeviceLost.cpp new file mode 100644 index 000000000..81b37a9d7 --- /dev/null +++ b/tests/VulkanDeviceLost.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +// A lost device is permanent for the process, so this test has its own binary. +#include "openshot_catch.h" +#include "FFmpegReader.h" +#include "GpuDevices.h" +#include "VulkanFrameImage.h" +extern "C" { +#include +#include +} + +TEST_CASE("A lost GPU device moves open readers and new work to the CPU", "[vulkan][recovery]") { + if (!openshot::VulkanRenderingEnabled()) { + WARN("Skipping Vulkan test: CPU mode, or no Vulkan library"); + return; + } + AVBufferRef* device = nullptr; + if (openshot::CreateSharedVulkanDevice(&device) < 0) { + WARN("Skipping Vulkan test: no Vulkan device is available"); + return; + } + av_buffer_unref(&device); + CHECK(openshot::VulkanResultName(VK_ERROR_DEVICE_LOST) == "VK_ERROR_DEVICE_LOST"); + openshot::FFmpegReader reader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + reader.Open(); + const bool gpu_decoded = reader.GetFrame(1)->GetDeferredImage() != nullptr; + + // Other errors (such as low GPU memory) are recoverable. + openshot::ReportVulkanResult(VK_ERROR_OUT_OF_DEVICE_MEMORY); + CHECK_FALSE(openshot::VulkanDeviceLost()); + CHECK(openshot::VulkanRenderingEnabled()); + + openshot::ReportVulkanResult(VK_ERROR_DEVICE_LOST); + CHECK(openshot::VulkanDeviceLost()); + CHECK_FALSE(openshot::VulkanRenderingEnabled()); + // The open reader reopens with CPU decoding instead of using the lost device. + auto frame = reader.GetFrame(48); + REQUIRE(frame); + CHECK_FALSE(frame->GetDeferredImage()); + CHECK(frame->GetImage()->width() == 1280); + if (!gpu_decoded) WARN("GPU decode was not active before the loss on this system"); + reader.Close(); +} diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp new file mode 100644 index 000000000..359ca948e --- /dev/null +++ b/tests/VulkanPresentation.cpp @@ -0,0 +1,241 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "openshot_catch.h" +#include "Frame.h" +#include "VulkanFrameImage.h" +#include "VulkanCompositor.h" +#include "GpuDevices.h" +#include "Qt/VideoRenderer.h" +#include "Qt/VulkanOpenGLPresenter.h" +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +} +namespace { +class Preview : public QOpenGLWidget { +public: + openshot::VulkanOpenGLPresenter presenter{reinterpret_cast(this)}; + bool nativeDraw = false; + bool textureStatePreserved = false; + void initializeGL() override { + auto* current = context(); + typedef const GLubyte* (APIENTRYP GetString)(GLenum); + auto getString = reinterpret_cast(current->getProcAddress("glGetString")); + qWarning() << "Presentation test Qt context:" << qVersion() << current->format() + << "OpenGL ES:" << current->isOpenGLES() + << "glCreateProgram:" << bool(current->getProcAddress("glCreateProgram")); + if (getString) + qWarning() << "Presentation test GL:" << reinterpret_cast(getString(GL_VENDOR)) + << reinterpret_cast(getString(GL_RENDERER)) + << reinterpret_cast(getString(GL_VERSION)); + } + void paintGL() override { + QPainter painter(this); + painter.fillRect(rect(), Qt::blue); + painter.beginNativePainting(); + // Qt also uploads textures for popup labels and screenshot overlays. + // Native drawing must leave its texture binding intact. + auto* gl = context()->functions(); + GLuint qtTextures[2] = {}; + gl->glGenTextures(2, qtTextures); + gl->glActiveTexture(GL_TEXTURE0); + gl->glBindTexture(GL_TEXTURE_2D, qtTextures[0]); +#ifdef __APPLE__ + GLuint qtRectangle = 0; + gl->glGenTextures(1, &qtRectangle); + gl->glBindTexture(GL_TEXTURE_RECTANGLE, qtRectangle); +#endif + gl->glActiveTexture(GL_TEXTURE1); + gl->glBindTexture(GL_TEXTURE_2D, qtTextures[1]); + nativeDraw = presenter.Draw(0, 0, width() * devicePixelRatioF(), height() * devicePixelRatioF()); + GLint active = 0, binding = 0, otherBinding = 0; + gl->glGetIntegerv(GL_ACTIVE_TEXTURE, &active); + gl->glActiveTexture(GL_TEXTURE1); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &otherBinding); + gl->glBindTexture(GL_TEXTURE_2D, 0); + gl->glActiveTexture(GL_TEXTURE0); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &binding); + textureStatePreserved = binding == GLint(qtTextures[0]) && active == GL_TEXTURE1 && + otherBinding == GLint(qtTextures[1]); +#ifdef __APPLE__ + GLint rectangle = 0; + gl->glGetIntegerv(GL_TEXTURE_BINDING_RECTANGLE, &rectangle); + textureStatePreserved = textureStatePreserved && rectangle == GLint(qtRectangle); + gl->glBindTexture(GL_TEXTURE_RECTANGLE, 0); + gl->glDeleteTextures(1, &qtRectangle); +#endif + gl->glBindTexture(GL_TEXTURE_2D, 0); + gl->glDeleteTextures(2, qtTextures); + painter.endNativePainting(); + painter.fillRect(QRect(0, 0, 20, 20), Qt::green); + } +}; +struct Frames { + AVBufferRef* device = nullptr; + AVBufferRef* pool = nullptr; + AVFrame* cpu = av_frame_alloc(); + AVFrame* gpu = av_frame_alloc(); + ~Frames() { + av_frame_free(&cpu); + av_frame_free(&gpu); + av_buffer_unref(&pool); + av_buffer_unref(&device); + } +}; +} +TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[vulkan][presentation]") { + // CTest reports "Skipping Vulkan test:" as skipped; see tests/CMakeLists.txt. + if (!openshot::VulkanRenderingEnabled()) { + WARN("Skipping Vulkan test: CPU mode, or no Vulkan library"); + return; + } + const QByteArray platform = qgetenv("QT_QPA_PLATFORM"); +#ifdef __linux__ + if ((platform.isEmpty() || platform.startsWith("xcb")) && !qEnvironmentVariableIsSet("DISPLAY") && + !qEnvironmentVariableIsSet("WAYLAND_DISPLAY")) { + WARN("Skipping Vulkan test: presentation needs a display with OpenGL"); + return; + } +#endif + if (platform == "offscreen" || platform == "minimal") { + WARN("Skipping Vulkan test: presentation needs a native OpenGL display"); + return; + } + Frames data; + if (openshot::CreateSharedVulkanDevice(&data.device) < 0) { + WARN("Skipping Vulkan test: no Vulkan device is available"); + return; + } + int argc = 1; + char name[] = "vulkan-presentation"; + char* argv[] = {name, nullptr}; +#ifdef _WIN32 + // Match the editor: ANGLE/Direct3D cannot import Vulkan Win32 resources. + if (!qEnvironmentVariableIsSet("QT_OPENGL")) + qputenv("QT_OPENGL", "desktop"); +#endif + QApplication app(argc, argv); + data.pool = av_hwframe_ctx_alloc(data.device); + REQUIRE(data.pool); + auto pool = reinterpret_cast(data.pool->data); + pool->format = AV_PIX_FMT_VULKAN; + pool->sw_format = AV_PIX_FMT_RGBA; + pool->width = pool->height = 64; + REQUIRE(av_hwframe_ctx_init(data.pool) >= 0); + REQUIRE(av_hwframe_get_buffer(data.pool, data.gpu, 0) >= 0); + data.cpu->format = AV_PIX_FMT_RGBA; + data.cpu->width = data.cpu->height = 64; + REQUIRE(av_frame_get_buffer(data.cpu, 32) >= 0); + for (int y = 0; y < 64; ++y) { + for (int x = 0; x < 64; ++x) { + auto p = data.cpu->data[0] + y * data.cpu->linesize[0] + x * 4; + p[0] = y < 32 ? 255 : 0; p[1] = 0; p[2] = y < 32 ? 0 : 255; p[3] = 255; + } + } + REQUIRE(av_hwframe_transfer_data(data.gpu, data.cpu, 0) >= 0); + auto frame = std::make_shared(); + frame->AddDeferredImage(std::make_shared(data.gpu, true)); + Preview widget; + widget.resize(128, 128); + VideoRenderer renderer; + renderer.OverrideWidget(reinterpret_cast(&widget)); + widget.show(); + QCoreApplication::processEvents(); + auto* context = widget.context(); + if (!context +#ifdef _WIN32 + || !context->hasExtension("GL_EXT_memory_object_win32") || !context->hasExtension("GL_EXT_semaphore_win32") +#elif !defined(__APPLE__) + || !context->hasExtension("GL_EXT_memory_object_fd") || !context->hasExtension("GL_EXT_semaphore_fd") +#endif + ) { + WARN("Skipping Vulkan test: OpenGL lacks Vulkan memory/semaphore interop"); + return; + } + const auto readbacks = openshot::VulkanFrameReadbackCount(); + renderer.paint(frame); + for (int i = 0; i < 20; ++i) { + QCoreApplication::processEvents(); + QThread::msleep(5); + } + QImage image; + // Exercise both semaphore directions repeatedly; a single frame can hide + // a race where Vulkan consumes its own release signal before GL waits. + for (int i = 0; i < 64; ++i) { + image = widget.grabFramebuffer(); + if (i == 0 && qEnvironmentVariableIsSet("OPENSHOT_PRESENTATION_CAPTURE")) + image.save(qEnvironmentVariable("OPENSHOT_PRESENTATION_CAPTURE")); + INFO("frame=" << i << " top=" << image.pixelColor(image.width()/2, image.height()/4).name().toStdString() + << " bottom=" << image.pixelColor(image.width()/2, 3*image.height()/4).name().toStdString()); + REQUIRE(widget.nativeDraw); + CHECK(widget.textureStatePreserved); + REQUIRE(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); + REQUIRE(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); + } + REQUIRE(widget.nativeDraw); + CHECK(widget.textureStatePreserved); + CHECK(widget.presenter.NativeDraws() >= 64); + CHECK(widget.presenter.FallbackFrames() == 0); + CHECK(widget.presenter.FallbackReason().empty()); + CHECK(openshot::VulkanFrameReadbackCount() == readbacks); + REQUIRE(frame->GetDeferredImage()); + REQUIRE_FALSE(image.isNull()); + if (qEnvironmentVariableIsSet("OPENSHOT_PRESENTATION_CAPTURE")) + image.save(qEnvironmentVariable("OPENSHOT_PRESENTATION_CAPTURE")); + INFO("center=" << image.pixelColor(image.width()/2, image.height()/2).name().toStdString() + << " overlay=" << image.pixelColor(5, 5).name().toStdString()); + CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); + CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); + CHECK(image.pixelColor(5, 5) == QColor(Qt::green)); + // CPU-origin artwork follows upload -> compositor -> native presentation. + QImage artwork(64, 64, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < 64; ++y) + for (int x = 0; x < 64; ++x) + artwork.setPixelColor(x, y, y < 32 ? Qt::red : Qt::blue); + auto uploaded = openshot::UploadVulkanImage(artwork); + frame->AddDeferredImage(openshot::RenderVulkanImage(uploaded, nullptr, 64, 64, QTransform(), 1.0, Qt::transparent)); + renderer.paint(frame); + for (int i = 0; i < 20; ++i) { + QCoreApplication::processEvents(); + QThread::msleep(5); + } + image = widget.grabFramebuffer(); + CHECK(widget.nativeDraw); + CHECK(widget.presenter.FallbackFrames() == 0); + CHECK(openshot::VulkanFrameReadbackCount() == readbacks); + CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); + CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); + // A new frame size replaces the presenter's shared OpenGL image. + QImage resized(96, 48, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < resized.height(); ++y) + for (int x = 0; x < resized.width(); ++x) + resized.setPixelColor(x, y, y < resized.height() / 2 ? Qt::blue : Qt::red); + frame->AddDeferredImage(openshot::RenderVulkanImage(openshot::UploadVulkanImage(resized), nullptr, + 96, 48, QTransform(), 1.0, Qt::transparent)); + renderer.paint(frame); + for (int i = 0; i < 20; ++i) { + QCoreApplication::processEvents(); + QThread::msleep(5); + } + image = widget.grabFramebuffer(); + CHECK(widget.nativeDraw); + CHECK(widget.presenter.FallbackFrames() == 0); + CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::blue)); + CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::red)); + // Recovery must not read pixels from a dead device or throw through paintGL. + openshot::ReportVulkanResult(VK_ERROR_DEVICE_LOST); + CHECK_NOTHROW(widget.presenter.Present(frame)); + CHECK_FALSE(widget.presenter.Draw(0, 0, 128, 128)); + CHECK(widget.presenter.FallbackReason().find("device lost") != std::string::npos); + CHECK(openshot::VulkanFrameReadbackCount() == readbacks); + widget.presenter.Clear(); + CHECK_FALSE(widget.presenter.Draw(0, 0, 128, 128)); + CHECK_FALSE(widget.presenter.FallbackReason().empty()); +} diff --git a/tests/generate-color-charts.py b/tests/generate-color-charts.py new file mode 100644 index 000000000..f3a169d10 --- /dev/null +++ b/tests/generate-color-charts.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +""" +Generate the color chart fixtures used by tests/ColorManagement.cpp. + +Each chart is a 4x4 grid of flat patches (160x96 px, 640x384 frame, 3 frames) +with known colors, (near-)losslessly encoded: + + examples/color-chart-sdr-bt709.mp4 8-bit H.264, BT.709 tagged + examples/color-chart-hlg.mp4 10-bit HEVC, BT.2020 + HLG (iPhone style) + examples/color-chart-pq.mp4 10-bit HEVC, BT.2020 + PQ (HDR10) + +Patches are SDR code values (PATCHES in tests/ColorManagement.cpp). HDR charts +place them per ITU-R BT.2408: BT.1886 display light with 100% = 203 nits. The +last two patches are 600 and 1000 nit highlights. + +Usage: generate-color-charts.py [ffmpeg] [output_dir] (needs numpy, libx264, libx265) +""" +# Local developer tools use explicit argument lists, never a shell. +import subprocess # nosec B404 +import sys + +import numpy as np + +FFMPEG = sys.argv[1] if len(sys.argv) > 1 else "ffmpeg" +OUT = sys.argv[2] if len(sys.argv) > 2 else "examples" +W, H, PW, PH = 640, 384, 160, 96 +PATCHES = [(255, 255, 255), (118, 118, 118), (224, 172, 140), (141, 85, 62), (98, 155, 210), + (87, 122, 59), (220, 30, 30), (40, 200, 40), (30, 40, 220), (240, 220, 40), + (20, 20, 20), 600.0, 1000.0] +WHITE = 203.0 +M709_TO_2020 = np.array([[0.6274, 0.3293, 0.0433], [0.0691, 0.9195, 0.0114], [0.0164, 0.0880, 0.8956]]) +K709, K2020 = (0.2126, 0.0722), (0.2627, 0.0593) + + +def pq(nits): + m1, m2, c1, c2, c3 = 2610 / 16384, 2523 / 4096 * 128, 3424 / 4096, 2413 / 4096 * 32, 2392 / 4096 * 32 + y = np.clip(nits / 10000.0, 0, 1) ** m1 + return ((c1 + c2 * y) / (1 + c3 * y)) ** m2 + + +def hlg(nits): # display light (BT.2020) -> HLG signal, 1000 nit display, gamma 1.2 + a, b, c = 0.17883277, 0.28466892, 0.55991073 + yd = nits @ np.array([K2020[0], 1 - sum(K2020), K2020[1]]) + ys = (np.maximum(yd, 1e-9) / 1000.0) ** (1 / 1.2) + e = np.clip(nits / 1000.0 / np.maximum(ys ** 0.2, 1e-9)[..., None], 0, 1) + return np.where(e <= 1 / 12, np.sqrt(3 * e), a * np.log(np.maximum(12 * e - b, 1e-12)) + c) + + +def chart_nits(): + image = np.zeros((H, W, 3)) + for i, patch in enumerate(PATCHES): + r, c = divmod(i, 4) + value = np.full(3, patch) if isinstance(patch, float) else (np.array(patch) / 255.0) ** 2.4 * WHITE + image[r * PH:(r + 1) * PH, c * PW:(c + 1) * PW] = value + return image + + +def write(name, rgb, k, bits, tags, encoder): + kr, kb = k + y = rgb @ np.array([kr, 1 - kr - kb, kb]) + cb, cr = (rgb[..., 2] - y) / (2 * (1 - kb)), (rgb[..., 0] - y) / (2 * (1 - kr)) + s = 1 << (bits - 8) + planes = np.stack([16 + 219 * y, 128 + 224 * cb, 128 + 224 * cr]) * s + planes = np.round(planes).clip(0, (1 << bits) - 1).astype(np.uint16 if bits > 8 else np.uint8) + pix_in = "yuv444p10le" if bits > 8 else "yuv444p" + pix_out = "yuv420p10le" if bits > 8 else "yuv420p" + # The developer selects FFmpeg; all encoding options are fixed arguments. + subprocess.run([FFMPEG, "-y", "-v", "error", "-f", "rawvideo", "-pix_fmt", pix_in, "-s", f"{W}x{H}", + "-r", "30", *tags, "-i", "-", *tags, "-pix_fmt", pix_out, *encoder, f"{OUT}/{name}"], # nosec B603 + input=planes.tobytes() * 3, check=True) + + +def tags(trc, primaries, matrix): + return ["-color_primaries", primaries, "-color_trc", trc, "-colorspace", matrix, "-color_range", "tv"] + + +nits = chart_nits() +# Near-lossless standard Main10 (as cameras produce; some hardware decoders +# reject lossless HEVC). +x265 = ["-c:v", "libx265", "-x265-params", "crf=1:log-level=error", "-tag:v", "hvc1"] +write("color-chart-sdr-bt709.mp4", np.clip(nits / WHITE, 0, 1) ** (1 / 2.4), K709, 8, + tags("bt709", "bt709", "bt709"), ["-c:v", "libx264", "-profile:v", "high", "-crf", "1"]) +write("color-chart-hlg.mp4", hlg(nits @ M709_TO_2020.T), K2020, 10, + tags("arib-std-b67", "bt2020", "bt2020nc"), x265) +write("color-chart-pq.mp4", pq(nits @ M709_TO_2020.T), K2020, 10, + tags("smpte2084", "bt2020", "bt2020nc"), x265)