diff --git a/.github/workflows/ffmpeg-compatibility.yml b/.github/workflows/ffmpeg-compatibility.yml
index 4443fa0f3..078298474 100644
--- a/.github/workflows/ffmpeg-compatibility.yml
+++ b/.github/workflows/ffmpeg-compatibility.yml
@@ -3,10 +3,16 @@
name: FFmpeg compatibility
on:
+ push:
+ branches: [vulkan-mvp]
pull_request:
paths:
- 'src/FFmpeg*'
+ - 'src/Vulkan*'
+ - 'src/GpuDevices*'
+ - 'src/Qt/Vulkan*'
- 'tests/FFmpeg*'
+ - 'tests/Vulkan*'
- 'tests/CMakeLists.txt'
- '.github/workflows/ffmpeg-compatibility.yml'
workflow_dispatch:
@@ -57,7 +63,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y cmake ninja-build pkg-config nasm catch2 \
libasound2-dev qtbase5-dev libqt5svg5-dev libbabl-dev libjsoncpp-dev \
- libx264-dev libvpx-dev libvorbis-dev
+ libx264-dev libvpx-dev libvorbis-dev libvulkan-dev glslang-tools
- name: Build isolated FFmpeg and audio dependencies
run: |
prefix="$RUNNER_TEMP/ffmpeg-dependencies"
@@ -83,9 +89,11 @@ jobs:
cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH="$RUNNER_TEMP/ffmpeg-dependencies" \
-DENABLE_LIB_DOCS=OFF -DENABLE_PYTHON=OFF -DENABLE_OPENCV=OFF \
- -DENABLE_MAGICK=OFF -DUSE_QT6=OFF
+ -DENABLE_MAGICK=OFF -DUSE_QT6=OFF -DENABLE_GPU_FRAME=ON
cmake --build build --parallel 2 --target \
- openshot-FFmpegCapabilities-test openshot-FFmpegWriter-test
+ openshot-FFmpegCapabilities-test openshot-FFmpegWriter-test \
+ openshot-VulkanCompositor-test openshot-VulkanPresentation-test
- name: Test capability validation and audio/video exports
working-directory: build
- run: ctest -L '^(FFmpegCapabilities|FFmpegWriter)$' --output-on-failure --timeout 180 -j2
+ # Hardware cases skip without Vulkan support; the CPU fallback still runs.
+ run: ctest -L '^(FFmpegCapabilities|FFmpegWriter|VulkanCompositor|VulkanPresentation)$' --output-on-failure --timeout 180 -j2
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 610e468cc..063f6aa92 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,13 +24,17 @@ init-builders:
linux-builder:
stage: build-libopenshot
variables:
+ OPENSHOT_QT_ROOT: /opt/openshot/qt-5.15.19
OPENCV_ROOT: /opt/opencv-4.13.0
OpenCV_DIR: /opt/opencv-4.13.0/lib/cmake/opencv4
+ # Runner image provides FFmpeg 7 (with Vulkan) and newer Vulkan headers in /opt
FFMPEG_ROOT: /opt/ffmpeg-7.1.3
+ VULKAN_HEADERS_ROOT: /opt/vulkan-headers
artifacts:
expire_in: 6 months
paths:
- build/install-x64/*
+ - build/ffmpeg-hardware-check/*
script:
- "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=linux-builder"
- if [ ! -f artifacts.zip ]; then
@@ -43,13 +47,14 @@ linux-builder:
- export PKG_CONFIG_PATH="$OPENCV_ROOT/lib/pkgconfig:$PKG_CONFIG_PATH"
- export LD_LIBRARY_PATH="$OPENCV_ROOT/lib:$LD_LIBRARY_PATH"
- export PATH="$OPENCV_ROOT/bin:$PATH"
- - if [ ! -f "$FFMPEG_ROOT/env.sh" ]; then echo "ERROR runner is missing $FFMPEG_ROOT"; exit 1; fi
+ - if [ ! -f "$FFMPEG_ROOT/env.sh" ]; then echo "ERROR runner is missing $FFMPEG_ROOT (FFmpeg 7 with Vulkan)"; exit 1; fi
- . "$FFMPEG_ROOT/env.sh"
+ - . "$OPENSHOT_QT_ROOT/env.sh"
- export FFMPEGDIR="$FFMPEG_ROOT"
- bash ci/linux/check-ffmpeg-hardware.sh "$FFMPEG_ROOT" "$CI_PROJECT_DIR/build/ffmpeg-hardware-check"
- mkdir -p build; cd build;
- pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0 || echo "WARNING PipeWire Wayland capture dependencies are missing; building without Wayland capture support"
- - cmake -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON ../
+ - cmake -U "Qt5*_DIR" -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_QT6=OFF -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$VULKAN_HEADERS_ROOT/include" -D"CMAKE_CXX_FLAGS=-I$VULKAN_HEADERS_ROOT/include" ../
- cmake --build . --parallel $(nproc)
- ldd src/libopenshot.so | tee libopenshot-deps.log
- if grep -qE "not found|libavcodec\.so\.58|libavutil\.so\.56" libopenshot-deps.log; then echo "ERROR unresolved libraries or system FFmpeg 4.2"; exit 1; fi
@@ -89,7 +94,8 @@ mac-builder:
- export PKG_CONFIG_PATH="$FFMPEGDIR/lib/pkgconfig:$PKG_CONFIG_PATH"
- export DYLD_LIBRARY_PATH=$FFMPEGDIR/lib:$OPENCV_ROOT/lib:$DYLD_LIBRARY_PATH
- mkdir -p build; cd build;
- - cmake -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.12" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../
+ - test -f "$FFMPEGDIR/lib/libMoltenVK.dylib" && test -x "$FFMPEGDIR/bin/glslangValidator"
+ - cmake -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$FFMPEGDIR/include" -D"Vulkan_LIBRARY:FILEPATH=$FFMPEGDIR/lib/libMoltenVK.dylib" -D"GLSLANG_VALIDATOR:FILEPATH=$FFMPEGDIR/bin/glslangValidator" -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.0.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.15" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../
- cmake --build . --parallel $(sysctl -n hw.ncpu)
- make install
- find "$CI_PROJECT_DIR/build/install-x64" -type f \( -name "*.dylib" -or -name "*.so" \) -exec sh -c 'for f do if ! otool -l "$f" | grep -q "$OPENCV_ROOT/lib"; then install_name_tool -add_rpath "$OPENCV_ROOT/lib" "$f" || true; fi; done' sh {} +
@@ -125,7 +131,7 @@ windows-builder-x64:
- 'if ($LASTEXITCODE -ne 0) { throw "Could not enumerate FFmpeg encoders" }; foreach ($codec in @("h264_nvenc", "hevc_nvenc", "h264_qsv", "hevc_qsv", "h264_amf", "hevc_amf")) { if ($gpuEncoders -notmatch "\b$codec\b") { throw "Windows FFmpeg is missing required hardware encoder $codec" } }'
- ffmpeg -hide_banner -devices | findstr /I "gdigrab"
- ffmpeg -hide_banner -devices | findstr /I "dshow"
- - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release"
+ - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -DENABLE_GPU_FRAME=ON
- cmake --build build --parallel 4
- $testList = ctest --test-dir build --show-only=json-v1 | ConvertFrom-Json
- if ($LASTEXITCODE -ne 0 -or @($testList.tests).Count -lt 400) { throw "Incomplete unit test discovery; check runtime DLL paths" }
@@ -202,7 +208,7 @@ windows-builder-x86:
- 'if ($LASTEXITCODE -ne 0) { throw "Could not enumerate FFmpeg encoders" }; foreach ($codec in @("h264_nvenc", "hevc_nvenc", "h264_qsv", "hevc_qsv", "h264_amf", "hevc_amf")) { if ($gpuEncoders -notmatch "\b$codec\b") { throw "Windows FFmpeg is missing required hardware encoder $codec" } }'
- ffmpeg -hide_banner -devices | findstr /I "gdigrab"
- ffmpeg -hide_banner -devices | findstr /I "dshow"
- - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32 -msse2 -mfpmath=sse" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32 -msse2 -mfpmath=sse"
+ - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32 -msse2 -mfpmath=sse" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32 -msse2 -mfpmath=sse" -DENABLE_GPU_FRAME=ON
- cmake --build build --parallel 4
- $testList = ctest --test-dir build --show-only=json-v1 | ConvertFrom-Json
- if ($LASTEXITCODE -ne 0 -or @($testList.tests).Count -lt 400) { throw "Incomplete unit test discovery; check runtime DLL paths" }
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 35e3a1bda..c3db62bdc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,7 +24,7 @@ For more information, please visit .
set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules")
################ PROJECT VERSION ####################
-set(PROJECT_VERSION_FULL "1.1.1")
+set(PROJECT_VERSION_FULL "1.2.0-dev1")
set(PROJECT_SO_VERSION 33)
# Remove the dash and anything following, to get the #.#.# version for project()
@@ -75,6 +75,7 @@ option(ENABLE_OPENCV "Build with OpenCV algorithms (requires Boost, Protobuf 3)"
option(ENABLE_WAYLAND_CAPTURE "Build Wayland screen capture support with xdg-desktop-portal and PipeWire (Linux only)" ON)
option(USE_HW_ACCEL "Enable hardware-accelerated encoding-decoding with FFmpeg 3.4+" ON)
option(ENABLE_VULKAN_BENCHMARK "Build the experimental Vulkan benchmark example" OFF)
+option(ENABLE_GPU_FRAME "Build experimental deferred GPU frame storage" OFF)
# Legacy commandline override
if (DISABLE_TESTS)
diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt
index ef43d543f..662dae963 100644
--- a/bindings/python/CMakeLists.txt
+++ b/bindings/python/CMakeLists.txt
@@ -89,6 +89,14 @@ if (CMAKE_VERSION VERSION_GREATER 3.20)
endif()
### Add the SWIG interface file (which defines all the SWIG methods)
+# Older UseSWIG versions do not track transitive %include headers. Regenerate
+# the wrapper when GPU diagnostics change, including in incremental builds.
+set(SWIG_MODULE_pyopenshot_EXTRA_DEPS
+ "${PROJECT_SOURCE_DIR}/src/GpuDevices.h"
+ "${PROJECT_SOURCE_DIR}/src/GpuMetrics.h"
+ "${PROJECT_SOURCE_DIR}/src/Timeline.h"
+ "${PROJECT_SOURCE_DIR}/src/FFmpegWriter.h"
+ "${PROJECT_SOURCE_DIR}/src/Qt/VulkanOpenGLPresenter.h")
swig_add_library(pyopenshot LANGUAGE python SOURCES openshot.i)
if (POLICY CMP0078)
diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i
index 6602b465c..99c6b3680 100644
--- a/bindings/python/openshot.i
+++ b/bindings/python/openshot.i
@@ -110,6 +110,13 @@ class QWidget;
#include "QtTextReader.h"
#include "KeyFrame.h"
#include "RendererBase.h"
+#if defined(OPENSHOT_ENABLE_GPU_FRAME)
+#include "Qt/VulkanOpenGLPresenter.h"
+#include "VulkanCompositor.h"
+#include "VulkanFrameImage.h"
+#include "GpuDevices.h"
+#endif
+#include "GpuMetrics.h"
#include "Settings.h"
#include "TimelineBase.h"
#include "Timeline.h"
@@ -520,6 +527,14 @@ static int openshot_swig_is_qwidget(PyObject *obj) {
%include "KeyFrame.h"
%include "AnimatedCurve.h"
%include "RendererBase.h"
+#if defined(OPENSHOT_ENABLE_GPU_FRAME)
+%include "Qt/VulkanOpenGLPresenter.h"
+namespace openshot { void ReleaseVulkanMemory(); uint64_t VulkanFrameReadbackCount(); uint64_t VulkanUploadFailureCount(); }
+%include "GpuDevices.h"
+%template(GpuDeviceVector) std::vector;
+%template(StringVector) std::vector;
+#endif
+%include "GpuMetrics.h"
%include "Settings.h"
%include "TimelineBase.h"
%include "Qt/VideoCacheThread.h"
diff --git a/ci/linux/check-ffmpeg-hardware.sh b/ci/linux/check-ffmpeg-hardware.sh
old mode 100644
new mode 100755
index 4c7275e21..45c3ade0f
--- a/ci/linux/check-ffmpeg-hardware.sh
+++ b/ci/linux/check-ffmpeg-hardware.sh
@@ -11,7 +11,7 @@ export LD_LIBRARY_PATH="$root/lib:${LD_LIBRARY_PATH:-}"
"$root/bin/ffmpeg" -hide_banner -version > "$log/version.txt"
"$root/bin/ffmpeg" -hide_banner -encoders > "$log/encoders.txt" 2>&1
"$root/bin/ffmpeg" -hide_banner -hwaccels > "$log/hwaccels.txt" 2>&1
-for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf h264_vaapi hevc_vaapi; do
+for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf h264_vaapi hevc_vaapi h264_vulkan hevc_vulkan; do
grep -Eq "[[:space:]]$codec[[:space:]]" "$log/encoders.txt" || { echo "Missing required encoder: $codec"; exit 1; }
done
ldd "$root/lib/libavcodec.so.61" > "$log/avcodec-dependencies.txt"
@@ -21,4 +21,4 @@ grep -q 'libvpl.so.2' "$log/avcodec-dependencies.txt"
"$root/bin/ffmpeg" -v error -i "$log/h264-aac.mp4" -f null -
"$root/bin/ffmpeg" -v error -f lavfi -i testsrc2=size=128x96:rate=4 -frames:v 4 -c:v libaom-av1 -cpu-used 8 -threads 2 -y "$log/av1.webm"
"$root/bin/ffmpeg" -v error -c:v libdav1d -i "$log/av1.webm" -f null -
-echo 'SUCCESS: NVENC, QSV, AMF and VA-API encoders compiled; dispatcher linked; H.264/AAC and AV1 software smoke tests passed'
+echo 'SUCCESS: NVENC, QSV, AMF, VA-API and Vulkan encoders compiled; dispatcher linked; H.264/AAC and AV1 software smoke tests passed'
diff --git a/ci/linux/setup-ffmpeg-vulkan.sh b/ci/linux/setup-ffmpeg-vulkan.sh
new file mode 100755
index 000000000..fa56a020c
--- /dev/null
+++ b/ci/linux/setup-ffmpeg-vulkan.sh
@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Install FFmpeg with Vulkan (plus the codecs the focal AppImage ships) into
+# /opt, without touching the system FFmpeg used by the CPU-only builds.
+# Ubuntu 20.04 (focal). Run as a sudo-capable user. Safe to re-run.
+# Not enabled: libwavpack (removed from FFmpeg 5+, native decoder used),
+# chromaprint (focal's library links FFmpeg 4.2, loading two FFmpegs at once).
+set -euo pipefail
+script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+
+FFMPEG_VERSION=${FFMPEG_VERSION:-7.1.3}
+VULKAN_HEADERS_TAG=${VULKAN_HEADERS_TAG:-v1.4.347}
+AOM_TAG=${AOM_TAG:-v3.15.2} # focal has 1.0, FFmpeg 7 needs 2.0+
+DAV1D_TAG=${DAV1D_TAG:-1.4.3} # focal has no libdav1d-dev
+NV_CODEC_TAG=${NV_CODEC_TAG:-n12.1.14.0} # NVIDIA driver 530+ for NVENC
+PREFIX=${PREFIX:-/opt/ffmpeg-${FFMPEG_VERSION}}
+VULKAN_PREFIX=${VULKAN_PREFIX:-/opt/vulkan-headers}
+SRC=${SRC:-/opt/src}
+JOBS=${JOBS:-$(nproc)}
+SKIP_APT=${SKIP_APT:-0}
+
+export DEBIAN_FRONTEND=noninteractive
+if [ "$SKIP_APT" != 1 ]; then
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends \
+ build-essential pkg-config nasm yasm git curl xz-utils cmake patch \
+ glslang-tools libvulkan-dev \
+ libx264-dev libx265-dev libvpx-dev meson ninja-build \
+ libmp3lame-dev libopus-dev libvorbis-dev libtheora-dev libspeex-dev \
+ libtwolame-dev libshine-dev libwavpack-dev libgsm1-dev libcodec2-dev \
+ libxvidcore-dev libwebp-dev libopenjp2-7-dev libsnappy-dev libsoxr-dev \
+ libass-dev libfreetype6-dev libfontconfig1-dev libfribidi-dev libxml2-dev \
+ libbluray-dev libbs2b-dev libcaca-dev libcdio-paranoia-dev flite1-dev \
+ libgme-dev libmysofa-dev libopenmpt-dev librsvg2-dev librubberband-dev \
+ libssh-dev libgnutls28-dev libvidstab-dev libzmq3-dev libzvbi-dev \
+ lv2-dev liblilv-dev ladspa-sdk frei0r-plugins-dev \
+ libpulse-dev libjack-jackd2-dev libopenal-dev libsdl2-dev \
+ ocl-icd-opencl-dev opencl-headers libgl-dev \
+ libdc1394-22-dev libiec61883-dev libavc1394-dev libraw1394-dev \
+ libdrm-dev libva-dev libvdpau-dev
+fi
+
+sudo mkdir -p "$SRC" && sudo chown "$(id -u):$(id -g)" "$SRC"
+cd "$SRC"
+
+# Vulkan headers: focal has 1.2.131, FFmpeg 7 needs 1.3.277+. Header-only.
+if [ ! -f "$VULKAN_PREFIX/include/vulkan/vulkan_core.h" ]; then
+ rm -rf Vulkan-Headers
+ git clone --depth 1 --branch "$VULKAN_HEADERS_TAG" https://github.com/KhronosGroup/Vulkan-Headers.git
+ # Copy directly: its CMakeLists needs CMake 3.22, focal has 3.16
+ sudo mkdir -p "$VULKAN_PREFIX"
+ sudo cp -r Vulkan-Headers/include "$VULKAN_PREFIX/"
+fi
+
+# NVENC/CUDA headers (header-only, loaded at runtime from the driver)
+if [ ! -f "$PREFIX/lib/pkgconfig/ffnvcodec.pc" ]; then
+ rm -rf nv-codec-headers
+ git clone --depth 1 --branch "$NV_CODEC_TAG" https://github.com/FFmpeg/nv-codec-headers.git
+ sudo make -C nv-codec-headers install PREFIX="$PREFIX"
+fi
+
+# Intel oneVPL dispatcher (supports both oneVPL and legacy Media SDK runtimes).
+# The dispatcher is bundled; GPU runtimes/drivers remain supplied by the host.
+if [ ! -f "$PREFIX/lib/pkgconfig/vpl.pc" ]; then
+ archive=libvpl-2.10.2.tar.gz
+ if [ ! -f "$archive" ]; then
+ curl -fsSL --retry 2 https://github.com/intel/libvpl/archive/refs/tags/v2.10.2.tar.gz -o "$archive"
+ fi
+ printf '%s %s\n' ad956ea7ecf14614325f59dfb44cc5ba08e2fcac373342d61c7db152ac651253 "$archive" | sha256sum -c -
+ mkdir -p libvpl-2.10.2
+ tar xzf "$archive" --strip-components=1 -C libvpl-2.10.2
+ cmake -S libvpl-2.10.2 -B libvpl-2.10.2/build -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=ON \
+ -DBUILD_TESTS=OFF -DBUILD_TOOLS=OFF -DBUILD_EXAMPLES=OFF -DINSTALL_EXAMPLE_CODE=OFF \
+ -DBUILD_DISPATCHER_ONEVPL_EXPERIMENTAL=OFF
+ cmake --build libvpl-2.10.2/build --parallel "$JOBS"
+ sudo cmake --install libvpl-2.10.2/build
+fi
+
+# AMD AMF headers, pinned to the 1.4.35 release commit and verified individually.
+# FFmpeg loads the AMF runtime from the installed AMD driver at export time.
+amf="$PREFIX/include/AMF"
+sudo mkdir -p "$amf"
+while read -r digest header; do
+ if ! printf '%s %s\n' "$digest" "$amf/$header" | sha256sum -c - >/dev/null 2>&1; then
+ mkdir -p "amf-headers/$(dirname "$header")"
+ curl -fsSL --retry 2 "https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/AMF/ba07d1b3c7c7ee0d846125cffc1d6d78a020c45a/amf/public/include/$header" -o "amf-headers/$header"
+ printf '%s %s\n' "$digest" "amf-headers/$header" | sha256sum -c -
+ sudo mkdir -p "$amf/$(dirname "$header")"
+ sudo cp "amf-headers/$header" "$amf/$header"
+ fi
+done <<'AMF_HEADERS'
+70afd5279be90e656c2ec270229c5a702a11435a670052370e96b270e6b7640f components/AMFXInput.h
+27770ae77d2f7a16eca9ab1f8b9ce87cf0c0350425b950bdd39189c2c91a519a components/Ambisonic2SRenderer.h
+05a9ebc32217e69b6c4659f4744dbda9e5a2792f3e56bc1bcf61c15dcfd46cab components/AudioCapture.h
+bbe91a1e744685a36269d3b6af9cd36d9716ee748c8d0ed95255a6a42b46892f components/Capture.h
+a3a89490d44adaffc2fa2cccc14313f8f65e5af7a6431d89d85bd80e36c628a1 components/ChromaKey.h
+2f6887d14516ebdcf034d59e39235f5d9974533ddb0b04d685c0c3b6d397d3d6 components/ColorSpace.h
+f3f18680d55a65fdf5e0351d3fde1dcd9bdd2d6caec268c8c915c4227ffa1edc components/Component.h
+2479eb2a6174a1b63552c509abac1b701eddcab5a5eac5a7f4d9f40274d588b4 components/ComponentCaps.h
+7c22337ff313320bd7fe20ef0fb00756735012367a5daa7c621725030f578863 components/CursorCapture.h
+693c8e491f90eaffd226c1f6b2da37d3a5ef1d3c610e554647f7d0b699832727 components/DisplayCapture.h
+9dde7e850c8bdecda82b0cee1b331294f6cc91b32b1d1504344a023cd4dda8af components/FFMPEGAudioConverter.h
+92f50d789bc18d5ebd143b2925e124594a89110eef7efaa8189eb063bf2f3aa8 components/FFMPEGAudioDecoder.h
+2493067de9b3f540ffaae8a6466e57ef75af4772f037a81785b11ee9511029f5 components/FFMPEGAudioEncoder.h
+87a9ae9480c3af6210b1282d1768e36673fcc1ef4970afa6edaa74481cf0831b components/FFMPEGComponents.h
+5d64b7d2d50ca17ba601885dcdb51bb16ef4dccf88323437967e8524f26194bc components/FFMPEGEncoderAV1.h
+3dae38db05bc55af525000c28559ff76625e0cddc2ad2c4c2af52eab69e915bc components/FFMPEGEncoderH264.h
+3ad5cc5d3284b253a671499efa29c7a46b83a6e6373aad7142c7adcbfae31875 components/FFMPEGEncoderHEVC.h
+be992e14086b04e2a7b5503471a6c262c58fda163c45ad6d00f8e425d9f4a820 components/FFMPEGFileDemuxer.h
+3fefbe1aa7aaa4c000a97efb2e5ba26dc87ec15d11b07ceb0f43c768a1d589a2 components/FFMPEGFileMuxer.h
+297b0ab2291baeb44f30f03ffbf791a5603289ab34f625d60a33150f2be1f7b6 components/FFMPEGVideoDecoder.h
+ce5fb69c0546dd08dff0ee680972c5ec5a0b2d9b95e509338358eb2827f30221 components/FRC.h
+b63e856b55db7a6853f444a6d4d32cfe006d9e0fb1d9be1b3079557534738591 components/HQScaler.h
+c4e168d29e3f7d9318435958c3c784ad4c497159a617bd7a5598a2951b1b15f6 components/MediaSource.h
+dae3564f20ef5db959d31dfc74c226453b039f4919ddb65027a183cdae95c171 components/PreAnalysis.h
+5a01d71e128cf8bcaa4945fe76d9aad0156c409df3b25616a75645ef15972dc3 components/PreProcessing.h
+550900e56c8254bf9c42d1b1d6538a390f7f7d3bf4826ee385b019585cb6cd22 components/SupportedCodecs.h
+146e7cd35acc846f6b75b696bbe13a5d790f885bc23d9913e7de66f515da789b components/VQEnhancer.h
+08f0162a631833f847d1b4b751e41e28b147a79ed19545b43631222d22e74507 components/VideoCapture.h
+b0e173fbfe920a4923f461c1bffc73d1cbc06d54b676647dcd6ec300a0beee65 components/VideoConverter.h
+588622f7d945c85c1ed334b497174fd46031e01664338bcf59c4b242920df886 components/VideoDecoderUVD.h
+0b43ba9a0bb0d701c117e038d07270907d840ea2d2b13655fec71eda5e96d562 components/VideoEncoderAV1.h
+b7827d2c3380a32cef350cf0f180802694f9f93db295f1c8e08238160d17c3d8 components/VideoEncoderHEVC.h
+d106ce15028a9ea42e7299cff4b1a8697614a746b96740c9251f9b943bac2edc components/VideoEncoderVCE.h
+9bc2d8477d67693b0cd4d433fbc6a5812277190f67be5f188f0206b463f256ec components/VideoStitch.h
+3c6e6515a8fec111eb1847e349124a8c66c1e78be2d148bc0b8f85b9d9994e2d components/ZCamLiveStream.h
+c9382c911fd3b5e12105c1d2a67eb11e62e9475068825222a285bb1314ac0ddd core/AudioBuffer.h
+1ad18d3c813c3f81a836d393bbe86dfc0b7a1d7f04a77a272eb1461ce85ae3a4 core/Buffer.h
+ef5721466fe48e1c038151018699519761e6e351e66e76a5de65386924653e20 core/Compute.h
+50acf69fb670bc6cf093384d1b698bf9e44cda8dd49e02ef24eacfb86ad8ccae core/ComputeFactory.h
+f9b4acb784038aad234ddec490e5f6ad6dacc4b14794eb84f3523d56dc3d6dd2 core/Context.h
+76d16ba57d48ae9e457f3d904698a1e7c625007ac8887db9bcecdb90fd36d835 core/CurrentTime.h
+f05771305598b534e1efa0dd28825267ea87418c477b198e373f6764d060c666 core/D3D12AMF.h
+eaac4d80dd3215638c70f8a4ca01791d26d9df77586f665a772bb51907a8035b core/Data.h
+56666589bf6ad20c6286a048da751c2d7722f70c73d962ee3c205a6d709b6aff core/Debug.h
+a5e0fa9290cae767a4d135962ee5e30302e65e8d0ca5e016c620e72b759c86c6 core/Dump.h
+9ba2ebe93af628bce621f29de493cddab87bf8a030eed504b25fcdc822c61b44 core/Factory.h
+304f6d6077e9b0733859d44a5e43a343ae374c8fe72c7d79741c0aca89c21162 core/Interface.h
+5ad5165e9de2afcfc2da828e8704c838b1f82e8ead7422fbcfc32c758672e1f3 core/Plane.h
+59a4d11ecd150af25ee0b8d1d15f2efdfe5cf1060b8e8ab539f6a46d4bff8bc8 core/Platform.h
+0059b787e78cd7c4e57212ccaab56ad7e17e9474a31337da0eac5b2d056a5dd8 core/PropertyStorage.h
+871136e46586baa1fc77121935b503aaa16523d006c3e435ce50698605675fad core/PropertyStorageEx.h
+a469ff33875484b5762199da757819056ff4a7535bfdcec1f4cf2ffc403b98f2 core/Result.h
+7f0c99bbfe156169440031f75b0e37c28c435dd140ecb4e5089d0b1f01912c47 core/Surface.h
+ae8c5fee671f02b90d3ccab312cfa6f74424351f070a7dd4ed7aba437fd55c30 core/Trace.h
+72841e751595a2a8f62c8ac4a3e630b181411af8e0741d716bbe3379c9bf9a99 core/Variant.h
+e26e25e658513f7120bb16d64d753d99205b456d83a1120a0a0d2ffdbf19f8c5 core/Version.h
+385b8ad96b59e2d67a6f7a78c6aa660a2f2cdad1bf8c7cadcd83387a3ad4996c core/VulkanAMF.h
+AMF_HEADERS
+
+# dav1d (fast AV1 decoder), built into the FFmpeg prefix
+if [ ! -f "$PREFIX/lib/pkgconfig/dav1d.pc" ]; then
+ rm -rf dav1d
+ git clone --depth 1 --branch "$DAV1D_TAG" https://code.videolan.org/videolan/dav1d.git
+ meson setup dav1d/build dav1d --prefix="$PREFIX" --libdir=lib --buildtype=release \
+ -Denable_tools=false -Denable_tests=false
+ ninja -C dav1d/build
+ sudo ninja -C dav1d/build install
+fi
+
+# libaom (AV1 encoding), built into the FFmpeg prefix
+if [ ! -f "$PREFIX/lib/pkgconfig/aom.pc" ]; then
+ rm -rf aom
+ git clone --depth 1 --branch "$AOM_TAG" https://aomedia.googlesource.com/aom
+ cmake -S aom -B aom/build-out -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+ -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=1 \
+ -DENABLE_TESTS=0 -DENABLE_EXAMPLES=0 -DENABLE_DOCS=0 -DENABLE_TOOLS=0
+ cmake --build aom/build-out --parallel "$JOBS"
+ sudo cmake --install aom/build-out
+fi
+
+if [ ! -d "ffmpeg-$FFMPEG_VERSION" ]; then
+ # GitHub mirror: ffmpeg.org is not always reachable from the runners
+ archive="FFmpeg-n$FFMPEG_VERSION.tar.gz"
+ curl -fsSL --retry 2 "https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n$FFMPEG_VERSION.tar.gz" -o "$archive"
+ printf '%s %s\n' "${FFMPEG_SHA256:-e0b04c4b43d7e6d67cb6710334fb513adf13ac860532f30e1d0ac4c231f232fb}" "$archive" | sha256sum -c -
+ tar xzf "$archive"
+ mv "FFmpeg-n$FFMPEG_VERSION" "ffmpeg-$FFMPEG_VERSION"
+fi
+cd "ffmpeg-$FFMPEG_VERSION"
+bash "$script_dir/../patch-ffmpeg-vulkan.sh" "$PWD"
+export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig"
+./configure --prefix="$PREFIX" --enable-gpl --enable-version3 \
+ --enable-shared --disable-static --disable-doc \
+ --extra-cflags="-I$VULKAN_PREFIX/include -I$PREFIX/include" \
+ --extra-ldflags="-L$PREFIX/lib" \
+ --enable-vulkan --enable-ffnvcodec --enable-nvenc --enable-nvdec --enable-cuvid \
+ --enable-libvpl --enable-amf \
+ --enable-vaapi --enable-vdpau --enable-libdrm --enable-opencl --enable-opengl \
+ --enable-libx264 --enable-libx265 --enable-libvpx --enable-libaom --enable-libdav1d \
+ --enable-libmp3lame --enable-libopus --enable-libvorbis --enable-libtheora \
+ --enable-libspeex --enable-libtwolame --enable-libshine \
+ --enable-libgsm --enable-libcodec2 --enable-libxvid --enable-libwebp \
+ --enable-libopenjpeg --enable-libsnappy --enable-libsoxr \
+ --enable-libass --enable-libfreetype --enable-libfontconfig --enable-libfribidi \
+ --enable-libxml2 --enable-libbluray --enable-libbs2b --enable-libcaca \
+ --enable-libcdio --enable-libflite --enable-libgme --enable-libmysofa \
+ --enable-libopenmpt --enable-librsvg --enable-librubberband --enable-libssh \
+ --enable-gnutls --enable-libvidstab --enable-libzmq --enable-libzvbi \
+ --enable-lv2 --enable-ladspa --enable-frei0r \
+ --enable-libpulse --enable-libjack --enable-openal --enable-sdl2 \
+ --enable-libdc1394 --enable-libiec61883
+make -j"$JOBS"
+sudo make install
+
+# Build env for GPU jobs: source this before cmake.
+sudo tee "$PREFIX/env.sh" >/dev/null < "$PREFIX/compiled-encoders.txt" 2>&1
+for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf h264_vaapi hevc_vaapi h264_vulkan hevc_vulkan; do
+ grep -Eq "[[:space:]]$codec[[:space:]]" "$PREFIX/compiled-encoders.txt" || { echo "Required encoder missing: $codec"; exit 1; }
+done
+echo "Done: source $PREFIX/env.sh"
diff --git a/ci/patch-ffmpeg-vulkan.sh b/ci/patch-ffmpeg-vulkan.sh
new file mode 100644
index 000000000..55b3faaea
--- /dev/null
+++ b/ci/patch-ffmpeg-vulkan.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Apply the FFmpeg 7.1 Vulkan host-import and staging-copy fixes.
+set -euo pipefail
+script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+source_dir=${1:?Specify the FFmpeg source directory}
+fix="$script_dir/patches/ffmpeg-vulkan-readback.patch"
+if patch --dry-run --reverse --silent -d "$source_dir" -p1 < "$fix" >/dev/null 2>&1; then
+ echo "FFmpeg Vulkan readback fixes already applied"
+else
+ patch --forward -d "$source_dir" -p1 < "$fix"
+fi
diff --git a/ci/patches/ffmpeg-vulkan-readback.patch b/ci/patches/ffmpeg-vulkan-readback.patch
new file mode 100644
index 000000000..2f4da0716
--- /dev/null
+++ b/ci/patches/ffmpeg-vulkan-readback.patch
@@ -0,0 +1,42 @@
+--- a/libavutil/hwcontext_vulkan.c
++++ b/libavutil/hwcontext_vulkan.c
+@@ -3813,13 +3813,15 @@
+ }
+ }
+
+- for (int i = 0; i < planes; i++)
+- av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset,
+- region[i].bufferRowLength,
+- swf->data[i],
+- swf->linesize[i],
+- swf->linesize[i],
+- region[i].imageExtent.height);
++ for (int i = 0; i < planes; i++) {
++ if (upload)
++ av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset, region[i].bufferRowLength,
++ swf->data[i], swf->linesize[i], swf->linesize[i], region[i].imageExtent.height);
++ else
++ av_image_copy_plane(swf->data[i], swf->linesize[i],
++ vkbuf->mapped_mem + region[i].bufferOffset, region[i].bufferRowLength,
++ swf->linesize[i], region[i].imageExtent.height);
++ }
+
+ if (!(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) && upload) {
+ ret = vk->FlushMappedMemoryRanges(hwctx->act_dev, 1,
+@@ -4079,7 +4081,15 @@
+ return AVERROR(EINVAL);
+
+ /* Setup buffers first */
+- if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY) {
++ /* AMD Linux host-pointer imports can fail submission with EFAULT and
++ * lose the device. Use owned staging memory for CPU transfers there.
++ * GPU-to-GPU decode/render/presentation/export paths are unchanged. */
++ int use_host_import = !!(p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY);
++#ifdef __linux__
++ if (p->props.properties.vendorID == 0x1002)
++ use_host_import = 0;
++#endif
++ if (use_host_import) {
+ err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload);
+ if (err >= 0)
+ host_mapped = 1;
diff --git a/ci/patches/ffmpeg-vulkan-readback.patch.license b/ci/patches/ffmpeg-vulkan-readback.patch.license
new file mode 100644
index 000000000..e8cd53c24
--- /dev/null
+++ b/ci/patches/ffmpeg-vulkan-readback.patch.license
@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC
+SPDX-License-Identifier: LGPL-3.0-or-later
diff --git a/ci/windows/setup-ffmpeg-windows-final.sh b/ci/windows/setup-ffmpeg-windows-final.sh
new file mode 100755
index 000000000..9ab7699a9
--- /dev/null
+++ b/ci/windows/setup-ffmpeg-windows-final.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Run using C:\msys64\usr\bin\bash.exe. No pacman changes.
+# Usage: bash setup-ffmpeg-windows.sh mingw64|mingw32
+set -euo pipefail
+script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+arch=${1:?Specify mingw64 or mingw32}
+case "$arch" in mingw64) cpu=x86_64 ;; mingw32) cpu=x86 ;; *) exit 2 ;; esac
+version=7.1.3
+root=/c/runner-setup
+prefix=${PREFIX:-$root/ffmpeg-$version-$arch}
+hw_prefix=${HW_PREFIX:-$root/hw-ffmpeg-$arch}
+work=${BUILD_DIR:-$root/build-$version-$arch}
+export PATH="$root/tools/bin:/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows"
+export PKG_CONFIG_LIBDIR="/$arch/lib/pkgconfig:/$arch/share/pkgconfig"
+export PKG_CONFIG_PATH="$hw_prefix/lib/pkgconfig:$prefix/lib/pkgconfig"
+mkdir -p "$root/logs" "$prefix" "$work"
+exec > >(tee -a "$root/logs/ffmpeg-$version-$arch.log") 2>&1
+trap 'echo "FAILED at line $LINENO"; exit 1' ERR
+echo "Building $version $arch at $(date)"
+gcc --version | head -1
+cd "$root"
+if [ ! -f "FFmpeg-n$version.tar.gz" ]; then
+ curl --cacert /usr/ssl/cert.pem -fL --retry 2 "https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n$version.tar.gz" -o "FFmpeg-n$version.tar.gz"
+fi
+printf '%s %s\n' e0b04c4b43d7e6d67cb6710334fb513adf13ac860532f30e1d0ac4c231f232fb "FFmpeg-n$version.tar.gz" | sha256sum -c -
+if [ ! -d "$root/FFmpeg-n$version" ]; then tar xzf "FFmpeg-n$version.tar.gz"; fi
+bash "$script_dir/../patch-ffmpeg-vulkan.sh" "$root/FFmpeg-n$version"
+cd "$work"
+opts=(--prefix="$prefix" --target-os=mingw32 --arch="$cpu"
+ --enable-gpl --enable-version3 --enable-shared --disable-static --disable-doc
+ --enable-ffnvcodec --enable-nvenc --enable-nvdec --enable-cuvid --enable-amf --enable-libvpl
+ --extra-cflags="-I$hw_prefix/include"
+ --enable-x86asm --x86asmexe="$root/tools/bin/nasm.exe" --enable-libdav1d --extra-ldflags="-L$prefix/lib")
+if [ "${WITH_VULKAN:-0}" = 1 ]; then
+ cflags="-I$hw_prefix/include -I$root/vulkan-$arch/include"
+ if [ "$arch" = mingw32 ]; then cflags="$cflags -msse2 -mfpmath=sse"; fi
+ opts+=(--enable-vulkan --extra-cflags="$cflags")
+else
+ opts+=(--disable-vulkan)
+fi
+# Enable the existing codec libraries explicitly; configure reports incompatible ones.
+for pair in libx264:x264 libx265:x265 libvpx:vpx libopus:opus libvorbis:vorbis libtheora:theoraenc libmp3lame:lame libwebp:libwebp libopenjpeg:libopenjp2 libass:libass libfreetype:freetype2 libfontconfig:fontconfig libfribidi:fribidi libsoxr:soxr; do
+ feature=${pair%%:*}; package=${pair#*:}
+ if pkg-config --exists "$package"; then opts+=("--enable-$feature"); fi
+done
+"$root/FFmpeg-n$version/configure" "${opts[@]}"
+make -j"${JOBS:-6}"
+make install
+# The oneVPL dispatcher is an application dependency; vendor GPU drivers remain system supplied.
+cp "$hw_prefix/bin/libvpl.dll" "$prefix/bin/"
+mkdir -p "$prefix/share/vpl/licensing"
+cp "$hw_prefix/share/vpl/licensing/"* "$prefix/share/vpl/licensing/"
+export PATH="$prefix/bin:$hw_prefix/bin:$PATH"
+"$prefix/bin/ffmpeg.exe" -version
+"$prefix/bin/ffmpeg.exe" -hide_banner -f lavfi -i testsrc2=size=128x96:rate=5 -t 1 -c:v mpeg4 -y "$prefix/smoke-test.avi"
+"$prefix/bin/ffprobe.exe" -v error -show_entries stream=codec_name,width,height "$prefix/smoke-test.avi"
+cat > "$prefix/env.sh" < "$prefix/compiled-encoders.txt" 2>&1
+for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf; do
+ grep -Eq "[[:space:]]$codec[[:space:]]" "$prefix/compiled-encoders.txt" || { echo "Missing required encoder: $codec"; exit 1; }
+done
+cp config.h config_components.h "$prefix/"
+echo "SUCCESS: $prefix includes NVIDIA NVENC, Intel QSV, and AMD AMF"
diff --git a/ci/windows/setup-ffmpeg-windows.sh b/ci/windows/setup-ffmpeg-windows.sh
new file mode 100755
index 000000000..c915b545b
--- /dev/null
+++ b/ci/windows/setup-ffmpeg-windows.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC
+# SPDX-License-Identifier: LGPL-3.0-or-later
+set -euo pipefail
+arch=${1:?Specify mingw64 or mingw32}
+export PATH="/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows"
+export WITH_VULKAN=${WITH_VULKAN:-1}
+bash "$(dirname "$0")/setup-hw-ffmpeg-windows.sh" "$arch"
+exec bash "$(dirname "$0")/setup-ffmpeg-windows-final.sh" "$@"
diff --git a/ci/windows/setup-hw-ffmpeg-windows.sh b/ci/windows/setup-hw-ffmpeg-windows.sh
new file mode 100755
index 000000000..37592c7b2
--- /dev/null
+++ b/ci/windows/setup-hw-ffmpeg-windows.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Build pinned hardware encoder dependencies without changing the installed toolchain.
+# Run with C:\msys64\usr\bin\bash.exe setup-hw-ffmpeg-windows.sh mingw64|mingw32.
+set -euo pipefail
+arch=${1:?Specify mingw64 or mingw32}
+case "$arch" in mingw64|mingw32) ;; *) exit 2 ;; esac
+root=/c/runner-setup
+prefix=${HW_PREFIX:-$root/hw-ffmpeg-$arch}
+export PATH="/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows"
+mkdir -p "$prefix" "$root/logs"
+exec > >(tee -a "$root/logs/hw-ffmpeg-$arch.log") 2>&1
+trap 'echo "FAILED at line $LINENO"' ERR
+fetch() {
+ local name=$1 url=$2 digest=$3
+ if [ ! -f "$root/$name" ]; then curl --cacert /usr/ssl/cert.pem -fL --retry 2 "$url" -o "$root/$name.part"; mv "$root/$name.part" "$root/$name"; fi
+ printf '%s %s\n' "$digest" "$root/$name" | sha256sum -c -
+}
+fetch nv-codec-headers-n12.1.14.0.tar.gz https://github.com/FFmpeg/nv-codec-headers/archive/refs/tags/n12.1.14.0.tar.gz 2fefaa227d2a3b4170797796425a59d1dd2ed5fd231db9b4244468ba327acd0b
+fetch libvpl-2.10.2.tar.gz https://github.com/intel/libvpl/archive/refs/tags/v2.10.2.tar.gz ad956ea7ecf14614325f59dfb44cc5ba08e2fcac373342d61c7db152ac651253
+for name in nv-codec-headers-n12.1.14.0 libvpl-2.10.2; do
+ source="$root/hw-sources/$name"
+ if [ ! -d "$source" ]; then mkdir -p "$source"; tar xzf "$root/$name.tar.gz" --strip-components=1 -C "$source"; fi
+done
+make -C "$root/hw-sources/nv-codec-headers-n12.1.14.0" install PREFIX="$prefix"
+cmake -G 'MinGW Makefiles' -S "$root/hw-sources/libvpl-2.10.2" -B "$root/hw-vpl-build-$arch" \
+ -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(cygpath -m "$prefix")" \
+ -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DBUILD_SHARED_LIBS=ON -DBUILD_TESTS=OFF \
+ -DBUILD_TOOLS=OFF -DBUILD_EXAMPLES=OFF -DINSTALL_EXAMPLE_CODE=OFF \
+ -DBUILD_DISPATCHER_ONEVPL_EXPERIMENTAL=OFF
+cmake --build "$root/hw-vpl-build-$arch" --parallel "${JOBS:-6}"
+cmake --install "$root/hw-vpl-build-$arch"
+# oneVPL installs Win32 DLLs under bin/x86; applications need the matching
+# dispatcher directly beside FFmpeg, just like the x64 build.
+if [ "$arch" = mingw32 ] && [ -f "$prefix/bin/x86/libvpl.dll" ]; then
+ cp "$prefix/bin/x86/libvpl.dll" "$prefix/bin/libvpl.dll"
+fi
+test -f "$prefix/bin/libvpl.dll"
+# AMD AMF 1.4.35 headers, pinned to the release commit. No SDK binaries needed.
+amf="$prefix/include/AMF"
+mkdir -p "$amf"
+while read -r digest header; do
+ mkdir -p "$amf/$(dirname "$header")"
+ if ! printf '%s %s\n' "$digest" "$amf/$header" | sha256sum -c - >/dev/null 2>&1; then
+ curl --cacert /usr/ssl/cert.pem -fL --retry 2 \
+ "https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/AMF/ba07d1b3c7c7ee0d846125cffc1d6d78a020c45a/amf/public/include/$header" -o "$amf/$header"
+ fi
+ printf '%s %s\n' "$digest" "$amf/$header" | sha256sum -c -
+done <<'AMF_HEADERS'
+70afd5279be90e656c2ec270229c5a702a11435a670052370e96b270e6b7640f components/AMFXInput.h
+27770ae77d2f7a16eca9ab1f8b9ce87cf0c0350425b950bdd39189c2c91a519a components/Ambisonic2SRenderer.h
+05a9ebc32217e69b6c4659f4744dbda9e5a2792f3e56bc1bcf61c15dcfd46cab components/AudioCapture.h
+bbe91a1e744685a36269d3b6af9cd36d9716ee748c8d0ed95255a6a42b46892f components/Capture.h
+a3a89490d44adaffc2fa2cccc14313f8f65e5af7a6431d89d85bd80e36c628a1 components/ChromaKey.h
+2f6887d14516ebdcf034d59e39235f5d9974533ddb0b04d685c0c3b6d397d3d6 components/ColorSpace.h
+f3f18680d55a65fdf5e0351d3fde1dcd9bdd2d6caec268c8c915c4227ffa1edc components/Component.h
+2479eb2a6174a1b63552c509abac1b701eddcab5a5eac5a7f4d9f40274d588b4 components/ComponentCaps.h
+7c22337ff313320bd7fe20ef0fb00756735012367a5daa7c621725030f578863 components/CursorCapture.h
+693c8e491f90eaffd226c1f6b2da37d3a5ef1d3c610e554647f7d0b699832727 components/DisplayCapture.h
+9dde7e850c8bdecda82b0cee1b331294f6cc91b32b1d1504344a023cd4dda8af components/FFMPEGAudioConverter.h
+92f50d789bc18d5ebd143b2925e124594a89110eef7efaa8189eb063bf2f3aa8 components/FFMPEGAudioDecoder.h
+2493067de9b3f540ffaae8a6466e57ef75af4772f037a81785b11ee9511029f5 components/FFMPEGAudioEncoder.h
+87a9ae9480c3af6210b1282d1768e36673fcc1ef4970afa6edaa74481cf0831b components/FFMPEGComponents.h
+5d64b7d2d50ca17ba601885dcdb51bb16ef4dccf88323437967e8524f26194bc components/FFMPEGEncoderAV1.h
+3dae38db05bc55af525000c28559ff76625e0cddc2ad2c4c2af52eab69e915bc components/FFMPEGEncoderH264.h
+3ad5cc5d3284b253a671499efa29c7a46b83a6e6373aad7142c7adcbfae31875 components/FFMPEGEncoderHEVC.h
+be992e14086b04e2a7b5503471a6c262c58fda163c45ad6d00f8e425d9f4a820 components/FFMPEGFileDemuxer.h
+3fefbe1aa7aaa4c000a97efb2e5ba26dc87ec15d11b07ceb0f43c768a1d589a2 components/FFMPEGFileMuxer.h
+297b0ab2291baeb44f30f03ffbf791a5603289ab34f625d60a33150f2be1f7b6 components/FFMPEGVideoDecoder.h
+ce5fb69c0546dd08dff0ee680972c5ec5a0b2d9b95e509338358eb2827f30221 components/FRC.h
+b63e856b55db7a6853f444a6d4d32cfe006d9e0fb1d9be1b3079557534738591 components/HQScaler.h
+c4e168d29e3f7d9318435958c3c784ad4c497159a617bd7a5598a2951b1b15f6 components/MediaSource.h
+dae3564f20ef5db959d31dfc74c226453b039f4919ddb65027a183cdae95c171 components/PreAnalysis.h
+5a01d71e128cf8bcaa4945fe76d9aad0156c409df3b25616a75645ef15972dc3 components/PreProcessing.h
+550900e56c8254bf9c42d1b1d6538a390f7f7d3bf4826ee385b019585cb6cd22 components/SupportedCodecs.h
+146e7cd35acc846f6b75b696bbe13a5d790f885bc23d9913e7de66f515da789b components/VQEnhancer.h
+08f0162a631833f847d1b4b751e41e28b147a79ed19545b43631222d22e74507 components/VideoCapture.h
+b0e173fbfe920a4923f461c1bffc73d1cbc06d54b676647dcd6ec300a0beee65 components/VideoConverter.h
+588622f7d945c85c1ed334b497174fd46031e01664338bcf59c4b242920df886 components/VideoDecoderUVD.h
+0b43ba9a0bb0d701c117e038d07270907d840ea2d2b13655fec71eda5e96d562 components/VideoEncoderAV1.h
+b7827d2c3380a32cef350cf0f180802694f9f93db295f1c8e08238160d17c3d8 components/VideoEncoderHEVC.h
+d106ce15028a9ea42e7299cff4b1a8697614a746b96740c9251f9b943bac2edc components/VideoEncoderVCE.h
+9bc2d8477d67693b0cd4d433fbc6a5812277190f67be5f188f0206b463f256ec components/VideoStitch.h
+3c6e6515a8fec111eb1847e349124a8c66c1e78be2d148bc0b8f85b9d9994e2d components/ZCamLiveStream.h
+c9382c911fd3b5e12105c1d2a67eb11e62e9475068825222a285bb1314ac0ddd core/AudioBuffer.h
+1ad18d3c813c3f81a836d393bbe86dfc0b7a1d7f04a77a272eb1461ce85ae3a4 core/Buffer.h
+ef5721466fe48e1c038151018699519761e6e351e66e76a5de65386924653e20 core/Compute.h
+50acf69fb670bc6cf093384d1b698bf9e44cda8dd49e02ef24eacfb86ad8ccae core/ComputeFactory.h
+f9b4acb784038aad234ddec490e5f6ad6dacc4b14794eb84f3523d56dc3d6dd2 core/Context.h
+76d16ba57d48ae9e457f3d904698a1e7c625007ac8887db9bcecdb90fd36d835 core/CurrentTime.h
+f05771305598b534e1efa0dd28825267ea87418c477b198e373f6764d060c666 core/D3D12AMF.h
+eaac4d80dd3215638c70f8a4ca01791d26d9df77586f665a772bb51907a8035b core/Data.h
+56666589bf6ad20c6286a048da751c2d7722f70c73d962ee3c205a6d709b6aff core/Debug.h
+a5e0fa9290cae767a4d135962ee5e30302e65e8d0ca5e016c620e72b759c86c6 core/Dump.h
+9ba2ebe93af628bce621f29de493cddab87bf8a030eed504b25fcdc822c61b44 core/Factory.h
+304f6d6077e9b0733859d44a5e43a343ae374c8fe72c7d79741c0aca89c21162 core/Interface.h
+5ad5165e9de2afcfc2da828e8704c838b1f82e8ead7422fbcfc32c758672e1f3 core/Plane.h
+59a4d11ecd150af25ee0b8d1d15f2efdfe5cf1060b8e8ab539f6a46d4bff8bc8 core/Platform.h
+0059b787e78cd7c4e57212ccaab56ad7e17e9474a31337da0eac5b2d056a5dd8 core/PropertyStorage.h
+871136e46586baa1fc77121935b503aaa16523d006c3e435ce50698605675fad core/PropertyStorageEx.h
+a469ff33875484b5762199da757819056ff4a7535bfdcec1f4cf2ffc403b98f2 core/Result.h
+7f0c99bbfe156169440031f75b0e37c28c435dd140ecb4e5089d0b1f01912c47 core/Surface.h
+ae8c5fee671f02b90d3ccab312cfa6f74424351f070a7dd4ed7aba437fd55c30 core/Trace.h
+72841e751595a2a8f62c8ac4a3e630b181411af8e0741d716bbe3379c9bf9a99 core/Variant.h
+e26e25e658513f7120bb16d64d753d99205b456d83a1120a0a0d2ffdbf19f8c5 core/Version.h
+385b8ad96b59e2d67a6f7a78c6aa660a2f2cdad1bf8c7cadcd83387a3ad4996c core/VulkanAMF.h
+AMF_HEADERS
+export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
+pkg-config --modversion ffnvcodec vpl
+echo "SUCCESS: hardware encoder dependencies installed in $prefix"
diff --git a/ci/windows/smoke-ffmpeg-windows.sh b/ci/windows/smoke-ffmpeg-windows.sh
new file mode 100755
index 000000000..a98357e1c
--- /dev/null
+++ b/ci/windows/smoke-ffmpeg-windows.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC
+# SPDX-License-Identifier: LGPL-3.0-or-later
+set -euo pipefail
+arch=${1:?mingw64 or mingw32}
+prefix=${PREFIX:-/c/runner-setup/ffmpeg-7.1.3-$arch}
+export PATH="$prefix/bin:/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows"
+exec > >(tee "/c/runner-setup/logs/ffmpeg-smoke-$arch.log") 2>&1
+"$prefix/bin/ffmpeg.exe" -version
+"$prefix/bin/ffmpeg.exe" -hide_banner -hwaccels | grep vulkan
+"$prefix/bin/ffmpeg.exe" -hide_banner -encoders > "$prefix/compiled-encoders.txt" 2>&1
+for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf; do
+ grep -Eq "[[:space:]]$codec[[:space:]]" "$prefix/compiled-encoders.txt"
+done
+"$prefix/bin/ffmpeg.exe" -hide_banner -devices > "$prefix/compiled-devices.txt" 2>&1
+grep -q gdigrab "$prefix/compiled-devices.txt"
+grep -q dshow "$prefix/compiled-devices.txt"
+"$prefix/bin/ffmpeg.exe" -hide_banner -f lavfi -i testsrc2=size=320x240:rate=24 -f lavfi -i sine=frequency=440:sample_rate=48000 -t 2 -c:v libx264 -preset fast -pix_fmt yuv420p -c:a aac -y "$prefix/h264-aac-test.mp4"
+"$prefix/bin/ffmpeg.exe" -v error -i "$prefix/h264-aac-test.mp4" -f null -
+"$prefix/bin/ffprobe.exe" -v error -show_entries stream=codec_name,width,height,sample_rate "$prefix/h264-aac-test.mp4"
+"$prefix/bin/ffmpeg.exe" -v error -i /c/runner-setup/libopenshot-vulkan-mvp/examples/test_video_sync.mp4 -frames:v 3 -f null -
+echo "SUCCESS: vendor encoders, screen/camera capture, Vulkan compiled in; H.264/AAC encode and decode $arch"
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 12e05cb9f..6a92d7dc6 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -64,6 +64,10 @@ target_compile_definitions(openshot-example PRIVATE
target_link_libraries(openshot-example openshot)
add_executable(openshot-html-example ExampleHtml.cpp)
+add_executable(openshot-timeline-benchmark TimelineBenchmark.cpp)
+target_link_libraries(openshot-timeline-benchmark openshot)
+add_executable(openshot-export-benchmark TimelineExportBenchmark.cpp)
+target_link_libraries(openshot-export-benchmark openshot)
target_link_libraries(openshot-html-example openshot Qt${QT_VERSION_MAJOR}::Gui)
if(ENABLE_VULKAN_BENCHMARK)
diff --git a/examples/TimelineBenchmark.cpp b/examples/TimelineBenchmark.cpp
new file mode 100644
index 000000000..b53324b37
--- /dev/null
+++ b/examples/TimelineBenchmark.cpp
@@ -0,0 +1,59 @@
+// Copyright (c) 2026 OpenShot Studios, LLC
+// SPDX-License-Identifier: LGPL-3.0-or-later
+#include "Clip.h"
+#include "Timeline.h"
+#include
+#include
+#include
+#include
+#include
+
+// Same workload for CPU and experimental GPU builds. Readback is optional so
+// processing and the existing QImage presentation boundary can be distinguished.
+int main(int argc, char** argv) {
+ if (argc < 2 || argc > 6) {
+ std::cerr << "Usage: openshot-timeline-benchmark VIDEO [FRAMES=120] [LAYERS=1] [READBACK=0] [DUMP.png]\n";
+ return 2;
+ }
+ try {
+ const int count = argc > 2 ? std::stoi(argv[2]) : 120;
+ const int layers = argc > 3 ? std::stoi(argv[3]) : 1;
+ const bool readback = argc > 4 && std::stoi(argv[4]) != 0;
+ if (count < 1 || layers < 1 || layers > 16) throw std::invalid_argument("Invalid frame/layer count");
+ std::vector> clips;
+ openshot::Timeline timeline(3840, 2160, openshot::Fraction(24, 1), 48000, 2, openshot::LAYOUT_STEREO);
+ timeline.SetMaxSize(854, 480);
+ for (int i = 0; i < layers; ++i) {
+ auto clip = std::make_unique(argv[1]);
+ clip->Layer(i);
+ if (i) {
+ clip->scale_x = openshot::Keyframe(0.7);
+ clip->scale_y = openshot::Keyframe(0.7);
+ clip->rotation = openshot::Keyframe(12.0 * i);
+ clip->alpha = openshot::Keyframe(0.6);
+ }
+ timeline.AddClip(clip.get());
+ clips.push_back(std::move(clip));
+ }
+ timeline.Open();
+ int deferred = 0;
+ std::shared_ptr last;
+ const auto start = std::chrono::steady_clock::now();
+ for (int i = 1; i <= count; ++i) {
+ last = timeline.GetFrame(i);
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ deferred += bool(last->GetDeferredImage());
+#endif
+ if (readback) last->GetImage();
+ }
+ const double seconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count();
+ std::cout << "frames=" << count << " layers=" << layers << " readback=" << readback
+ << " deferred_frames=" << deferred << " seconds=" << seconds
+ << " fps=" << count / seconds << std::endl;
+ if (argc > 5 && !last->GetImage()->save(argv[5])) throw std::runtime_error("Dump failed");
+ timeline.Close();
+ } catch (const std::exception& error) {
+ std::cerr << error.what() << '\n';
+ return 1;
+ }
+}
diff --git a/examples/TimelineExportBenchmark.cpp b/examples/TimelineExportBenchmark.cpp
new file mode 100644
index 000000000..3739de5ba
--- /dev/null
+++ b/examples/TimelineExportBenchmark.cpp
@@ -0,0 +1,75 @@
+// Copyright (c) 2026 OpenShot Studios, LLC
+// SPDX-License-Identifier: LGPL-3.0-or-later
+#include "Clip.h"
+#include "Timeline.h"
+#include "FFmpegWriter.h"
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+#include "VulkanFrameImage.h"
+#endif
+#include
+#include
+#include
+#include
+
+// Identical full-resolution rendering + NVENC workload for CPU/GPU builds.
+// Optional audio input exercises timeline mixing and muxing. Output is replaced.
+int main(int argc,char** argv) {
+ if(argc<3 || argc>7){
+ std::cerr<<"Usage: openshot-export-benchmark VIDEO OUTPUT.mp4 [FRAMES=120] [LAYERS=2] [AUDIO|-] [CODEC=h264_nvenc]\n";
+ return 2;
+ }
+ try {
+ const int count=argc>3?std::stoi(argv[3]):120;
+ const int layers=argc>4?std::stoi(argv[4]):2;
+ if(count<1 || layers<1 || layers>16) throw std::invalid_argument("Invalid frame/layer count");
+ std::vector> clips;
+ openshot::Timeline timeline(3840,2160,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO);
+ timeline.SetMaxSize(3840,2160);
+ for(int index=0;index(argv[1]);
+ clip->Layer(index);
+ if(index){
+ clip->scale_x=openshot::Keyframe(0.7);
+ clip->scale_y=openshot::Keyframe(0.65);
+ clip->shear_x=openshot::Keyframe(0.1);
+ clip->rotation=openshot::Keyframe(12.0*index);
+ clip->alpha=openshot::Keyframe(0.6);
+ }
+ timeline.AddClip(clip.get());clips.push_back(std::move(clip));
+ }
+ if(argc>5 && std::string(argv[5])!="-"){
+ auto audio=std::make_unique(argv[5]);audio->Layer(-1);
+ // Do not render cover art embedded in an audio file.
+ audio->has_video=openshot::Keyframe(0.0);
+ timeline.AddClip(audio.get());clips.push_back(std::move(audio));
+ }
+ timeline.Open();
+ openshot::FFmpegWriter writer(argv[2]);
+ writer.SetVideoOptions(true,argc>6?argv[6]:"h264_nvenc",openshot::Fraction(24,1),3840,2160,openshot::Fraction(1,1),false,false,40000000);
+ writer.SetAudioOptions(true,"aac",48000,2,openshot::LAYOUT_STEREO,192000);
+ writer.Open();
+ int deferred=0;
+ std::shared_ptr last;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ const auto readbacks=openshot::VulkanFrameReadbackCount();
+#endif
+ const auto start=std::chrono::steady_clock::now();
+ for(int n=1;n<=count;++n){
+ last=timeline.GetFrame(n);
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ deferred+=bool(last->GetDeferredImage());
+#endif
+ writer.WriteFrame(last);
+ }
+ writer.Close();
+ const double seconds=std::chrono::duration(std::chrono::steady_clock::now()-start).count();
+ std::cout<<"frames="<] [--overlay ] [--frames ]\n"
<< " [--mode ] [--vaapi-device ] [--row ]\n"
<< " [--dump-dir ] [--dump-every ]\n"
+ << " [--cpu-only | --vulkan-only] (select compositor)\n"
<< "Rows: CPU->CPU, CPU->Vk, VAAPI->CPU, VAAPI->Vk, CUDA->CPU, CUDA->Vk, Vulkan->CPU, VulkanDirect->Vk\n";
std::exit(0);
} else {
@@ -2312,7 +2313,9 @@ int main(int argc, char* argv[]) {
const QImage overlay_image = LoadOverlayImage(options);
std::vector results;
auto row_enabled = [&](const std::string& name) {
- return options.row_name.empty() || options.row_name == name;
+ const bool vulkan_composite = name.size() >= 4 && name.compare(name.size() - 4, 4, "->Vk") == 0;
+ return (vulkan_composite ? options.run_vulkan : options.run_cpu) &&
+ (options.row_name.empty() || options.row_name == name);
};
auto run_row = [&](BenchmarkResult (*fn)(const BenchmarkOptions&, const QImage&),
const std::string& fallback_name) {
@@ -2393,7 +2396,7 @@ int main(int argc, char* argv[]) {
add_unavailable_row("VulkanDirect->Vk", "Vulkan runtime unavailable");
}
if (results.empty())
- throw std::runtime_error("Unknown row name: " + options.row_name);
+ throw std::runtime_error("No rows selected: check --row and compositor filters");
for (const auto& result : results) {
result.stats.Print(result.name + " / " + result.decode_backend + " / " + result.composite_backend);
@@ -2406,6 +2409,10 @@ int main(int argc, char* argv[]) {
std::cout << "\nMatrix Summary\n";
PrintResultTable(results);
+ const bool processed_frames = std::any_of(results.begin(), results.end(),
+ [](const BenchmarkResult& result) { return !result.skipped && result.stats.frames > 0; });
+ if (!processed_frames)
+ throw std::runtime_error("No benchmark row successfully processed frames");
return 0;
} catch (const std::exception& exc) {
std::cerr << "Benchmark failed: " << exc.what() << "\n";
diff --git a/examples/build-render-backend.sh b/examples/build-render-backend.sh
new file mode 100644
index 000000000..4e58dec3a
--- /dev/null
+++ b/examples/build-render-backend.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Build isolated CPU/Vulkan experiments without replacing installed libraries.
+set -euo pipefail
+
+backend=${OPENSHOT_RENDER_BACKEND:-cpu}
+case "$backend" in
+ cpu) vulkan=OFF; target=openshot-benchmark ;;
+ vulkan) vulkan=ON; target=openshot-vulkan-benchmark ;;
+ *) echo 'OPENSHOT_RENDER_BACKEND must be cpu or vulkan' >&2; exit 2 ;;
+esac
+
+source_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
+build_dir=${OPENSHOT_BUILD_DIR:-"$source_dir/build-mvp-$backend"}
+
+# Additional arguments are CMake dependency/toolchain settings. Backend and
+# Release settings below take precedence so the two experiments stay comparable.
+cmake -S "$source_dir" -B "$build_dir" "$@" \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DENABLE_VULKAN_BENCHMARK="$vulkan" \
+ -DENABLE_GPU_FRAME="$vulkan" \
+ -DENABLE_PYTHON=ON \
+ -DBUILD_TESTING=ON
+cmake --build "$build_dir" --target "$target" openshot-timeline-benchmark pyopenshot --parallel "${OPENSHOT_BUILD_JOBS:-4}"
+
+if [[ "$backend" == vulkan ]]; then
+ echo "Built experimental Vulkan benchmark: $build_dir/examples/$target"
+ echo 'Experimental GPU library enabled; verify GPU residency and fallbacks with the timeline benchmark.'
+else
+ echo "Built CPU library and benchmark: $build_dir/tests/$target"
+fi
diff --git a/examples/color-chart-hlg.mp4 b/examples/color-chart-hlg.mp4
new file mode 100644
index 000000000..cd8183092
Binary files /dev/null and b/examples/color-chart-hlg.mp4 differ
diff --git a/examples/color-chart-pq.mp4 b/examples/color-chart-pq.mp4
new file mode 100644
index 000000000..ae79235d4
Binary files /dev/null and b/examples/color-chart-pq.mp4 differ
diff --git a/examples/color-chart-sdr-bt709.mp4 b/examples/color-chart-sdr-bt709.mp4
new file mode 100644
index 000000000..2dfff15f9
Binary files /dev/null and b/examples/color-chart-sdr-bt709.mp4 differ
diff --git a/examples/run-timeline-benchmarks.py b/examples/run-timeline-benchmarks.py
new file mode 100644
index 000000000..a3fffc862
--- /dev/null
+++ b/examples/run-timeline-benchmarks.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC
+# SPDX-License-Identifier: LGPL-3.0-or-later
+"""
+Run comparable CPU/GPU timeline workloads sequentially and retain raw evidence.
+
+Example (inherit the Vulkan ICD and FFmpeg library environment):
+ python3 examples/run-timeline-benchmarks.py --cpu-build build --vulkan-build build-vk \
+ --video /path/to/4k.mp4 --output /tmp/timeline-results.json
+"""
+import argparse
+import datetime
+import hashlib
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import shutil
+import statistics
+# Local developer tools use explicit argument lists, never a shell.
+import subprocess # nosec B404
+import sys
+
+RESULT = re.compile(
+ r"frames=(\d+) layers=(\d+) readback=(\d+) deferred_frames=(\d+) "
+ r"seconds=([\d.eE+-]+) fps=([\d.eE+-]+)"
+)
+
+
+def capture(command, timeout=20):
+ try:
+ # Developer-selected local tools run with argument lists, never a shell.
+ result = subprocess.run(command, text=True, stdout=subprocess.PIPE, # nosec B603
+ stderr=subprocess.STDOUT, timeout=timeout)
+ return {"command": command, "returncode": result.returncode, "output": result.stdout}
+ except (OSError, subprocess.TimeoutExpired) as error:
+ return {"command": command, "error": str(error)}
+
+
+def validate_result(row, frames, layers, readback):
+ """Reject incomplete runs and unexpected CPU fallback before recording FPS."""
+ match = RESULT.search(row.get("output", ""))
+ if row.get("returncode") == 0 and match:
+ got_frames, got_layers, got_readback, deferred, seconds, fps = match.groups()
+ row.update(frames=int(got_frames), deferred_frames=int(deferred),
+ seconds=float(seconds), fps=float(fps))
+ row["valid"] = (int(got_frames) == frames and int(got_layers) == layers
+ and int(got_readback) == readback and float(seconds) > 0
+ and (row["backend"] != "vulkan" or int(deferred) == frames))
+ if not row["valid"]:
+ row["error"] = "Incomplete workload or Vulkan CPU fallback detected"
+ else:
+ row["valid"] = False
+
+
+def verify_image(row, image_path, cpu_images, layers, readback):
+ """Compare a GPU result with the preceding CPU trial of the same workload."""
+ from PIL import Image, ImageChops, ImageStat
+ backend = row["backend"]
+ try:
+ with Image.open(image_path) as loaded:
+ pixels = loaded.convert("RGB")
+ means = ImageStat.Stat(pixels).mean
+ row["image"] = {"path": str(image_path), "size": pixels.size, "mean_rgb": means}
+ if backend == "cpu":
+ cpu_images[(layers, readback)] = pixels
+ else:
+ reference = cpu_images[(layers, readback)]
+ if pixels.size != reference.size:
+ raise ValueError("GPU image dimensions differ from CPU")
+ row["image"]["cpu_mae_rgb"] = ImageStat.Stat(ImageChops.difference(pixels, reference)).mean
+ reference_means = ImageStat.Stat(reference).mean
+ if sum(reference_means) > 3 and sum(means) < 0.1 * sum(reference_means):
+ raise ValueError("GPU image is black/near-black while matching CPU image is not")
+ except (OSError, ValueError) as error:
+ row["valid"] = False
+ row["error"] = str(error)
+
+
+def summarize(backend, layers, readback, successes):
+ """Keep failed trials out of the aggregate measurements."""
+ return {"backend": backend, "layers": layers, "readback": readback,
+ "valid_runs": len(successes),
+ "median_fps": statistics.median(successes) if successes else None,
+ "min_fps": min(successes) if successes else None,
+ "max_fps": max(successes) if successes else None}
+
+
+def measure(args, binaries, video, report, output):
+ """Run sequential trials, retaining CPU images for the GPU comparisons."""
+ def save():
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_text(json.dumps(report, indent=2) + "\n")
+
+ if args.verify_images:
+ image_dir = output.parent / (output.stem + "-frames")
+ image_dir.mkdir(parents=True, exist_ok=True)
+ cpu_images = {}
+ failed = False
+ for layers in (1, 2):
+ for readback in (0, 1):
+ for backend, binary in binaries.items():
+ successes = []
+ for trial in range(1, args.runs + 1):
+ command = [str(binary), str(video), str(args.frames), str(layers), str(readback)]
+ image_path = None
+ if args.verify_images:
+ image_path = image_dir / f"{backend}-layers{layers}-readback{readback}-run{trial}.png"
+ command.append(str(image_path))
+ evidence = capture(command, args.timeout)
+ row = {"backend": backend, "layers": layers, "readback": readback,
+ "trial": trial, **evidence}
+ validate_result(row, args.frames, layers, readback)
+ if row["valid"] and image_path:
+ verify_image(row, image_path, cpu_images, layers, readback)
+ failed |= not row["valid"]
+ if row["valid"]:
+ successes.append(row["fps"])
+ report["measurements"].append(row)
+ print(f"{backend} layers={layers} readback={readback} trial={trial}: "
+ f"{row.get('fps', 'no result')} FPS valid={row['valid']}", flush=True)
+ save()
+ report["summary"].append(summarize(backend, layers, readback, successes))
+ return failed
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--cpu-build", required=True, type=Path)
+ parser.add_argument("--vulkan-build", required=True, type=Path)
+ parser.add_argument("--video", required=True, type=Path)
+ parser.add_argument("--frames", default=120, type=int)
+ parser.add_argument("--runs", default=3, type=int)
+ parser.add_argument("--timeout", default=180, type=int)
+ parser.add_argument("--output", default=Path("timeline-results.json"), type=Path)
+ parser.add_argument("--note", default="", help="Record background services or other measurement conditions")
+ parser.add_argument("--verify-images", action="store_true", help="Save final-frame PNGs and compare CPU/GPU pixels (requires Pillow)")
+ args = parser.parse_args()
+ if min(args.frames, args.runs, args.timeout) < 1:
+ parser.error("frames, runs and timeout must be positive")
+ video = args.video.resolve(strict=True)
+ binaries = {name: (directory / "examples/openshot-timeline-benchmark").resolve(strict=True)
+ for name, directory in (("cpu", args.cpu_build), ("vulkan", args.vulkan_build))}
+ output = args.output.resolve()
+ if output == video or output in binaries.values():
+ parser.error("output must not overwrite an input file")
+ repo = Path(__file__).resolve().parent.parent
+ report = {
+ "started_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+ "machine": {"platform": platform.platform(), "processor": platform.processor(),
+ "logical_cpus": os.cpu_count()},
+ "video": {"path": str(video), "bytes": video.stat().st_size,
+ "mtime_ns": video.stat().st_mtime_ns},
+ "frames": args.frames, "runs": args.runs, "measurement_note": args.note,
+ "environment": {name: os.environ[name] for name in (
+ "VK_ICD_FILENAMES", "VK_DRIVER_FILES", "LD_LIBRARY_PATH", "QT_QPA_PLATFORM",
+ "OMP_NUM_THREADS") if name in os.environ},
+ "revision": capture(["git", "-C", str(repo), "rev-parse", "HEAD"]),
+ "working_tree": capture(["git", "-C", str(repo), "status", "--short"]),
+ "binaries": {}, "measurements": [], "summary": [],
+ "notes": "Each trial is a fresh process; includes initialization. Run without other GPU/CPU workloads. "
+ "Readback=0 excludes presentation; readback=1 includes QImage materialization, not Qt display. "
+ "Compare library versions below before attributing differences to the backend."
+ }
+ for name, binary in binaries.items():
+ report["binaries"][name] = {"path": str(binary),
+ "sha256": hashlib.sha256(binary.read_bytes()).hexdigest(),
+ "libraries": capture(["ldd", str(binary)])}
+ for name, command in (("gpu", ["nvidia-smi", "--query-gpu=name,driver_version,memory.total,memory.used,utilization.gpu", "--format=csv,noheader"]),
+ ("source", ["ffprobe", "-v", "error", "-show_streams", "-show_format", "-of", "json", str(video)])):
+ if shutil.which(command[0]):
+ report[name] = capture(command)
+
+ if shutil.which("nvidia-smi"):
+ report["gpu_processes"] = capture(["nvidia-smi", "--query-compute-apps=pid,process_name,used_gpu_memory", "--format=csv,noheader"])
+
+ failed = measure(args, binaries, video, report, output)
+ report["finished_utc"] = datetime.datetime.now(datetime.timezone.utc).isoformat()
+ report["valid"] = not failed
+ output.write_text(json.dumps(report, indent=2) + "\n")
+ print(f"Results: {output}")
+ return 1 if failed else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/src/AudioWaveformer.cpp b/src/AudioWaveformer.cpp
index 3c0a5eb10..90437b6f2 100644
--- a/src/AudioWaveformer.cpp
+++ b/src/AudioWaveformer.cpp
@@ -31,6 +31,36 @@ using namespace std;
using namespace openshot;
+namespace {
+// Waveform extraction reads every frame exactly once, front to back, so a large
+// reader cache only costs memory (and, because CacheMemory sizes are rescanned on
+// every insert, time). Keep readers owned by the waveformer nearly cache-free.
+constexpr int64_t WAVEFORM_READER_CACHE_BYTES = 2 * 1024 * 1024;
+
+void LimitWaveformReaderCache(ReaderBase* reader)
+{
+ if (!reader) {
+ return;
+ }
+ if (CacheBase* cache = reader->GetCache()) {
+ cache->SetMaxBytes(WAVEFORM_READER_CACHE_BYTES);
+ }
+}
+
+// Linearly interpolate a keyframe at a fractional (1-based) frame position
+double KeyframeValueAt(const Keyframe& keyframe, double frame)
+{
+ const double floor_frame = std::floor(frame);
+ const int64_t index = static_cast(floor_frame);
+ const double value = keyframe.GetValue(index);
+ const double fraction = frame - floor_frame;
+ if (fraction <= 0.0) {
+ return value;
+ }
+ return value + (keyframe.GetValue(index + 1) - value) * fraction;
+}
+}
+
// Default constructor
AudioWaveformer::AudioWaveformer(ReaderBase* new_reader) :
reader(new_reader),
@@ -81,9 +111,20 @@ AudioWaveformData AudioWaveformer::ExtractSamples(int channel, int num_per_secon
return base;
}
+// Extract the mixed envelope plus one envelope per channel, in a single decode pass
+AudioWaveformData AudioWaveformer::ExtractChannelSamples(int num_per_second, bool normalize) {
+ AudioWaveformData data;
+ if (!reader) {
+ return data;
+ }
+ ReaderBase* source = ResolveWaveformReader();
+ return ExtractSamplesFromReader(source, -1, num_per_second, normalize, true);
+}
+
AudioWaveformData AudioWaveformer::ExtractSamples(const std::string& path, int channel, int num_per_second, bool normalize) {
FFmpegReader temp_reader(path);
temp_reader.Open();
+ LimitWaveformReaderCache(&temp_reader);
// Disable video for speed
bool has_video = temp_reader.info.has_video;
temp_reader.info.has_video = false;
@@ -102,6 +143,7 @@ AudioWaveformData AudioWaveformer::ExtractSamples(const std::string& path,
bool normalize) {
FFmpegReader temp_reader(path);
temp_reader.Open();
+ LimitWaveformReaderCache(&temp_reader);
bool has_video = temp_reader.info.has_video;
temp_reader.info.has_video = false;
Fraction source_fps = temp_reader.info.fps;
@@ -121,96 +163,121 @@ AudioWaveformData AudioWaveformer::ApplyKeyframes(const AudioWaveformData& base,
int channel,
bool normalize) {
AudioWaveformData data;
- if (num_per_second <= 0) {
+ if (source_fps.ToDouble() <= 0.0) {
return data;
}
+ if (channel != -1 && (channel < 0 || channel >= source_channels)) {
+ return data;
+ }
+
+ data = ApplyKeyframes(base.max_samples, base.rms_samples, time_keyframe, volume_keyframe, project_fps, num_per_second);
+
+ if (normalize) {
+ float samples_max = 0.0f;
+ for (auto v : data.max_samples) {
+ samples_max = std::max(samples_max, std::abs(v));
+ }
+ if (samples_max > 0.0f) {
+ data.scale(static_cast(data.max_samples.size()), 1.0f / samples_max);
+ }
+ }
- double project_fps_value = project_fps.ToDouble();
- double source_fps_value = source_fps.ToDouble();
- if (project_fps_value <= 0.0 || source_fps_value <= 0.0) {
+ return data;
+}
+
+AudioWaveformData AudioWaveformer::ApplyKeyframes(const std::vector& max_samples,
+ const std::vector& rms_samples,
+ const Keyframe* time_keyframe,
+ const Keyframe* volume_keyframe,
+ const Fraction& project_fps,
+ int num_per_second) {
+ AudioWaveformData data;
+ if (num_per_second <= 0) {
return data;
}
- if (channel != -1 && (channel < 0 || channel >= source_channels)) {
+ const double project_fps_value = project_fps.ToDouble();
+ if (project_fps_value <= 0.0) {
return data;
}
- size_t base_total = base.max_samples.size();
+ const size_t base_total = max_samples.size();
if (base_total == 0) {
return data;
}
+ const size_t rms_total = std::min(rms_samples.size(), base_total);
- // Determine output duration from time curve (if any). Time curves are in project-frame domain.
- int64_t output_frames = 0;
- if (time_keyframe && time_keyframe->GetCount() > 0) {
- output_frames = time_keyframe->GetLength();
+ // A single point cannot remap time, and an empty volume curve has no gain to apply
+ if (time_keyframe && time_keyframe->GetCount() < 2) {
+ time_keyframe = nullptr;
}
- if (output_frames <= 0) {
- // Default to source duration derived from base waveform length
- double source_duration = static_cast(base_total) / static_cast(num_per_second);
- output_frames = static_cast(std::llround(source_duration * project_fps_value));
+ if (volume_keyframe && volume_keyframe->GetCount() < 1) {
+ volume_keyframe = nullptr;
}
- double output_duration_seconds = static_cast(output_frames) / project_fps_value;
- int total_samples = static_cast(std::ceil(output_duration_seconds * num_per_second));
+ // Output duration follows the time curve (project frames), otherwise the source envelope
+ int64_t total_samples = static_cast(base_total);
+ if (time_keyframe) {
+ const int64_t output_frames = time_keyframe->GetLength();
+ if (output_frames <= 0) {
+ return data;
+ }
+ const double output_duration_seconds = static_cast(output_frames) / project_fps_value;
+ total_samples = static_cast(std::ceil(output_duration_seconds * num_per_second));
+ }
if (total_samples <= 0) {
return data;
}
- data.resize(total_samples);
- data.zero(total_samples);
+ data.max_samples.assign(static_cast(total_samples), 0.0f);
+ data.rms_samples.assign(static_cast(total_samples), 0.0f);
+
+ const double frames_per_sample = project_fps_value / static_cast(num_per_second);
+ const double samples_per_frame = static_cast(num_per_second) / project_fps_value;
- for (int i = 0; i < total_samples; ++i) {
- double out_time = static_cast(i) / static_cast(num_per_second);
- // Time keyframes are defined in project-frame domain; evaluate using project frames
- double project_frame = out_time * project_fps_value;
- double mapped_project_frame = time_keyframe ? time_keyframe->GetValue(project_frame) : project_frame;
- // Convert mapped project frame to seconds (project FPS), then to waveform index
- double source_time = mapped_project_frame / project_fps_value;
- double source_index = source_time * static_cast(num_per_second);
+ for (int64_t i = 0; i < total_samples; ++i) {
+ // 1-based project frame at this output sample (keyframes use 1-based frame numbers)
+ const double project_frame = static_cast(i) * frames_per_sample + 1.0;
+
+ double source_index = static_cast(i);
+ if (time_keyframe) {
+ const double mapped_frame = KeyframeValueAt(*time_keyframe, project_frame);
+ source_index = (mapped_frame - 1.0) * samples_per_frame;
+ }
// Sample base waveform (nearest with simple linear blend)
- int idx0 = static_cast(std::floor(source_index));
- int idx1 = idx0 + 1;
- double frac = source_index - static_cast(idx0);
+ const int64_t idx0 = static_cast(std::floor(source_index));
+ const int64_t idx1 = idx0 + 1;
+ const double frac = source_index - static_cast(idx0);
float max_sample = 0.0f;
float rms_sample = 0.0f;
- if (idx0 >= 0 && idx0 < static_cast(base_total)) {
- max_sample = base.max_samples[idx0];
- rms_sample = base.rms_samples[idx0];
+ if (idx0 >= 0 && idx0 < static_cast(base_total)) {
+ max_sample = max_samples[idx0];
+ if (idx0 < static_cast(rms_total)) {
+ rms_sample = rms_samples[idx0];
+ }
}
- if (idx1 >= 0 && idx1 < static_cast(base_total)) {
- max_sample = static_cast((1.0 - frac) * max_sample + frac * base.max_samples[idx1]);
- rms_sample = static_cast((1.0 - frac) * rms_sample + frac * base.rms_samples[idx1]);
+ if (frac > 0.0 && idx1 >= 0 && idx1 < static_cast(base_total)) {
+ max_sample = static_cast((1.0 - frac) * max_sample + frac * max_samples[idx1]);
+ if (idx1 < static_cast(rms_total)) {
+ rms_sample = static_cast((1.0 - frac) * rms_sample + frac * rms_samples[idx1]);
+ }
}
double gain = 1.0;
if (volume_keyframe) {
- double project_frame = out_time * project_fps_value;
- gain = volume_keyframe->GetValue(project_frame);
+ gain = std::abs(KeyframeValueAt(*volume_keyframe, project_frame));
}
- max_sample = static_cast(max_sample * gain);
- rms_sample = static_cast(rms_sample * gain);
-
- data.max_samples[i] = max_sample;
- data.rms_samples[i] = rms_sample;
- }
- if (normalize) {
- float samples_max = 0.0f;
- for (auto v : data.max_samples) {
- samples_max = std::max(samples_max, std::abs(v));
- }
- if (samples_max > 0.0f) {
- data.scale(total_samples, 1.0f / samples_max);
- }
+ data.max_samples[i] = static_cast(max_sample * gain);
+ data.rms_samples[i] = static_cast(rms_sample * gain);
}
return data;
}
-AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_reader, int channel, int num_per_second, bool normalize) {
+AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_reader, int channel, int num_per_second, bool normalize, bool per_channel) {
AudioWaveformData data;
if (!source_reader || num_per_second <= 0) {
@@ -220,6 +287,9 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r
// Open reader (if needed)
if (!source_reader->IsOpen()) {
source_reader->Open();
+ if (source_reader == detached_reader.get()) {
+ LimitWaveformReaderCache(source_reader);
+ }
}
const auto retry_delay = std::chrono::milliseconds(100);
@@ -245,14 +315,10 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r
}
};
- int sample_rate = source_reader->info.sample_rate;
+ int64_t sample_rate = source_reader->info.sample_rate;
if (sample_rate <= 0) {
sample_rate = num_per_second;
}
- int sample_divisor = sample_rate / num_per_second;
- if (sample_divisor <= 0) {
- sample_divisor = 1;
- }
// Determine length of video frames (for waveform)
int64_t reader_video_length = source_reader->info.video_length;
@@ -276,16 +342,21 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r
return data;
}
- const bool known_duration = reader_duration > 0.0f && reader_video_length > 0;
- int total_samples = known_duration ? static_cast(std::ceil(reader_duration * num_per_second)) : 0;
- if (source_reader->info.channels == 0) {
+ const int reader_channels = source_reader->info.channels;
+ if (reader_channels <= 0) {
return data;
}
- if (channel != -1 && (channel < 0 || channel >= source_reader->info.channels)) {
+ if (channel != -1 && (channel < 0 || channel >= reader_channels)) {
return data;
}
+ // Per-channel envelopes describe every source channel, so they need all channels
+ per_channel = per_channel && channel == -1;
+
+ const bool known_duration = reader_duration > 0.0f && reader_video_length > 0;
+ const int total_samples = known_duration ? static_cast(std::ceil(reader_duration * num_per_second)) : 0;
+
// Resize and clear audio buffers when duration is known. Some audio-only
// formats, especially raw FLAC, may not expose a container duration even
// though frames are readable, so those are appended dynamically below.
@@ -296,104 +367,146 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r
data.max_samples.reserve(num_per_second);
data.rms_samples.reserve(num_per_second);
}
+ if (per_channel) {
+ data.channel_max_samples.assign(reader_channels, std::vector(known_duration ? total_samples : 0, 0.0f));
+ data.channel_rms_samples.assign(reader_channels, std::vector(known_duration ? total_samples : 0, 0.0f));
+ }
+
+ // Channels which contribute to the requested envelope
+ std::vector selected_channels;
+ for (int channel_index = 0; channel_index < reader_channels; channel_index++) {
+ if (channel == -1 || channel == channel_index) {
+ selected_channels.push_back(channel_index);
+ }
+ }
+
+ // Per-channel accumulators for the current chunk
+ std::vector chunk_max(reader_channels, 0.0f);
+ std::vector chunk_squared_sum(reader_channels, 0.0);
+ std::vector channel_samples(reader_channels, nullptr);
+
+ // Chunk k covers source samples [boundary(k - 1), boundary(k)). Boundaries are rounded
+ // from the exact rational position, so rates like 44100 / 200 = 220.5 never drift.
+ auto chunk_boundary = [&](int64_t chunk_index) -> int64_t {
+ return ((chunk_index + 1) * sample_rate + num_per_second / 2) / num_per_second;
+ };
int extracted_index = 0;
- int sample_index = 0;
+ int64_t samples_read = 0;
+ int64_t chunk_sample_count = 0;
+ int64_t next_boundary = std::max(1, chunk_boundary(0));
float samples_max = 0.0f;
- float chunk_max = 0.0f;
- double chunk_squared_sum = 0.0;
- int channel_count = (channel == -1) ? source_reader->info.channels : 1;
- std::vector channels(source_reader->info.channels, nullptr);
const double frame_rate = fps_value > 0.0 ? fps_value : 30.0;
const int64_t max_unknown_frames = static_cast(std::ceil(frame_rate * 60.0 * 60.0 * 12.0));
int empty_audio_frames = 0;
- auto append_sample = [&](float max_sample, float rms_sample) {
- if (known_duration) {
- if (extracted_index >= total_samples) {
- return;
- }
- data.max_samples[extracted_index] = max_sample;
- data.rms_samples[extracted_index] = rms_sample;
- } else {
- data.max_samples.push_back(max_sample);
- data.rms_samples.push_back(rms_sample);
- }
- samples_max = std::max(samples_max, max_sample);
- extracted_index++;
+ auto has_room = [&]() {
+ return !known_duration || extracted_index < total_samples;
};
- try {
- for (int64_t f = 1;
- (known_duration ? f <= reader_video_length && extracted_index < total_samples : f <= max_unknown_frames);
- f++) {
- std::shared_ptr frame = get_frame_with_retry(f);
- if (!known_duration && frame && frame->number < f) {
- break;
- }
- int sample_count = frame->GetAudioSamplesCount();
- if (sample_count <= 0) {
- if (!known_duration && extracted_index > 0 && ++empty_audio_frames >= 3) {
- break;
- }
- continue;
+ // Store the current chunk as one envelope sample, then reset the accumulators
+ auto emit_chunk = [&]() {
+ if (chunk_sample_count <= 0) {
+ return;
+ }
+ if (has_room()) {
+ float mixed_max = 0.0f;
+ double mixed_squared_sum = 0.0;
+ for (int channel_index : selected_channels) {
+ mixed_max = std::max(mixed_max, chunk_max[channel_index]);
+ mixed_squared_sum += chunk_squared_sum[channel_index];
}
- empty_audio_frames = 0;
-
- for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) {
- if (channel == channel_index || channel == -1) {
- channels[channel_index] = frame->GetAudioSamples(channel_index);
- }
+ const double mixed_count = static_cast(chunk_sample_count) * static_cast(selected_channels.size());
+ const float mixed_rms = static_cast(std::sqrt(mixed_squared_sum / mixed_count));
+
+ if (known_duration) {
+ data.max_samples[extracted_index] = mixed_max;
+ data.rms_samples[extracted_index] = mixed_rms;
+ } else {
+ data.max_samples.push_back(mixed_max);
+ data.rms_samples.push_back(mixed_rms);
}
-
- for (int s = 0; s < sample_count; s++) {
- for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) {
- if (channel == channel_index || channel == -1) {
- float *samples = channels[channel_index];
- if (!samples) {
- continue;
- }
- float abs_sample = std::abs(samples[s]);
- chunk_squared_sum += static_cast(samples[s]) * static_cast(samples[s]);
- chunk_max = std::max(chunk_max, abs_sample);
+ if (per_channel) {
+ for (int channel_index = 0; channel_index < reader_channels; channel_index++) {
+ const float channel_rms = static_cast(std::sqrt(
+ chunk_squared_sum[channel_index] / static_cast(chunk_sample_count)));
+ if (known_duration) {
+ data.channel_max_samples[channel_index][extracted_index] = chunk_max[channel_index];
+ data.channel_rms_samples[channel_index][extracted_index] = channel_rms;
+ } else {
+ data.channel_max_samples[channel_index].push_back(chunk_max[channel_index]);
+ data.channel_rms_samples[channel_index].push_back(channel_rms);
}
}
+ }
+ samples_max = std::max(samples_max, mixed_max);
+ extracted_index++;
+ }
- sample_index += 1;
+ std::fill(chunk_max.begin(), chunk_max.end(), 0.0f);
+ std::fill(chunk_squared_sum.begin(), chunk_squared_sum.end(), 0.0);
+ chunk_sample_count = 0;
+ next_boundary = std::max(samples_read + 1, chunk_boundary(extracted_index));
+ };
- if (sample_index % sample_divisor == 0) {
- float avg_squared_sum = 0.0f;
- if (channel_count > 0) {
- avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_divisor * channel_count));
- }
+ for (int64_t f = 1;
+ (known_duration ? f <= reader_video_length && extracted_index < total_samples : f <= max_unknown_frames);
+ f++) {
+ std::shared_ptr frame = get_frame_with_retry(f);
+ if (!frame) {
+ break;
+ }
+ if (!known_duration && frame->number < f) {
+ break;
+ }
+ const int sample_count = frame->GetAudioSamplesCount();
+ if (sample_count <= 0) {
+ if (!known_duration && extracted_index > 0 && ++empty_audio_frames >= 3) {
+ break;
+ }
+ continue;
+ }
+ empty_audio_frames = 0;
- if (!known_duration || extracted_index < total_samples) {
- append_sample(chunk_max, std::sqrt(avg_squared_sum));
- }
+ const int frame_channels = frame->GetAudioChannelsCount();
+ for (int channel_index : selected_channels) {
+ channel_samples[channel_index] = channel_index < frame_channels ? frame->GetAudioSamples(channel_index) : nullptr;
+ }
- sample_index = 0;
- chunk_max = 0.0f;
- chunk_squared_sum = 0.0;
+ int64_t offset = 0;
+ while (offset < sample_count) {
+ const int64_t span = std::min(sample_count - offset, next_boundary - samples_read);
+ for (int channel_index : selected_channels) {
+ const float* samples = channel_samples[channel_index];
+ if (!samples) {
+ continue;
+ }
+ float span_max = chunk_max[channel_index];
+ double span_squared_sum = chunk_squared_sum[channel_index];
+ for (int64_t s = offset; s < offset + span; s++) {
+ const float value = samples[s];
+ span_max = std::max(span_max, std::abs(value));
+ span_squared_sum += static_cast(value) * static_cast(value);
+ }
+ chunk_max[channel_index] = span_max;
+ chunk_squared_sum[channel_index] = span_squared_sum;
+ }
+ offset += span;
+ samples_read += span;
+ chunk_sample_count += span;
- if (known_duration && extracted_index >= total_samples) {
- break;
- }
+ if (samples_read >= next_boundary) {
+ emit_chunk();
+ if (!has_room()) {
+ break;
}
}
}
- } catch (...) {
- throw;
}
- if (sample_index > 0 && (!known_duration || extracted_index < total_samples)) {
- float avg_squared_sum = 0.0f;
- if (channel_count > 0) {
- avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_index * channel_count));
- }
-
- append_sample(chunk_max, std::sqrt(avg_squared_sum));
- }
+ // Keep the trailing partial chunk (end of the media)
+ emit_chunk();
if (normalize && samples_max > 0.0f) {
float scale = 1.0f / samples_max;
diff --git a/src/AudioWaveformer.h b/src/AudioWaveformer.h
index 73ed9566f..73c591fa0 100644
--- a/src/AudioWaveformer.h
+++ b/src/AudioWaveformer.h
@@ -35,6 +35,10 @@ namespace openshot {
std::vector max_samples;
std::vector rms_samples;
+ /// Optional per-channel envelopes, filled by ExtractChannelSamples()
+ std::vector> channel_max_samples;
+ std::vector> channel_rms_samples;
+
/// Resize both datasets
void resize(int total_samples) {
max_samples.resize(total_samples);
@@ -47,20 +51,30 @@ namespace openshot {
std::fill(rms_samples.begin(), rms_samples.end(), 0);
}
- /// Scale # of values by some factor
+ /// Scale # of values by some factor (including any per-channel envelopes)
void scale(int total_samples, float factor) {
for (auto s = 0; s < total_samples; s++) {
max_samples[s] *= factor;
rms_samples[s] *= factor;
}
+ for (auto& channel : channel_max_samples) {
+ for (auto& value : channel) value *= factor;
+ }
+ for (auto& channel : channel_rms_samples) {
+ for (auto& value : channel) value *= factor;
+ }
}
- /// Clear and free memory of both datasets
+ /// Clear and free memory of all datasets
void clear() {
max_samples.clear();
max_samples.shrink_to_fit();
rms_samples.clear();
rms_samples.shrink_to_fit();
+ channel_max_samples.clear();
+ channel_max_samples.shrink_to_fit();
+ channel_rms_samples.clear();
+ channel_rms_samples.shrink_to_fit();
}
/// Return a vector of vectors (containing both datasets)
@@ -70,6 +84,22 @@ namespace openshot {
output.push_back(rms_samples);
return output;
}
+
+ /// Number of per-channel envelopes available
+ int channel_count() const {
+ return static_cast(channel_max_samples.size());
+ }
+
+ /// Return the max and RMS envelopes of one channel (empty when unavailable)
+ std::vector> channel_vectors(int channel) const {
+ std::vector> output(2);
+ if (channel >= 0 && channel < channel_count()
+ && channel < static_cast(channel_rms_samples.size())) {
+ output[0] = channel_max_samples[channel];
+ output[1] = channel_rms_samples[channel];
+ }
+ return output;
+ }
};
/**
@@ -100,6 +130,16 @@ namespace openshot {
/// @brief Extract audio samples from a media file path (audio-only fast path)
AudioWaveformData ExtractSamples(const std::string& path, int channel, int num_per_second, bool normalize);
+ /// @brief Extract the mixed envelope and every channel's envelope in a single decode pass
+ ///
+ /// max_samples / rms_samples hold the all-channel envelope (the same values as
+ /// ExtractSamples(-1, ...)), and channel_max_samples / channel_rms_samples hold one
+ /// envelope per source channel (the same values as ExtractSamples(channel, ...)).
+ /// Clip keyframes are NOT applied, since per-channel envelopes describe the source media.
+ /// @param num_per_second How many samples per second to return
+ /// @param normalize Scale all envelopes by the largest mixed peak, so it becomes 1.0
+ AudioWaveformData ExtractChannelSamples(int num_per_second, bool normalize);
+
/// @brief Apply time and volume keyframes to an existing waveform data set
AudioWaveformData ApplyKeyframes(const AudioWaveformData& base,
const openshot::Keyframe* time_keyframe,
@@ -111,6 +151,17 @@ namespace openshot {
int channel,
bool normalize);
+ /// @brief Apply time and volume keyframes to existing max / RMS envelopes
+ ///
+ /// Keyframes are evaluated in the project-frame domain (1-based frames). Pass nullptr for
+ /// either keyframe to skip it. Time curves with fewer than 2 points are ignored.
+ AudioWaveformData ApplyKeyframes(const std::vector& max_samples,
+ const std::vector& rms_samples,
+ const openshot::Keyframe* time_keyframe,
+ const openshot::Keyframe* volume_keyframe,
+ const openshot::Fraction& project_fps,
+ int num_per_second);
+
/// @brief Convenience: extract then apply keyframes in one step from a file path
AudioWaveformData ExtractSamples(const std::string& path,
const openshot::Keyframe* time_keyframe,
@@ -124,7 +175,7 @@ namespace openshot {
~AudioWaveformer();
private:
- AudioWaveformData ExtractSamplesFromReader(openshot::ReaderBase* source_reader, int channel, int num_per_second, bool normalize);
+ AudioWaveformData ExtractSamplesFromReader(openshot::ReaderBase* source_reader, int channel, int num_per_second, bool normalize, bool per_channel = false);
openshot::ReaderBase* ResolveSourceReader(openshot::ReaderBase* source_reader);
openshot::Fraction ResolveSourceFPS(openshot::ReaderBase* source_reader);
openshot::ReaderBase* ResolveWaveformReader();
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 87867c6aa..eb3db458a 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -60,6 +60,7 @@ set(OPENSHOT_SOURCES
ChunkReader.cpp
ChunkWriter.cpp
Color.cpp
+ ColorManagement.cpp
Clip.cpp
ClipBase.cpp
Coordinate.cpp
@@ -185,6 +186,9 @@ set(CMAKE_MACOSX_RPATH 0)
############### CREATE LIBRARY #################
# Create shared openshot library
add_library(openshot SHARED)
+if(ENABLE_GPU_FRAME)
+ target_compile_definitions(openshot PUBLIC OPENSHOT_ENABLE_GPU_FRAME=1)
+endif()
target_sources(openshot PRIVATE
${OPENSHOT_SOURCES}
@@ -237,7 +241,7 @@ add_feature_info("Wayland screen capture" OPENSHOT_WAYLAND_CAPTURE "Use the opti
# Find JUCE-based openshot Audio libraries
if(NOT TARGET OpenShot::Audio)
# Only load if necessary (not for integrated builds)
- find_package(OpenShotAudio 1.0.2 REQUIRED)
+ find_package(OpenShotAudio 1.1.0 REQUIRED)
endif()
target_link_libraries(openshot PUBLIC OpenShot::Audio)
@@ -433,6 +437,8 @@ message(STATUS "Found Qt${QT_VERSION_MAJOR} ${Qt${QT_VERSION_MAJOR}Core_VERSION_
set(QT_VERSION_STR ${Qt${QT_VERSION_MAJOR}Core_VERSION_STRING} CACHE STRING "Qt version linked with" FORCE)
mark_as_advanced(QT_VERSION_STR)
+target_sources(openshot PRIVATE GpuMetrics.cpp)
+
################### FFMPEG #####################
# Find FFmpeg libraries (used for video encoding / decoding)
find_package(FFmpeg REQUIRED
@@ -441,6 +447,54 @@ find_package(FFmpeg REQUIRED
)
set(all_comps avcodec avdevice avformat avutil swscale)
+if(ENABLE_GPU_FRAME)
+ find_package(FFmpeg REQUIRED COMPONENTS avfilter)
+ list(APPEND all_comps avfilter)
+ target_sources(openshot PRIVATE VulkanFrameImage.cpp)
+ target_sources(openshot PRIVATE VulkanCompositor.cpp)
+ target_sources(openshot PRIVATE GpuDevices.cpp VulkanLoader.cpp)
+ if(APPLE)
+ enable_language(OBJCXX)
+ target_sources(openshot PRIVATE MacVulkanInterop.mm MacVulkanInterop.h Qt/VulkanOpenGLPresenterMac.mm Qt/VulkanOpenGLPresenter.h)
+ set_property(TARGET openshot PROPERTY OBJCXX_STANDARD 17)
+ target_link_libraries(openshot PRIVATE "-framework Foundation" "-framework CoreFoundation" objc "-framework Metal" "-framework CoreVideo" "-framework IOSurface" "-framework OpenGL")
+ else()
+ target_sources(openshot PRIVATE Qt/VulkanOpenGLPresenter.cpp Qt/VulkanOpenGLPresenter.h)
+ endif()
+ find_package(Vulkan REQUIRED)
+ find_program(GLSLANG_VALIDATOR glslangValidator)
+ if(NOT GLSLANG_VALIDATOR)
+ message(FATAL_ERROR "ENABLE_GPU_FRAME requires glslangValidator")
+ endif()
+ # Embed SPIR-V in the library so installed builds need no shader files.
+ set(VULKAN_SHADER_HEADERS)
+ file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/vulkan")
+ foreach(shader affine encode_nv12)
+ set(shader_header "${CMAKE_CURRENT_BINARY_DIR}/vulkan/${shader}.comp.h")
+ add_custom_command(OUTPUT "${shader_header}"
+ COMMAND "${GLSLANG_VALIDATOR}" -V -S comp --vn "openshot_${shader}_spv"
+ -o "${shader_header}" "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/${shader}.comp"
+ DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/${shader}.comp" VERBATIM)
+ list(APPEND VULKAN_SHADER_HEADERS "${shader_header}")
+ endforeach()
+ add_custom_target(openshot-vulkan-shaders DEPENDS ${VULKAN_SHADER_HEADERS})
+ add_dependencies(openshot openshot-vulkan-shaders)
+ target_include_directories(openshot PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/vulkan")
+ # Vulkan is loaded at runtime (VulkanLoader.cpp), so OpenShot starts without it.
+ # macOS links the MoltenVK it bundles; nothing to load there.
+ target_compile_definitions(openshot PRIVATE VK_NO_PROTOTYPES)
+ if(APPLE)
+ target_link_libraries(openshot PRIVATE Vulkan::Vulkan)
+ else()
+ target_include_directories(openshot PRIVATE ${Vulkan_INCLUDE_DIRS})
+ target_link_libraries(openshot PRIVATE ${CMAKE_DL_LIBS})
+ endif()
+ find_package(PkgConfig QUIET)
+ if(PKG_CONFIG_FOUND)
+ pkg_check_modules(VULKAN_FRAME_PC QUIET vulkan)
+ target_include_directories(openshot PRIVATE ${VULKAN_FRAME_PC_INCLUDE_DIRS})
+ endif()
+endif()
set(version_comps avcodec avformat avutil)
# Pick a resampler. Prefer swresample if possible
diff --git a/src/CacheMemory.cpp b/src/CacheMemory.cpp
index a89c9f7ee..1b7758e1f 100644
--- a/src/CacheMemory.cpp
+++ b/src/CacheMemory.cpp
@@ -255,7 +255,19 @@ void CacheMemory::CleanUp()
// Create a scoped lock, to protect the cache from multiple threads
const std::lock_guard lock(*cacheMutex);
- while (GetBytes() > max_bytes && frame_numbers.size() > 20)
+ size_t retained_frames = 20;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // A CPU-oriented twenty-frame floor can pin gigabytes of GPU memory
+ // across reader, mapper and timeline caches. Honor byte budgets for
+ // caches holding hardware surfaces while retaining the newest frame.
+ for (const auto& entry : frames) {
+ if (entry.second->GetDeferredImage()) {
+ retained_frames = 1;
+ break;
+ }
+ }
+#endif
+ while (GetBytes() > max_bytes && frame_numbers.size() > retained_frames)
{
// Get the oldest frame number.
int64_t frame_to_remove = frame_numbers.back();
diff --git a/src/Clip.cpp b/src/Clip.cpp
index 0fc043d87..712b050b5 100644
--- a/src/Clip.cpp
+++ b/src/Clip.cpp
@@ -10,6 +10,7 @@
//
// SPDX-License-Identifier: LGPL-3.0-or-later
+#include "GpuMetrics.h"
#include
#include "Clip.h"
@@ -17,6 +18,10 @@
#include "Exceptions.h"
#include "FFmpegReader.h"
#include "FrameMapper.h"
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+#include "VulkanCompositor.h"
+#include "VulkanFrameImage.h"
+#endif
#include "QtImageReader.h"
#include "QtTextReader.h"
#include "ChunkReader.h"
@@ -26,6 +31,7 @@
#include "effects/AudioVisualization.h"
#include
+#include
#include
#include
#include
@@ -503,12 +509,27 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun
// Get time mapped frame object (used to increase speed, change direction, etc...)
apply_timemapping(frame);
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // Audio-only/disabled-video layers have nothing to paint in the basic
+ // pipeline. Keep their audio, without manufacturing a CPU canvas.
+ if (options && (!reader->info.has_video || has_video.GetInt(frame->number) == 0) &&
+ !Waveform() && effects.empty() &&
+ (!timeline || static_cast(timeline)->Effects().empty()))
+ return frame;
+#endif
+
// Apply waveform image (if any)
apply_waveform(frame, timeline_size);
// Apply effects BEFORE applying keyframes (if any local or global effects are used)
apply_effects(frame, timeline_frame_number, options, true);
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // Timeline layers only need the composited result, so skip the separate transform pass
+ if (options && background_frame && apply_vulkan_layer(frame, background_frame, timeline_size))
+ return frame;
+#endif
+
// Apply keyframe / transforms to current clip image
apply_keyframes(frame, timeline_size);
@@ -773,6 +794,7 @@ std::shared_ptr Clip::GetOrCreateFrame(int64_t number, bool enable_time)
"number", number, "clip_frame_number", clip_frame_number);
// Attempt to get a frame (but this could fail if a reader has just been closed)
+ if (reader->info.has_single_image) gpu_metrics::Record(gpu_metrics::ImageSource);
auto reader_frame = reader->GetFrame(clip_frame_number);
if (reader_frame) {
// Override frame # (due to time-mapping might change it)
@@ -1352,6 +1374,37 @@ void Clip::RemoveEffect(EffectBase* effect)
void Clip::apply_background(std::shared_ptr frame,
std::shared_ptr background_frame,
bool update_frame_image) {
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // Audio-only frames contribute audio in Timeline::add_layer, but have no
+ // image to composite. Do not materialize the GPU canvas for an empty layer.
+ if (!frame->has_image_data) {
+ if (update_frame_image) {
+ if (auto background = background_frame->GetDeferredImage()) frame->AddDeferredImage(background);
+ else frame->AddImage(background_frame->GetImage());
+ }
+ return;
+ }
+ if (composite == COMPOSITE_SOURCE_OVER) {
+ auto source = frame->GetDeferredImage();
+ auto background = background_frame->GetDeferredImage();
+ QColor clear;
+ if (std::dynamic_pointer_cast(source) &&
+ (std::dynamic_pointer_cast(background) || background_frame->GetBlankColor(clear))) {
+ try {
+ auto result = RenderVulkanImage(source, background, background_frame->GetWidth(),
+ background_frame->GetHeight(), QTransform(), 1.0, clear);
+ gpu_metrics::Record(gpu_metrics::RenderGpu);
+ background_frame->AddDeferredImage(result);
+ if (update_frame_image) frame->AddDeferredImage(result);
+ return;
+ } catch (const std::exception& error) {
+ Logger::Instance()->Log(std::string("Vulkan composite CPU fallback: ") + error.what());
+ }
+ }
+ }
+#endif
+ gpu_metrics::Record(gpu_metrics::RenderCpu);
+ gpu_metrics::Reason gpu_reason("Composite");
// Add background canvas
std::shared_ptr background_canvas = background_frame->GetImage();
QPainter painter(background_canvas.get());
@@ -1374,9 +1427,13 @@ void Clip::apply_effects(std::shared_ptr frame, int64_t timeline_frame_nu
{
// Apply the effect to this frame
if (effect->info.apply_before_clip && before_keyframes) {
+ gpu_metrics::Reason gpu_reason("Effect");
effect->ProcessFrame(frame, frame->number);
+ if (effect->info.has_video) gpu_metrics::RecordRender(*frame);
} else if (!effect->info.apply_before_clip && !before_keyframes) {
+ gpu_metrics::Reason gpu_reason("Effect");
effect->ProcessFrame(frame, frame->number);
+ if (effect->info.has_video) gpu_metrics::RecordRender(*frame);
}
}
@@ -1394,6 +1451,58 @@ bool Clip::isNear(double a, double b)
return fabs(a - b) < 0.000001;
}
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+namespace {
+std::shared_ptr VulkanSource(const std::shared_ptr& frame) {
+ if (!VulkanRenderingEnabled()) return {};
+ auto source = std::dynamic_pointer_cast(frame->GetDeferredImage());
+ if (source || !frame->has_image_data) return source;
+ try {
+ auto uploaded = UploadVulkanImage(*frame->GetImage());
+ frame->AddDeferredImage(uploaded);
+ return std::dynamic_pointer_cast(uploaded);
+ } catch (const std::exception& error) {
+ // Log the first failure and then every 100th, so repeats stay visible
+ // without flooding the log; the total is in VulkanUploadFailureCount().
+ RecordVulkanUploadFailure();
+ const auto failures = VulkanUploadFailureCount();
+ if (failures == 1 || failures % 100 == 0)
+ Logger::Instance()->Log("GPU image upload unavailable; using CPU (" + std::to_string(failures) +
+ " failures): " + error.what(), Logger::LevelWarning);
+ return {};
+ }
+}
+}
+bool Clip::apply_vulkan_layer(std::shared_ptr frame, std::shared_ptr background_frame, QSize timeline_size)
+{
+ if (!frame->has_image_data || composite != COMPOSITE_SOURCE_OVER ||
+ corner_radius.GetValue(frame->number) != 0.0 || display != FRAME_DISPLAY_NONE)
+ return false;
+ // Effects applied after keyframes (and timeline transitions/masks) need the transformed layer
+ for (auto effect : effects)
+ if (!effect->info.apply_before_clip)
+ return false;
+ if (timeline && !static_cast(timeline)->Effects().empty())
+ return false;
+
+ auto source = VulkanSource(frame);
+ auto background = background_frame->GetDeferredImage();
+ QColor clear;
+ if (!source || !(std::dynamic_pointer_cast(background) || background_frame->GetBlankColor(clear)))
+ return false;
+ try {
+ background_frame->AddDeferredImage(RenderVulkanImage(source, background, timeline_size.width(),
+ timeline_size.height(), get_transform(frame, timeline_size.width(), timeline_size.height()),
+ alpha.GetValue(frame->number), clear));
+ gpu_metrics::Record(gpu_metrics::RenderGpu);
+ return true;
+ } catch (const std::exception& error) {
+ Logger::Instance()->Log(std::string("Vulkan layer CPU fallback: ") + error.what());
+ return false;
+ }
+}
+#endif
+
// Apply keyframes to the source frame (if any)
void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) {
// Skip out if video was disabled or only an audio frame (no visualisation in use)
@@ -1402,6 +1511,25 @@ void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) {
return;
}
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (composite == COMPOSITE_SOURCE_OVER &&
+ corner_radius.GetValue(frame->number) == 0.0 && display == FRAME_DISPLAY_NONE) {
+ if (auto source = VulkanSource(frame)) {
+ try {
+ frame->AddDeferredImage(RenderVulkanImage(source, nullptr, timeline_size.width(),
+ timeline_size.height(), get_transform(frame, timeline_size.width(), timeline_size.height()),
+ alpha.GetValue(frame->number)));
+ gpu_metrics::Record(gpu_metrics::RenderGpu);
+ return;
+ } catch (const std::exception& error) {
+ Logger::Instance()->Log(std::string("Vulkan transform CPU fallback: ") + error.what());
+ }
+ }
+ }
+#endif
+
+ gpu_metrics::Record(gpu_metrics::RenderCpu);
+ gpu_metrics::Reason gpu_reason("Transform");
// Get image from clip, and create transparent background image
std::shared_ptr source_image = frame->GetImage();
std::shared_ptr background_canvas = std::make_shared(timeline_size.width(),
@@ -1557,8 +1685,12 @@ QSize Clip::scale_size(QSize source_size, ScaleType source_scale, int target_wid
// Get QTransform from keyframes
QTransform Clip::get_transform(std::shared_ptr frame, int width, int height)
{
- // Get image from clip
- std::shared_ptr source_image = frame->GetImage();
+ // Transform geometry does not require CPU pixels for a deferred surface.
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ const QSize source_dimensions(frame->GetWidth(), frame->GetHeight());
+#else
+ const QSize source_dimensions = frame->GetImage()->size();
+#endif
const double margin_value = std::max(0.0, std::min(0.5, margin.GetValue(frame->number)));
const double margin_pixels = margin_value * std::min(width, height);
@@ -1568,7 +1700,7 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig
const double layout_height = std::max(1.0, height - (margin_pixels * 2.0));
/* RESIZE SOURCE IMAGE - based on scale type */
- QSize source_size = scale_size(source_image->size(), scale, layout_width, layout_height);
+ QSize source_size = scale_size(source_dimensions, scale, layout_width, layout_height);
// Text rasterizes extra detail for a scaled-up SCALE_NONE clip; draw it at its layout size
if (scale == SCALE_NONE) {
@@ -1758,8 +1890,8 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig
transform.translate(-origin_x_offset,-origin_y_offset);
}
// SCALE CLIP (if needed)
- float source_width_scale = (float(source_size.width()) / float(source_image->width())) * sx;
- float source_height_scale = (float(source_size.height()) / float(source_image->height())) * sy;
+ float source_width_scale = (float(source_size.width()) / float(source_dimensions.width())) * sx;
+ float source_height_scale = (float(source_size.height()) / float(source_dimensions.height())) * sy;
if (!isNear(source_width_scale, 1.0) || !isNear(source_height_scale, 1.0)) {
transform.scale(source_width_scale, source_height_scale);
}
diff --git a/src/Clip.h b/src/Clip.h
index 072cb0c4e..c1150c3a7 100644
--- a/src/Clip.h
+++ b/src/Clip.h
@@ -145,6 +145,12 @@ namespace openshot {
/// Apply keyframes to an openshot::Frame and use an existing background frame (if any)
void apply_keyframes(std::shared_ptr frame, QSize timeline_size);
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ /// Transform and composite onto the background in one Vulkan pass. Returns false
+ /// when an effect needs the transformed layer by itself, or the pass is unavailable.
+ bool apply_vulkan_layer(std::shared_ptr frame, std::shared_ptr background_frame, QSize timeline_size);
+#endif
+
/// Apply waveform image to an openshot::Frame and use an existing background frame (if any)
void apply_waveform(std::shared_ptr frame, QSize timeline_size);
diff --git a/src/ColorManagement.cpp b/src/ColorManagement.cpp
new file mode 100644
index 000000000..cfb441627
--- /dev/null
+++ b/src/ColorManagement.cpp
@@ -0,0 +1,255 @@
+// Copyright (c) 2026 OpenShot Studios, LLC
+// SPDX-License-Identifier: LGPL-3.0-or-later
+#include "ColorManagement.h"
+
+#include
+#include
+#include
+
+extern "C" {
+#include
+#include
+#include
+}
+
+namespace openshot {
+namespace color {
+namespace {
+
+// BT.2100 PQ (SMPTE ST 2084)
+constexpr double kPqM1 = 2610.0 / 16384.0, kPqM2 = 2523.0 / 4096.0 * 128.0;
+constexpr double kPqC1 = 3424.0 / 4096.0, kPqC2 = 2413.0 / 4096.0 * 32.0, kPqC3 = 2392.0 / 4096.0 * 32.0;
+// BT.2100 HLG
+constexpr double kHlgA = 0.17883277, kHlgB = 0.28466892, kHlgC = 0.55991073;
+constexpr double kHlgGamma = 1.2; // system gamma at the nominal 1000 nit peak
+// SDR display EOTF (BT.1886), as BT.2408 uses to place SDR within HDR.
+constexpr double kSdrGamma = 2.4;
+
+// Linear BT.2020 RGB to linear BT.709 RGB (BT.2087).
+constexpr float kBt2020To709[9] = {
+ 1.6604910f, -0.5876411f, -0.0728499f,
+ -0.1245505f, 1.1328999f, -0.0083494f,
+ -0.0181508f, -0.1005789f, 1.1187297f};
+constexpr float kLuma2020[3] = {0.2627f, 0.6780f, 0.0593f};
+constexpr float kLuma709[3] = {0.2126f, 0.7152f, 0.0722f};
+
+// Lookups stay small enough for L1 cache. 12-bit signal indices exceed the
+// 10-bit sources' precision (and PQ's 12-bit visibility threshold).
+constexpr int kSignalBits = 12;
+constexpr int kSignalSize = 1 << kSignalBits;
+// Encode lookup indexed by the float's exponent and top 10 mantissa bits,
+// covering linear 2^-24 (below half an 8-bit code) to 1.0.
+constexpr uint32_t kEncodeShift = 13;
+constexpr uint32_t kEncodeBase = (127u - 24u) << (23 - kEncodeShift);
+constexpr int kEncodeSize = int((127u << (23 - kEncodeShift)) - kEncodeBase) + 1;
+
+// Tone scale lookup indexed by the brightest channel's exponent and top 7
+// mantissa bits (under 0.4% apart), from the knee to 64x reference white.
+constexpr uint32_t kToneShift = 16;
+constexpr uint32_t kToneBase = (127u - 1u) << (23 - kToneShift);
+constexpr int kToneSize = int(((127u + 6u) << (23 - kToneShift)) - kToneBase);
+
+inline uint32_t FloatBits(float value) { uint32_t bits; std::memcpy(&bits, &value, sizeof(bits)); return bits; }
+inline int ToneIndex(float brightest) {
+ const int index = int(FloatBits(brightest) >> kToneShift) - int(kToneBase);
+ return std::clamp(index, 0, kToneSize - 1);
+}
+inline int EncodeIndex(float linear) {
+ const int index = int(FloatBits(linear) >> kEncodeShift) - int(kEncodeBase);
+ return std::clamp(index, 0, kEncodeSize - 1);
+}
+
+double PqToNits(double e) {
+ const double p = std::pow(std::clamp(e, 0.0, 1.0), 1.0 / kPqM2);
+ return 10000.0 * std::pow(std::max(p - kPqC1, 0.0) / (kPqC2 - kPqC3 * p), 1.0 / kPqM1);
+}
+double HlgToScene(double e) {
+ e = std::clamp(e, 0.0, 1.0);
+ return e <= 0.5 ? e * e / 3.0 : (std::exp((e - kHlgC) / kHlgA) + kHlgB) / 12.0;
+}
+bool IsBt2020Matrix(AVColorSpace matrix) {
+ return matrix == AVCOL_SPC_BT2020_NCL || matrix == AVCOL_SPC_BT2020_CL;
+}
+} // namespace
+
+AVColorSpace DecodeMatrix(AVColorSpace tagged, int width, int height) {
+ switch (tagged) {
+ case AVCOL_SPC_BT709:
+ case AVCOL_SPC_BT470BG:
+ case AVCOL_SPC_SMPTE170M:
+ case AVCOL_SPC_SMPTE240M:
+ case AVCOL_SPC_FCC:
+ case AVCOL_SPC_BT2020_NCL:
+ case AVCOL_SPC_BT2020_CL:
+ return tagged;
+ default:
+ // Untagged files keep BT.601 at every size: earlier OpenShot exports
+ // and older FFmpeg encodes used it without tagging, and
+ // cameras and phones tag their files.
+ (void) width; (void) height;
+ return AVCOL_SPC_SMPTE170M;
+ }
+}
+
+AVColorSpace EncodeMatrix(int width, int height) {
+ // Tagged; this is also what players assume for untagged files.
+ return (width >= 1280 || height > 576) ? AVCOL_SPC_BT709 : AVCOL_SPC_SMPTE170M;
+}
+
+int SwsColorspace(AVColorSpace matrix) {
+ switch (matrix) {
+ case AVCOL_SPC_BT709: return SWS_CS_ITU709;
+ case AVCOL_SPC_FCC: return SWS_CS_FCC;
+ case AVCOL_SPC_SMPTE240M: return SWS_CS_SMPTE240M;
+ case AVCOL_SPC_BT2020_NCL:
+ case AVCOL_SPC_BT2020_CL: return SWS_CS_BT2020;
+ default: return SWS_CS_ITU601;
+ }
+}
+
+float SourceColor::RelativePeak() const {
+ if (transfer == Transfer::Sdr) return 1.0f;
+ const double nits = transfer == Transfer::HLG ? kDefaultPeakNits : peak_nits;
+ return float(std::clamp(nits / kReferenceWhiteNits, 1.0, 10000.0 / kReferenceWhiteNits));
+}
+
+double FramePeakNits(const AVFrame* frame) {
+ if (!frame) return 0.0;
+ if (const AVFrameSideData* light = av_frame_get_side_data(frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL)) {
+ const auto* cll = reinterpret_cast(light->data);
+ if (cll->MaxCLL > 0) return double(cll->MaxCLL);
+ }
+ if (const AVFrameSideData* display = av_frame_get_side_data(frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA)) {
+ const auto* mastering = reinterpret_cast(display->data);
+ if (mastering->has_luminance && mastering->max_luminance.den > 0) {
+ const double nits = av_q2d(mastering->max_luminance);
+ if (nits > 0.0) return nits;
+ }
+ }
+ return 0.0;
+}
+
+SourceColor DescribeFrame(const AVFrame* frame, int width, int height, double peak_nits) {
+ SourceColor result;
+ const AVColorSpace tagged = frame ? frame->colorspace : AVCOL_SPC_UNSPECIFIED;
+ result.matrix = DecodeMatrix(tagged, width, height);
+ result.full_range = frame && frame->color_range == AVCOL_RANGE_JPEG;
+ if (frame) {
+ if (frame->color_trc == AVCOL_TRC_SMPTE2084) result.transfer = Transfer::PQ;
+ else if (frame->color_trc == AVCOL_TRC_ARIB_STD_B67) result.transfer = Transfer::HLG;
+ // Unspecified primaries follow a BT.2020 matrix, as players do.
+ result.bt2020 = frame->color_primaries == AVCOL_PRI_BT2020 ||
+ (frame->color_primaries == AVCOL_PRI_UNSPECIFIED && IsBt2020Matrix(tagged));
+ }
+ if (peak_nits > 0.0) result.peak_nits = peak_nits;
+ return result;
+}
+
+float ToneMap(float value, float relative_peak) {
+ if (value <= kToneKnee || relative_peak <= 1.0f) return std::min(value, 1.0f);
+ // Linear below the knee; above it an extended-Reinhard shoulder with unit
+ // slope at the knee that reaches 1.0 exactly at the content peak.
+ const float span = 1.0f - kToneKnee;
+ const float t = (value - kToneKnee) / span;
+ const float w = std::max((relative_peak - kToneKnee) / span, 1.0f);
+ const float shoulder = t * (1.0f + t / (w * w)) / (1.0f + t);
+ return std::min(kToneKnee + span * shoulder, 1.0f);
+}
+
+SdrConverter::SdrConverter(const SourceColor& source_color)
+ : source(source_color), peak(source_color.RelativePeak()), linearize(kSignalSize), encode(kEncodeSize) {
+ for (int i = 0; i < kSignalSize; ++i) {
+ // Bucket centre of the 16-bit codes sharing this index.
+ const double e = (i + 0.5) / kSignalSize;
+ double value;
+ switch (source.transfer) {
+ case Transfer::PQ: value = PqToNits(e) / kReferenceWhiteNits; break;
+ case Transfer::HLG: value = HlgToScene(e); break;
+ default: value = std::pow(e, kSdrGamma); break;
+ }
+ linearize[i] = float(value);
+ }
+ if (source.transfer == Transfer::HLG) {
+ // OOTF: display = peak * Ys^(gamma-1) * scene, relative to reference white.
+ hlg_gain.resize(kSignalSize + 1);
+ for (int i = 0; i <= kSignalSize; ++i)
+ hlg_gain[i] = float(kDefaultPeakNits / kReferenceWhiteNits * std::pow(double(i) / kSignalSize, kHlgGamma - 1.0));
+ }
+ tone_scale.resize(kToneSize);
+ for (int i = 0; i < kToneSize; ++i) {
+ const uint32_t bits = ((uint32_t(i) + kToneBase) << kToneShift) | (1u << (kToneShift - 1));
+ float brightest;
+ std::memcpy(&brightest, &bits, sizeof(brightest));
+ tone_scale[i] = brightest > kToneKnee ? ToneMap(brightest, peak) / brightest : 1.0f;
+ }
+ for (int i = 0; i < kEncodeSize; ++i) {
+ // Middle of the floats sharing this index.
+ const uint32_t bits = ((uint32_t(i) + kEncodeBase) << kEncodeShift) | (1u << (kEncodeShift - 1));
+ float linear;
+ std::memcpy(&linear, &bits, sizeof(linear));
+ encode[i] = uint8_t(std::lround(255.0 * std::pow(std::min(double(linear), 1.0), 1.0 / kSdrGamma)));
+ }
+}
+
+void SdrConverter::ToSdr(float rgb[3]) const {
+ if (source.bt2020) {
+ const float r = rgb[0], g = rgb[1], b = rgb[2];
+ rgb[0] = kBt2020To709[0] * r + kBt2020To709[1] * g + kBt2020To709[2] * b;
+ rgb[1] = kBt2020To709[3] * r + kBt2020To709[4] * g + kBt2020To709[5] * b;
+ rgb[2] = kBt2020To709[6] * r + kBt2020To709[7] * g + kBt2020To709[8] * b;
+ }
+ // Out-of-gamut colors clip per channel.
+ for (int c = 0; c < 3; ++c) rgb[c] = std::max(rgb[c], 0.0f);
+ const float brightest = std::max(rgb[0], std::max(rgb[1], rgb[2]));
+ if (brightest > kToneKnee) {
+ // Scale all channels by the max-channel curve, preserving hue.
+ const float scale = ToneMap(brightest, peak) / brightest;
+ for (int c = 0; c < 3; ++c) rgb[c] *= scale;
+ }
+ for (int c = 0; c < 3; ++c) rgb[c] = std::min(rgb[c], 1.0f);
+}
+
+template
+void SdrConverter::ConvertRows(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes,
+ int width, int height) const {
+ const float* luma = Gamut ? kLuma2020 : kLuma709;
+ constexpr int shift = 16 - kSignalBits;
+ #pragma omp parallel for if(int64_t(width) * height >= 65536) schedule(static)
+ for (int y = 0; y < height; ++y) {
+ const uint16_t* in = reinterpret_cast(reinterpret_cast(src) + int64_t(y) * src_stride_bytes);
+ uint8_t* out = dst + int64_t(y) * dst_stride_bytes;
+ for (int x = 0; x < width; ++x, in += 4, out += 4) {
+ float r = linearize[in[0] >> shift], g = linearize[in[1] >> shift], b = linearize[in[2] >> shift];
+ if (Hlg) {
+ const float scene_luma = luma[0] * r + luma[1] * g + luma[2] * b;
+ const float gain = hlg_gain[std::min(kSignalSize, int(scene_luma * kSignalSize + 0.5f))];
+ r *= gain; g *= gain; b *= gain;
+ }
+ if (Gamut) {
+ const float r2 = kBt2020To709[0] * r + kBt2020To709[1] * g + kBt2020To709[2] * b;
+ const float g2 = kBt2020To709[3] * r + kBt2020To709[4] * g + kBt2020To709[5] * b;
+ const float b2 = kBt2020To709[6] * r + kBt2020To709[7] * g + kBt2020To709[8] * b;
+ r = r2; g = g2; b = b2;
+ }
+ // Clip out-of-gamut colors, then scale by the max-channel tone curve.
+ r = std::max(r, 0.0f); g = std::max(g, 0.0f); b = std::max(b, 0.0f);
+ const float scale = tone_scale[ToneIndex(std::max(r, std::max(g, b)))];
+ out[0] = encode[EncodeIndex(r * scale)];
+ out[1] = encode[EncodeIndex(g * scale)];
+ out[2] = encode[EncodeIndex(b * scale)];
+ out[3] = uint8_t(in[3] >> 8);
+ }
+ }
+}
+
+void SdrConverter::Convert(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes,
+ int width, int height) const {
+ const bool hlg = source.transfer == Transfer::HLG;
+ if (hlg && source.bt2020) ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height);
+ else if (hlg) ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height);
+ else if (source.bt2020) ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height);
+ else ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height);
+}
+
+} // namespace color
+} // namespace openshot
diff --git a/src/ColorManagement.h b/src/ColorManagement.h
new file mode 100644
index 000000000..dcd0a63f2
--- /dev/null
+++ b/src/ColorManagement.h
@@ -0,0 +1,87 @@
+// Copyright (c) 2026 OpenShot Studios, LLC
+// SPDX-License-Identifier: LGPL-3.0-or-later
+#ifndef OPENSHOT_COLOR_MANAGEMENT_H
+#define OPENSHOT_COLOR_MANAGEMENT_H
+
+#include
+#include
+
+extern "C" {
+#include
+}
+struct AVFrame;
+
+namespace openshot {
+namespace color {
+
+/// SDR graphics/reference white in HDR signals (ITU-R BT.2408). HDR sources
+/// map this luminance to SDR 100%.
+constexpr double kReferenceWhiteNits = 203.0;
+/// Nominal HLG display peak (BT.2100) and the PQ peak assumed without metadata.
+constexpr double kDefaultPeakNits = 1000.0;
+/// Linear SDR level (1.0 = reference white) above which highlights compress.
+/// Must match TONE_KNEE in src/vulkan/affine.comp.
+constexpr float kToneKnee = 0.8f;
+
+enum class Transfer : int { Sdr = 0, PQ = 1, HLG = 2 };
+
+/// The YUV matrix to decode with: the tagged matrix, or BT.601 when untagged
+/// (how OpenShot exports before 2026 and older FFmpeg encodes were written).
+AVColorSpace DecodeMatrix(AVColorSpace tagged, int width, int height);
+/// The YUV matrix OpenShot encodes with (and tags) for an output size:
+/// BT.709 for HD and larger, BT.601 for SD.
+AVColorSpace EncodeMatrix(int width, int height);
+/// The swscale coefficient table index (SWS_CS_*) for a matrix.
+int SwsColorspace(AVColorSpace matrix);
+
+/// How a decoded source must be converted into OpenShot's SDR BT.709 RGBA.
+struct SourceColor {
+ AVColorSpace matrix = AVCOL_SPC_BT709;
+ bool full_range = false;
+ Transfer transfer = Transfer::Sdr;
+ bool bt2020 = false; ///< BT.2020 primaries: convert gamut to BT.709
+ double peak_nits = kDefaultPeakNits; ///< HDR content peak, for tone mapping
+
+ /// True when transfer or gamut conversion is required (HDR or wide gamut).
+ bool NeedsConversion() const { return transfer != Transfer::Sdr || bt2020; }
+ /// Peak in units of reference white (>= 1); 1 means no tone mapping.
+ float RelativePeak() const;
+};
+
+/// Describe a frame's color. peak_nits is the content peak from stream
+/// metadata (0 = unknown); it is only used for PQ.
+SourceColor DescribeFrame(const AVFrame* frame, int width, int height, double peak_nits = 0.0);
+/// Content peak from a frame's HDR10 static metadata (MaxCLL, then mastering
+/// display maximum), or 0 when absent.
+double FramePeakNits(const AVFrame* frame);
+
+/// CPU conversion of nonlinear R'G'B'A (16-bit, native endian, full range, in
+/// the source's primaries and transfer) into SDR BT.709 RGBA8. Lookup tables
+/// are built once per SourceColor; Convert is thread-safe.
+class SdrConverter {
+public:
+ explicit SdrConverter(const SourceColor& source);
+ const SourceColor& Source() const { return source; }
+ void Convert(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes,
+ int width, int height) const;
+ /// Convert one linear-light RGB value (1.0 = reference white, source
+ /// primaries) to SDR display RGB in [0,1]. Exposed for tests.
+ void ToSdr(float rgb[3]) const;
+private:
+ SourceColor source;
+ float peak;
+ std::vector linearize; ///< 12-bit signal -> linear (relative to reference white)
+ std::vector hlg_gain; ///< HLG OOTF gain by scene luminance
+ std::vector encode; ///< linear (by float exponent/mantissa) -> BT.1886 8-bit code
+ std::vector tone_scale; ///< brightest channel -> tone curve scale
+ template
+ void ConvertRows(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes,
+ int width, int height) const;
+};
+
+/// Tone-map one linear SDR-relative value (max channel) to [0,1].
+float ToneMap(float value, float relative_peak);
+
+} // namespace color
+} // namespace openshot
+#endif
diff --git a/src/CrashHandler.cpp b/src/CrashHandler.cpp
index 153e8ab66..d009d0529 100644
--- a/src/CrashHandler.cpp
+++ b/src/CrashHandler.cpp
@@ -55,6 +55,23 @@ CrashHandler *CrashHandler::Instance()
}
#ifdef __MINGW32__
+// GpuDebug: with OPENSHOT_GPU_DEBUG set, save a small crash dump (thread
+// stacks only) to %USERPROFILE%\.openshot_qt for debugging test builds.
+static void writeMiniDump()
+{
+ const char* profile = getenv("USERPROFILE");
+ if (!getenv("OPENSHOT_GPU_DEBUG") || !profile)
+ return;
+ char path[MAX_PATH];
+ snprintf(path, sizeof(path), "%s\\.openshot_qt\\openshot-crash-%lu.dmp", profile, GetCurrentProcessId());
+ HANDLE file = CreateFileA(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+ if (file == INVALID_HANDLE_VALUE)
+ return;
+ if (MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), file, MiniDumpNormal, NULL, NULL, NULL))
+ fprintf(stderr, "Crash dump saved: %s\n", path);
+ CloseHandle(file);
+}
+
// Windows exception handler
void CrashHandler::abortHandler(int signum)
{
@@ -76,6 +93,7 @@ void CrashHandler::abortHandler(int signum)
// Dump a stack trace.
printStackTrace(stderr, 63);
+ writeMiniDump();
// Quit
exit( signum );
diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp
index 415e1c9af..863d97a7e 100644
--- a/src/EffectBase.cpp
+++ b/src/EffectBase.cpp
@@ -26,6 +26,10 @@
#include "FFmpegReader.h"
#include "QtImageReader.h"
#include "Logger.h"
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ #include "VulkanCompositor.h"
+ #include "VulkanFrameImage.h"
+#endif
#include
#include
#include
@@ -502,6 +506,11 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_
return empty_mask;
}
+ return ResolveReaderMaskImage(target_image->width(),target_image->height(),frame_number);
+}
+
+std::shared_ptr EffectBase::ResolveReaderMaskImage(int width,int height,int64_t frame_number) {
+
if (!mask_reader)
return {};
@@ -515,8 +524,8 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_
if (mask_reader->info.has_single_image &&
cached_single_mask_image &&
- cached_single_mask_width == target_image->width() &&
- cached_single_mask_height == target_image->height()) {
+ cached_single_mask_width == width &&
+ cached_single_mask_height == height) {
source_mask = cached_single_mask_image;
used_cached_scaled = true;
}
@@ -541,16 +550,54 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_
auto scaled_mask = std::make_shared(
source_mask->scaled(
- target_image->width(), target_image->height(),
+ width, height,
Qt::IgnoreAspectRatio, Qt::SmoothTransformation));
if (mask_reader->info.has_single_image) {
cached_single_mask_image = scaled_mask;
- cached_single_mask_width = target_image->width();
- cached_single_mask_height = target_image->height();
+ cached_single_mask_width = width;
+ cached_single_mask_height = height;
}
return scaled_mask;
}
+std::shared_ptr EffectBase::ResolveReaderMaskSurface(int width, int height, int64_t frame_number) {
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (!mask_reader)
+ return {};
+ std::shared_ptr surface;
+ bool still = true;
+ #pragma omp critical (open_effect_mask_reader)
+ {
+ try {
+ if (!mask_reader->IsOpen())
+ mask_reader->Open();
+ // Still images are scaled and uploaded once, then reused.
+ still = mask_reader->info.has_single_image;
+ if (!still) {
+ auto source_frame = mask_reader->GetFrame(MapMaskFrameNumber(frame_number));
+ if (source_frame)
+ surface = source_frame->GetDeferredImage();
+ }
+ } catch (const std::exception& e) {
+ Logger::Instance()->Log(
+ std::string("EffectBase::ResolveReaderMaskSurface unable to read mask frame: ") + e.what());
+ }
+ }
+ if (surface) {
+ // One GPU pass converts decoder formats to RGBA and scales to the target.
+ const QTransform scale = QTransform::fromScale(double(width) / surface->GetWidth(),
+ double(height) / surface->GetHeight());
+ return RenderVulkanImage(surface, nullptr, width, height, scale, 1.0, Qt::transparent, true);
+ }
+ auto image = ResolveReaderMaskImage(width, height, frame_number);
+ if (image && !image->isNull())
+ return UploadVulkanImage(*image);
+#else
+ (void) width; (void) height; (void) frame_number;
+#endif
+ return {};
+}
+
std::shared_ptr EffectBase::TrackedObjectMask(std::shared_ptr target_image, int64_t frame_number) const {
#ifdef USE_OPENCV
if (!target_image || target_image->isNull() || trackedObjects.empty())
@@ -657,29 +704,76 @@ std::shared_ptr EffectBase::ProcessFrame(std::shared_ptrGetImage();
- if (!pre_image || pre_image->isNull())
- return GetFrame(frame, frame_number);
+ const auto blend_cpu = [&](std::shared_ptr original_image, std::shared_ptr output_frame) {
+ if (!output_frame)
+ return output_frame;
+ auto effected_image = output_frame->GetImage();
+ if (!effected_image || effected_image->isNull() ||
+ effected_image->size() != original_image->size())
+ return output_frame;
- const auto original_image = std::make_shared(pre_image->copy());
- auto output_frame = GetFrame(frame, frame_number);
- if (!output_frame)
- return output_frame;
- auto effected_image = output_frame->GetImage();
- if (!effected_image || effected_image->isNull() ||
- effected_image->size() != original_image->size())
- return output_frame;
+ auto mask_image = GetMaskImage(effected_image, frame_number);
+ if (!mask_image || mask_image->isNull())
+ return output_frame;
- auto mask_image = GetMaskImage(effected_image, frame_number);
- if (!mask_image || mask_image->isNull())
+ if (UseCustomMaskBlend(frame_number))
+ ApplyCustomMaskBlend(original_image, effected_image, mask_image, frame_number);
+ else
+ BlendWithMask(original_image, effected_image, mask_image);
return output_frame;
+ };
+
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // GPU frames keep their input surface, and the mask blend stays on the GPU.
+ std::shared_ptr original_surface;
+ const bool drive = UseCustomMaskBlend(frame_number) && MaskDrivesAmount(frame_number);
+ if (VulkanRenderingEnabled() && RendersOnVulkan() && (!UseCustomMaskBlend(frame_number) || drive)) {
+ original_surface = frame->GetDeferredImage();
+ // Clip images are uploaded as the effect would; the effect reuses this upload.
+ if (!original_surface && ParentClip() && frame->has_image_data) {
+ try {
+ original_surface = UploadVulkanImage(*frame->GetImage());
+ } catch (const std::exception& error) {
+ Logger::Instance()->Log(std::string("Vulkan effect mask upload failed: ") + error.what());
+ }
+ }
+ }
+ if (original_surface) {
+ auto output_frame = GetFrame(frame, frame_number);
+ if (!output_frame)
+ return output_frame;
+ const int width = original_surface->GetWidth(), height = original_surface->GetHeight();
+ if (auto effected = output_frame->GetDeferredImage()) {
+ // A resized result has no matching mask area, as on the CPU.
+ if (effected->GetWidth() != width || effected->GetHeight() != height)
+ return output_frame;
+ try {
+ // Generated masks only need the target size, not its pixels.
+ std::shared_ptr mask;
+ if (ResolveMaskSourceEffect()) {
+ auto generated = GetMaskImage(std::make_shared(width, height, QImage::Format_RGBA8888_Premultiplied), frame_number);
+ if (generated && !generated->isNull())
+ mask = UploadVulkanImage(*generated);
+ } else
+ mask = ResolveReaderMaskSurface(width, height, frame_number);
+ if (!mask)
+ return output_frame;
+ output_frame->AddDeferredImage(BlendMaskVulkanImage(original_surface, effected, mask, mask_invert, drive));
+ return output_frame;
+ } catch (const std::exception& error) {
+ Logger::Instance()->Log(std::string("Vulkan effect mask CPU fallback: ") + error.what());
+ }
+ }
+ return blend_cpu(std::make_shared(original_surface->Materialize()), output_frame);
+ }
+#endif
- if (UseCustomMaskBlend(frame_number))
- ApplyCustomMaskBlend(original_image, effected_image, mask_image, frame_number);
- else
- BlendWithMask(original_image, effected_image, mask_image);
+ auto pre_image = frame->GetImage();
+ if (!pre_image || pre_image->isNull())
+ return GetFrame(frame, frame_number);
- return output_frame;
+ const auto original_image = std::make_shared(pre_image->copy());
+ return blend_cpu(original_image, GetFrame(frame, frame_number));
}
/// Parent clip object of this reader (which can be unparented and NULL)
diff --git a/src/EffectBase.h b/src/EffectBase.h
index 9da9c7149..143c417c6 100644
--- a/src/EffectBase.h
+++ b/src/EffectBase.h
@@ -25,6 +25,7 @@
namespace openshot
{
+ class FrameImage; // GPU surface; only produced in Vulkan builds
class ReaderBase;
/**
@@ -93,6 +94,13 @@ namespace openshot
return GetMaskImage(target_image, frame_number);
}
+ /// Resolve reader artwork without requiring CPU pixels from the target frame.
+ std::shared_ptr ResolveReaderMaskImage(int width, int height, int64_t frame_number);
+
+ /// Reader mask as a GPU surface at the target size. Video masks decoded on
+ /// the GPU stay there; other masks are read on the CPU and uploaded.
+ std::shared_ptr ResolveReaderMaskSurface(int width, int height, int64_t frame_number);
+
/// Optional override for effects that need custom mask behavior.
virtual bool UseCustomMaskBlend(int64_t frame_number) const { return false; }
@@ -103,6 +111,13 @@ namespace openshot
/// Optional override for effects that apply mask processing inside GetFrame().
virtual bool HandlesMaskInternally() const { return false; }
+ /// Effects with a Vulkan path keep masked GPU frames on the GPU.
+ virtual bool RendersOnVulkan() const { return false; }
+
+ /// True when the custom mask blend is the squared "drive amount" blend,
+ /// which the GPU applies too.
+ virtual bool MaskDrivesAmount(int64_t frame_number) const { return false; }
+
public:
/// Parent effect (which properties will set this effect properties)
EffectBase* parentEffect;
diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp
index 842bf69fb..55786c205 100644
--- a/src/FFmpegReader.cpp
+++ b/src/FFmpegReader.cpp
@@ -13,6 +13,7 @@
//
// SPDX-License-Identifier: LGPL-3.0-or-later
+#include "GpuMetrics.h"
#include // for std::this_thread::sleep_for
#include // for std::chrono::milliseconds
#include
@@ -24,10 +25,19 @@
#include "FFmpegUtilities.h"
#include "FFmpegColorRange.h"
+#include "ColorManagement.h"
+
+extern "C" {
+#include
+}
#include "PreviewSize.h"
#include "effects/CropHelpers.h"
#include "FFmpegReader.h"
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+#include "GpuDevices.h"
+#include "VulkanFrameImage.h"
+#endif
#include "Exceptions.h"
#include "MemoryTrim.h"
#include "Timeline.h"
@@ -142,6 +152,20 @@ bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64
// Get hardware pix format
static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
{
+#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__)
+ if (ctx->hw_device_ctx && reinterpret_cast(ctx->hw_device_ctx->data)->type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX) {
+ for (auto format = pix_fmts; *format != AV_PIX_FMT_NONE; ++format)
+ if (*format == AV_PIX_FMT_VIDEOTOOLBOX) return *format;
+ return avcodec_default_get_format(ctx, pix_fmts);
+ }
+#endif
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (ctx->hw_device_ctx && reinterpret_cast(ctx->hw_device_ctx->data)->type == AV_HWDEVICE_TYPE_VULKAN) {
+ for (auto format = pix_fmts; *format != AV_PIX_FMT_NONE; ++format)
+ if (*format == AV_PIX_FMT_VULKAN) return *format;
+ return avcodec_default_get_format(ctx, pix_fmts);
+ }
+#endif
const enum AVPixelFormat *p;
// Prefer only the format matching the selected hardware decoder
@@ -247,6 +271,12 @@ void FFmpegReader::Open() {
pFormatCtx = NULL;
{
hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0);
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // GPU builds decode through Vulkan unless the CPU backend is selected
+ // at runtime. Decode failures reopen this reader in software.
+ hw_de_on = !force_sw_decode && VulkanRenderingEnabled();
+ vulkan_processor.reset();
+#endif
hw_decode_failed = false;
hw_decode_error_count = 0;
hw_decode_succeeded = false;
@@ -300,6 +330,17 @@ void FFmpegReader::Open() {
// Get codec and codec context from stream
const AVCodec *pCodec = avcodec_find_decoder(codecId);
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // Full FFmpeg builds prefer libdav1d for AV1, but that decoder cannot
+ // produce Vulkan frames. Keep it as the default for software retries.
+ if (codecId == AV_CODEC_ID_AV1 && hw_de_on && VulkanAv1DecodeSupported()) {
+ const AVCodec* native_av1 = avcodec_find_decoder_by_name("av1");
+ if (native_av1)
+ pCodec = native_av1;
+ }
+#endif
+ if (!pCodec)
+ throw InvalidCodec("A valid video codec could not be found for this file.", path);
AVDictionary *opts = NULL;
int retry_decode_open = 2;
// If hw accel is selected but hardware cannot handle repeat with software decoding
@@ -309,6 +350,21 @@ void FFmpegReader::Open() {
if (hw_de_on && (retry_decode_open==2)) {
// Up to here no decision is made if hardware or software decode
hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ hw_de_supported = 0;
+ for (int index = 0; const AVCodecHWConfig* config = avcodec_get_hw_config(pCodec, index); ++index) {
+ if (config->device_type ==
+#ifdef __APPLE__
+ AV_HWDEVICE_TYPE_VIDEOTOOLBOX &&
+#else
+ AV_HWDEVICE_TYPE_VULKAN &&
+#endif
+ (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) {
+ hw_de_supported = 1;
+ break;
+ }
+ }
+#endif
}
#endif
retry_decode_open = 0;
@@ -316,14 +372,15 @@ void FFmpegReader::Open() {
// Set number of threads equal to number of processors (not to exceed 16)
pCodecCtx->thread_count = std::min(FF_VIDEO_NUM_PROCESSORS, 16);
- if (pCodec == NULL) {
- throw InvalidCodec("A valid video codec could not be found for this file.", path);
- }
-
// Init options
av_dict_set(&opts, "strict", "experimental", 0);
#if USE_HW_ACCEL
if (hw_de_on && hw_de_supported) {
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // Vulkan decode surfaces must not be held by frame-thread workers
+ // while the consumer waits on their timeline semaphores.
+ pCodecCtx->thread_type = FF_THREAD_SLICE;
+#endif
// Open Hardware Acceleration
int i_decoder_hw = 0;
char adapter[256];
@@ -413,9 +470,21 @@ void FFmpegReader::Open() {
Logger::Instance()->AppendDebugMethod("Decode Device not present using default");
}
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+#ifdef __APPLE__
+ hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
+#else
+ hw_de_av_device_type = AV_HWDEVICE_TYPE_VULKAN;
+#endif
+ adapter_ptr = nullptr;
+#endif
hw_device_ctx = NULL;
// Here the first hardware initialisations are made
+#if defined(OPENSHOT_ENABLE_GPU_FRAME) && !defined(__APPLE__)
+ if (CreateSharedVulkanDevice(&hw_device_ctx) >= 0) {
+#else
if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) {
+#endif
const char* hw_name = av_hwdevice_get_type_name(hw_de_av_device_type);
std::string hw_msg = "HW decode active: ";
hw_msg += (hw_name ? hw_name : "unknown");
@@ -453,7 +522,12 @@ void FFmpegReader::Open() {
}
else {
Logger::Instance()->Log("HW decode active: no (falling back to software)");
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ hw_de_supported = 0;
+ pCodecCtx->get_format = avcodec_default_get_format;
+#else
throw InvalidCodec("Hardware device create failed.", path);
+#endif
}
}
#endif // USE_HW_ACCEL
@@ -485,6 +559,9 @@ void FFmpegReader::Open() {
((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
#endif // ENABLE_VAAPI
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx, nullptr);
+#endif
if (constraints) {
if (pCodecCtx->coded_width < constraints->min_width ||
pCodecCtx->coded_height < constraints->min_height ||
@@ -656,10 +733,26 @@ void FFmpegReader::Open() {
}
// Process video stream side data (rotation, spherical metadata, etc)
+ hdr_peak_nits = 0.0;
+ sdr_converter.reset();
for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) {
AVStream* st = pFormatCtx->streams[i];
if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
size_t side_data_size = 0;
+ if (static_cast(i) == videoStream) {
+ // HDR10 static metadata sets the tone mapping peak (MaxCLL, then mastering peak).
+ const uint8_t *light = ffmpeg_stream_get_side_data(
+ st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL, &side_data_size);
+ if (light && side_data_size >= sizeof(AVContentLightMetadata))
+ hdr_peak_nits = reinterpret_cast(light)->MaxCLL;
+ const uint8_t *mastering = ffmpeg_stream_get_side_data(
+ st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, &side_data_size);
+ if (hdr_peak_nits <= 0.0 && mastering && side_data_size >= sizeof(AVMasteringDisplayMetadata)) {
+ const auto *display = reinterpret_cast(mastering);
+ if (display->has_luminance && display->max_luminance.den > 0)
+ hdr_peak_nits = av_q2d(display->max_luminance);
+ }
+ }
const uint8_t *displaymatrix = ffmpeg_stream_get_side_data(
st, AV_PKT_DATA_DISPLAYMATRIX, &side_data_size);
if (displaymatrix &&
@@ -703,6 +796,24 @@ void FFmpegReader::Open() {
const int final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 2);
working_cache.SetMaxBytesFromInfo(working_cache_frames, info.width, info.height, info.sample_rate, info.channels);
final_cache.SetMaxBytesFromInfo(final_cache_frames, info.width, info.height, info.sample_rate, info.channels);
+ if (!info.has_video && info.has_audio && info.sample_rate > 0 && info.channels > 0 && info.fps.ToDouble() > 0.0) {
+ // Audio-only frames carry no image, so a width x height budget is never
+ // reached. The cache would then keep every frame of the file, and since
+ // CacheMemory rescans its frames on each insert, long reads turn quadratic.
+ const int64_t samples_per_frame = static_cast(std::ceil(info.sample_rate / info.fps.ToDouble()));
+ const int64_t audio_frame_bytes = samples_per_frame * info.channels * static_cast(sizeof(float));
+ final_cache.SetMaxBytes(final_cache_frames * audio_frame_bytes);
+ }
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (hw_de_on && hw_de_supported) {
+ // Full-resolution hardware images consume device memory, not RAM.
+ // Bound each reader cache independently of FPS/thread-count heuristics.
+ constexpr int64_t gpu_cache_bytes = 256LL * 1024 * 1024;
+ // Pending video/audio frames cannot be evicted before they are paired.
+ // Keep the existing working-cache budget; bound finalized images below.
+ final_cache.SetMaxBytes(std::min(final_cache.GetMaxBytes(), gpu_cache_bytes));
+ }
+#endif
// Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp.
// This method allows us to shift timestamps to ensure at least 1 stream is starting at zero.
@@ -733,6 +844,9 @@ void FFmpegReader::Close() {
// Mark as "closed"
is_open = false;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ vulkan_processor.reset();
+#endif
// Keep track of most recent packet
AVPacket *recent_packet = packet;
@@ -1252,6 +1366,14 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) {
// Prevent async calls to the remainder of this code
const std::lock_guard lock(getFrameMutex);
+#if USE_HW_ACCEL && defined(OPENSHOT_ENABLE_GPU_FRAME)
+ // A lost GPU device cannot decode; reopen this reader on the CPU.
+ if (hw_de_on && !force_sw_decode && VulkanDeviceLost()) {
+ hw_decode_failed = true;
+ ReopenWithoutHardwareDecode(requested_frame);
+ }
+#endif
+
// Check the cache a 2nd time (due to the potential previous lock)
frame = final_cache.GetFrame(requested_frame);
if (frame) {
@@ -1460,7 +1582,11 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) {
// Copy and return the largest processed frame (assuming it was the last in the video file)
std::shared_ptr f = CreateFrame(largest_frame_processed);
if (frame->has_image_data) {
- f->AddImage(std::make_shared(frame->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = frame->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*frame->GetImage()));
}
// Use solid color (if no image data found)
@@ -1478,10 +1604,18 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) {
std::shared_ptr f = CreateFrame(largest_frame_processed);
if (last_final_video_frame && last_final_video_frame->has_image_data
&& last_final_video_frame->number <= requested_frame) {
- f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*last_final_video_frame->GetImage()));
} else if (last_video_frame && last_video_frame->has_image_data
&& last_video_frame->number <= requested_frame) {
- f->AddImage(std::make_shared(last_video_frame->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*last_video_frame->GetImage()));
} else {
f->AddColor(info.width, info.height, "#000");
}
@@ -1565,8 +1699,16 @@ bool FFmpegReader::GetAVFrame() {
#if USE_HW_ACCEL
// Get the format from the variables set in get_hw_dec_format
+#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__)
+ hw_de_av_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX;
+ hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
+#elif defined(OPENSHOT_ENABLE_GPU_FRAME)
+ hw_de_av_pix_fmt = AV_PIX_FMT_VULKAN;
+ hw_de_av_device_type = AV_HWDEVICE_TYPE_VULKAN;
+#else
hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
hw_de_av_device_type = hw_de_av_device_type_global;
+#endif
#endif // USE_HW_ACCEL
if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) {
Logger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts);
@@ -1634,6 +1776,15 @@ bool FFmpegReader::GetAVFrame() {
#if USE_HW_ACCEL
if (hw_de_on && hw_de_supported) {
int err;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if ((next_frame2->format == AV_PIX_FMT_VULKAN
+#ifdef __APPLE__
+ || next_frame2->format == AV_PIX_FMT_VIDEOTOOLBOX
+#endif
+ )) {
+ decoded_frame = next_frame2;
+ } else
+#endif
if (next_frame2->format == hw_de_av_pix_fmt) {
if ((err = av_hwframe_transfer_data(next_frame, next_frame2, 0)) < 0) {
Logger::Instance()->AppendDebugMethod(
@@ -1672,7 +1823,11 @@ bool FFmpegReader::GetAVFrame() {
decoded_frame = next_frame2;
}
- if (!decoded_frame->data[0]) {
+ if (!decoded_frame->data[0]
+#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__)
+ && !(decoded_frame->format == AV_PIX_FMT_VIDEOTOOLBOX && decoded_frame->data[3])
+#endif
+ ) {
Logger::Instance()->AppendDebugMethod(
"FFmpegReader::GetAVFrame (Decoded frame missing image data)",
"format", decoded_frame->format,
@@ -1688,10 +1843,38 @@ bool FFmpegReader::GetAVFrame() {
hw_decode_error_count = 0;
#if USE_HW_ACCEL
if (hw_de_on && hw_de_supported && !force_sw_decode) {
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ const bool previously_decoded_on_gpu = hw_decode_succeeded;
+ hw_decode_succeeded = hw_decode_succeeded || (next_frame2->format == AV_PIX_FMT_VULKAN
+#ifdef __APPLE__
+ || next_frame2->format == AV_PIX_FMT_VIDEOTOOLBOX
+#endif
+ );
+ if (!previously_decoded_on_gpu && hw_decode_succeeded)
+ Logger::Instance()->Log("GPU decode confirmed: " + info.vcodec + "; " +
+ (next_frame2->format == AV_PIX_FMT_VULKAN ? "Vulkan" : "VideoToolbox") + "; " + path,
+ Logger::LevelInfo);
+#else
hw_decode_succeeded = true;
+#endif
}
#endif
+ const auto* metrics_format = av_pix_fmt_desc_get(static_cast(next_frame2->format));
+ if (!info.has_single_image)
+ gpu_metrics::Record(metrics_format && (metrics_format->flags & AV_PIX_FMT_FLAG_HWACCEL)
+ ? gpu_metrics::DecodeGpu : gpu_metrics::DecodeCpu);
packet_status.video_decoded++;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // Cover art and still images are decoded on the CPU by design.
+ if (packet_status.video_decoded == 1 && !hw_decode_succeeded && !info.has_single_image)
+ Logger::Instance()->Log("GPU decode not observed: CPU video frame received; codec=" +
+ info.vcodec + "; " + path, Logger::LevelInfo);
+#endif
+
+ // HDR10 metadata may only arrive with frames. Keep the first value so
+ // tone mapping does not change from frame to frame.
+ if (hdr_peak_nits <= 0.0)
+ hdr_peak_nits = color::FramePeakNits(decoded_frame);
// Allocate image (align 32 for simd)
AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format);
@@ -1699,6 +1882,16 @@ bool FFmpegReader::GetAVFrame() {
decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format);
const int decoded_width = decoded_frame->width > 0 ? decoded_frame->width : info.width;
const int decoded_height = decoded_frame->height > 0 ? decoded_frame->height : info.height;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (decoded_pix_fmt == AV_PIX_FMT_VULKAN
+#ifdef __APPLE__
+ || decoded_pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX
+#endif
+ ) {
+ if (av_frame_ref(pFrame, decoded_frame) < 0)
+ throw OutOfMemory("Failed to retain Vulkan frame", path);
+ } else {
+#endif
if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, decoded_width, decoded_height) <= 0) {
throw OutOfMemory("Failed to allocate image buffer", path);
}
@@ -1712,6 +1905,9 @@ bool FFmpegReader::GetAVFrame() {
pFrame->color_primaries = decoded_frame->color_primaries;
pFrame->color_trc = decoded_frame->color_trc;
pFrame->chroma_location = decoded_frame->chroma_location;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ }
+#endif
// Get display PTS from video frame, often different than packet->pts.
// Sending packets to the decoder (i.e. packet->pts) is async,
@@ -2018,6 +2214,39 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
}
}
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (pFrame && (pFrame->format == AV_PIX_FMT_VULKAN
+#ifdef __APPLE__
+ || pFrame->format == AV_PIX_FMT_VIDEOTOOLBOX
+#endif
+ )) {
+ try {
+ if (!vulkan_processor) vulkan_processor = std::make_shared();
+ auto f = CreateFrame(current_frame);
+ f->AddDeferredImage(vulkan_processor->Process(pFrame, width, height,
+ color::DescribeFrame(pFrame, src_width, src_height, hdr_peak_nits),
+ openshot::Settings::Instance()->HIGH_QUALITY_SCALING));
+ gpu_metrics::Record(gpu_metrics::RenderGpu);
+ if (ApplyOrientationMetadata() && std::fabs(source_rotation) >= 0.0001)
+ Logger::Instance()->AppendDebugMethod("Vulkan reader: source orientation requires scaled CPU readback");
+ ApplyFrameOrientation(f);
+ if (first_video_frame == 0 && !is_seeking) first_video_frame = current_frame;
+ working_cache.Add(f);
+ last_video_frame = f;
+ RemoveAVFrame(pFrame);
+ pFrame = nullptr;
+ video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds;
+ return;
+ } catch (const std::exception& error) {
+ Logger::Instance()->Log(std::string("Vulkan processing failed; retrying software decode: ") + error.what(), Logger::LevelWarning);
+ hw_decode_failed = true;
+ RemoveAVFrame(pFrame);
+ pFrame = nullptr;
+ return;
+ }
+ }
+#endif
+
// Determine required buffer size and allocate buffer
const int bytes_per_pixel = 4;
int raw_buffer_size = (width * height * bytes_per_pixel) + 128;
@@ -2027,21 +2256,36 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
int buffer_size = ((raw_buffer_size + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT;
buffer = (unsigned char*) aligned_malloc(buffer_size, ALIGNMENT);
- // Copy picture data from one AVFrame (or AVPicture) to another one.
- AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
+ // HDR (PQ/HLG) and BT.2020 sources keep 16 bits through scaling, then
+ // convert to SDR BT.709. Other sources convert straight to RGBA8.
+ const color::SourceColor source_color = color::DescribeFrame(pFrame, src_width, src_height, hdr_peak_nits);
+ const bool convert_color = source_color.NeedsConversion();
+ const AVPixelFormat rgb_pix_fmt = convert_color ? AV_PIX_FMT_RGBA64 : PIX_FMT_RGBA;
+ if (convert_color) {
+ hdr_buffer.resize(size_t(width) * height * 4);
+ av_image_fill_arrays(pFrameRGB->data, pFrameRGB->linesize,
+ reinterpret_cast(hdr_buffer.data()), AV_PIX_FMT_RGBA64, width, height, 1);
+ const auto* previous = sdr_converter ? &sdr_converter->Source() : nullptr;
+ if (!previous || previous->transfer != source_color.transfer || previous->bt2020 != source_color.bt2020 ||
+ previous->RelativePeak() != source_color.RelativePeak())
+ sdr_converter = std::make_shared(source_color);
+ } else {
+ // Copy picture data from one AVFrame (or AVPicture) to another one.
+ AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height);
+ }
int scale_mode = SWS_FAST_BILINEAR;
if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) {
scale_mode = SWS_BICUBIC;
}
- img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL);
+ img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, rgb_pix_fmt, scale_mode, NULL, NULL, NULL);
if (!img_convert_ctx)
throw OutOfMemory("Failed to initialize sws context", path);
- const int *src_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
- const int *dst_coeff = sws_getCoefficients(SWS_CS_DEFAULT);
+ // Decode with the source's tagged YUV matrix (BT.601 when untagged).
+ const int *coefficients = sws_getCoefficients(color::SwsColorspace(source_color.matrix));
const int dst_full_range = 1; // RGB outputs are full-range
- sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0,
- dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16);
+ sws_setColorspaceDetails(img_convert_ctx, coefficients, src_full_range ? 1 : 0,
+ coefficients, dst_full_range, 0, 1 << 16, 1 << 16);
if (!pFrame || !pFrame->data[0] || pFrame->linesize[0] <= 0) {
#if USE_HW_ACCEL
@@ -2064,6 +2308,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
}
// Resize / Convert to RGB
+ gpu_metrics::Record(gpu_metrics::RenderCpu);
const int scaled_lines = sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0,
original_height, pFrameRGB->data, pFrameRGB->linesize);
if (scaled_lines <= 0) {
@@ -2087,6 +2332,9 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) {
return;
}
+ if (convert_color)
+ sdr_converter->Convert(hdr_buffer.data(), pFrameRGB->linesize[0], buffer, width * bytes_per_pixel, width, height);
+
// Create or get the existing frame object
std::shared_ptr f = CreateFrame(current_frame);
@@ -2821,7 +3069,11 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
// visual order, especially when decode/prefetch is out-of-order.
std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1);
if (previous_frame_instance && previous_frame_instance->has_image_data) {
- f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*previous_frame_instance->GetImage()));
}
// Fall back to last finalized timeline image (survives cache churn).
@@ -2829,7 +3081,11 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
&& last_final_video_frame
&& last_final_video_frame->has_image_data
&& last_final_video_frame->number <= f->number) {
- f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*last_final_video_frame->GetImage()));
}
// Fall back to the last decoded image only when it is not from the future.
@@ -2837,7 +3093,11 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
&& last_video_frame
&& last_video_frame->has_image_data
&& last_video_frame->number <= f->number) {
- f->AddImage(std::make_shared(last_video_frame->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*last_video_frame->GetImage()));
}
// Last-resort fallback if no prior image is available.
@@ -2850,8 +3110,10 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
}
}
+ // Every frame waiting for its audio keeps a decoded image, so keep this margin
+ // small: 0.25 sec covers partial and delayed audio packets.
double audio_pts_diff = audio_pts_seconds - frame_pts_seconds;
- if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 1.0)
+ if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 0.25)
|| (recent_pts_diff > 1.5)
|| packet_status.audio_eof || packet_status.end_of_file) {
// Audio stream is past this frame (so it must be done)
@@ -2886,21 +3148,35 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
if (info.has_single_image) {
// For still-image video (including attached cover art), reuse the most
// recent image so playback does not stall waiting for video EOF.
+ // QImage copies share pixels until written, and keep the cache key
+ // that lets the GPU reuse one upload instead of one per frame.
std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1);
if (previous_frame_instance && previous_frame_instance->has_image_data) {
- f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*previous_frame_instance->GetImage()));
}
if (!f->has_image_data
&& last_final_video_frame
&& last_final_video_frame->has_image_data
&& last_final_video_frame->number <= f->number) {
- f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*last_final_video_frame->GetImage()));
}
if (!f->has_image_data
&& last_video_frame
&& last_video_frame->has_image_data
&& last_video_frame->number <= f->number) {
- f->AddImage(std::make_shared(last_video_frame->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*last_video_frame->GetImage()));
}
}
@@ -2910,19 +3186,31 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) {
if (!f->has_image_data && is_video_ready && is_audio_ready) {
std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1);
if (previous_frame_instance && previous_frame_instance->has_image_data) {
- f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*previous_frame_instance->GetImage()));
}
if (!f->has_image_data
&& last_final_video_frame
&& last_final_video_frame->has_image_data
&& last_final_video_frame->number <= f->number) {
- f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*last_final_video_frame->GetImage()));
}
if (!f->has_image_data
&& last_video_frame
&& last_video_frame->has_image_data
&& last_video_frame->number <= f->number) {
- f->AddImage(std::make_shared(last_video_frame->GetImage()->copy()));
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface);
+ else
+#endif
+ f->AddImage(std::make_shared(*last_video_frame->GetImage()));
}
}
@@ -3049,6 +3337,13 @@ void FFmpegReader::CheckFPS() {
void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) {
// Remove pFrame (if exists)
if (remove_frame) {
+ // Referenced hardware frames release their image through AVBufferRef.
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (remove_frame->hw_frames_ctx) {
+ av_frame_free(&remove_frame);
+ return;
+ }
+#endif
// Free memory
av_freep(&remove_frame->data[0]);
#ifndef WIN32
diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h
index c8f1abfce..b7b918e06 100644
--- a/src/FFmpegReader.h
+++ b/src/FFmpegReader.h
@@ -27,6 +27,7 @@
#include
#include
#include
+#include
#include "AudioLocation.h"
#include "CacheMemory.h"
#include "Clip.h"
@@ -36,6 +37,9 @@
namespace openshot {
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ class VulkanFrameProcessor;
+#endif
/**
* @brief This struct holds the packet counts and end-of-file detection for an openshot::FFmpegReader.
*
@@ -100,9 +104,15 @@ namespace openshot {
* r.Close();
* @endcode
*/
+ namespace color { class SdrConverter; }
+
class FFmpegReader : public ReaderBase {
private:
std::string path;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ std::shared_ptr vulkan_processor;
+ int hw_de_on = 0; // Per-reader state for concurrent Vulkan/CPU readers.
+#endif
AVFormatContext *pFormatCtx;
int videoStream, audioStream;
@@ -166,6 +176,9 @@ namespace openshot {
// Cached conversion contexts and frames for performance
SwsContext *img_convert_ctx = nullptr; ///< Cached video scaler context
+ double hdr_peak_nits = 0.0; ///< HDR10 content peak from metadata (0 = unknown)
+ std::shared_ptr sdr_converter; ///< HDR/wide gamut to SDR (CPU path)
+ std::vector hdr_buffer; ///< 16-bit RGBA scratch for SDR conversion
SWRCONTEXT *avr_ctx = nullptr; ///< Cached audio resample context
AVFrame *pFrameRGB_cached = nullptr; ///< Temporary frame used for video conversion
diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp
index 96e7dbbec..e0b065b7b 100644
--- a/src/FFmpegWriter.cpp
+++ b/src/FFmpegWriter.cpp
@@ -13,6 +13,7 @@
//
// SPDX-License-Identifier: LGPL-3.0-or-later
+#include "GpuMetrics.h"
#include
#include
#include
@@ -27,26 +28,46 @@
#include "FFmpegUtilities.h"
#include "FFmpegWriter.h"
+#include "FFmpegColorRange.h"
+#include "ColorManagement.h"
#include "Exceptions.h"
#include "Frame.h"
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+#include "GpuDevices.h"
+#include "VulkanFrameImage.h"
+#include "VulkanCompositor.h"
+#include "VulkanEncodeConverter.h"
+#ifdef __APPLE__
+#include "MacVulkanInterop.h"
+#endif
+#endif
#include "OpenMPUtilities.h"
#include "Settings.h"
#include "Logger.h"
using namespace openshot;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+namespace {
+bool IsVulkanEncoder(const std::string& name) {
+ return name == "h264_vulkan" || name == "hevc_vulkan" || name == "av1_vulkan";
+}
+std::string SoftwareEncoder(const std::string& name) {
+ const std::vector candidates = name == "h264_vulkan" ? std::vector{"libx264", "libopenh264"}
+ : name == "hevc_vulkan" ? std::vector{"libx265"}
+ : std::vector{"libsvtav1", "libaom-av1", "librav1e"};
+ for (const auto& candidate : candidates)
+ if (avcodec_find_encoder_by_name(candidate.c_str())) return candidate;
+ return {};
+}
+}
+#endif
+
// Multiplexer parameters temporary storage
AVDictionary *mux_dict = NULL;
#if USE_HW_ACCEL
-int hw_en_on = 1; // Is set in UI
-int hw_en_supported = 0; // Is set by FFmpegWriter
-AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE;
-AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI;
-static AVBufferRef *hw_device_ctx = NULL;
-AVFrame *hw_frame = NULL;
-
-static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height)
+static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height, AVPixelFormat format)
{
AVBufferRef *hw_frames_ref;
AVHWFramesContext *frames_ctx = NULL;
@@ -57,7 +78,7 @@ static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int6
return -1;
}
frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
- frames_ctx->format = hw_en_av_pix_fmt;
+ frames_ctx->format = format;
frames_ctx->sw_format = AV_PIX_FMT_NV12;
frames_ctx->width = width;
frames_ctx->height = height;
@@ -101,14 +122,44 @@ void FFmpegWriter::Open() {
if (!is_open) {
// Open the writer
is_open = true;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ gpu_export_failed = false;
+ gpu_export_logged = false;
+ // FFmpeg binds a GPU frame to the first CUDA context it is sent to. Start
+ // each export with fresh pools, so no frame is tied to an old encoder.
+ ReleaseVulkanMemory();
+#endif
// Prepare streams (if needed)
if (!prepare_streams)
PrepareStreams();
// Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers
- if (info.has_video && video_st)
+ if (info.has_video && video_st) {
+#if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL
+ try {
+ open_video(oc, video_st);
+ } catch (const std::exception& error) {
+ // Choose the replacement before the muxer header is written.
+ // Switching encoders after packets are emitted is unsafe.
+ const auto fallback = IsVulkanEncoder(info.vcodec) ? SoftwareEncoder(info.vcodec) : std::string();
+ if (fallback.empty() || write_header) throw;
+ Logger::Instance()->Log("Vulkan encoder CPU fallback: " + std::string(error.what()) + "; using " + fallback, Logger::LevelWarning);
+ av_buffer_unref(&hw_device_ctx);
+ SetVideoOptions(true, fallback, info.fps, info.width, info.height, info.pixel_ratio,
+ info.interlaced_frame, info.top_field_first, info.video_bit_rate);
+ add_video_stream(video_st);
+ open_video(oc, video_st);
+ }
+#else
open_video(oc, video_st);
+#endif
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ Logger::Instance()->Log("GPU export setup: encoder=" + info.vcodec +
+ "; hardware encoding configured=" + (hw_en_on && hw_en_supported ? "yes" : "no") +
+ "; frame interop is checked when frames arrive", Logger::LevelInfo);
+#endif
+ }
if (info.has_audio && audio_st)
open_audio(oc, audio_st);
@@ -167,11 +218,31 @@ void FFmpegWriter::initialize_streams() {
// Set video export options
void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) {
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // Only encoders the probe opened are safe; some drivers and loaders crash.
+ const auto gpu_encoders = IsVulkanEncoder(codec) ? GpuEncoders() : std::vector();
+ if (IsVulkanEncoder(codec) && (!USE_HW_ACCEL || std::find(gpu_encoders.begin(), gpu_encoders.end(), codec) == gpu_encoders.end())) {
+ const auto fallback = SoftwareEncoder(codec);
+ if (!fallback.empty()) {
+ Logger::Instance()->Log("Vulkan encoder unavailable; using CPU encoder " + fallback, Logger::LevelWarning);
+ codec = fallback;
+ }
+ }
+#endif
// Set the video options
if (codec.length() > 0) {
const AVCodec *new_codec;
// Check if the codec selected is a hardware accelerated codec
#if USE_HW_ACCEL
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (IsVulkanEncoder(codec)) {
+ new_codec = avcodec_find_encoder_by_name(codec.c_str());
+ hw_en_on = hw_en_supported = true;
+ hw_en_av_pix_fmt = AV_PIX_FMT_VULKAN;
+ hw_en_av_device_type = AV_HWDEVICE_TYPE_VULKAN;
+ } else
+#endif
+ {
#if defined(__linux__)
if (strstr(codec.c_str(), "_vaapi") != NULL) {
new_codec = avcodec_find_encoder_by_name(codec.c_str());
@@ -223,6 +294,7 @@ void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction f
#else // unknown OS
new_codec = avcodec_find_encoder_by_name(codec.c_str());
#endif //__linux__/_WIN32/__APPLE__
+ }
#else // USE_HW_ACCEL
new_codec = avcodec_find_encoder_by_name(codec.c_str());
#endif // USE_HW_ACCEL
@@ -1068,6 +1140,12 @@ void FFmpegWriter::WriteHeader() {
}
// Add a frame to the queue waiting to be encoded.
+void FFmpegWriter::WriteMeasuredFrame(ReaderBase* reader, int64_t number) {
+ gpu_metrics::Scope scope(true);
+ gpu_metrics::RecordExportFrame();
+ WriteFrame(reader->GetFrame(number));
+}
+
void FFmpegWriter::WriteFrame(std::shared_ptr frame) {
// Check for open reader (or throw exception)
if (!is_open)
@@ -1119,6 +1197,7 @@ void FFmpegWriter::WriteFrameAt(std::shared_ptr frame, int64_t
// Write all frames in the queue to the video file.
void FFmpegWriter::write_frame(std::shared_ptr frame) {
+ gpu_metrics::Reason gpu_reason("Encoder");
// Flip writing flag
is_writing = true;
@@ -1145,6 +1224,9 @@ void FFmpegWriter::write_frame(std::shared_ptr frame) {
}
// Deallocate buffer and AVFrame
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (!frame_final->hw_frames_ctx)
+#endif
av_freep(&(frame_final->data[0]));
AV_FREE_FRAME(&frame_final);
av_frames.erase(frame);
@@ -1466,6 +1548,10 @@ void FFmpegWriter::Close() {
close_video(oc, video_st);
if (audio_st)
close_audio(oc, audio_st);
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ // Free pooled frames still tied to this encoder's CUDA context
+ ReleaseVulkanMemory();
+#endif
// Remove single software scaler
if (img_convert_ctx)
@@ -1640,7 +1726,7 @@ AVStream *FFmpegWriter::add_audio_stream() {
}
// Add a video output stream
-AVStream *FFmpegWriter::add_video_stream() {
+AVStream *FFmpegWriter::add_video_stream(AVStream* existing) {
// Find the video codec
const AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
if (codec == NULL)
@@ -1652,7 +1738,7 @@ AVStream *FFmpegWriter::add_video_stream() {
}
// Create a new video stream
- AVStream* st = avformat_new_stream(oc, codec);
+ AVStream* st = existing ? existing : avformat_new_stream(oc, codec);
if (!st)
throw OutOfMemory("Could not allocate memory for the video stream.", path);
@@ -1672,9 +1758,11 @@ AVStream *FFmpegWriter::add_video_stream() {
c->sample_aspect_ratio.den = info.pixel_ratio.den;
/* Init video encoder options */
+ // VA-API AV1 supports bitrate control; the legacy AV1 quality conversion
+ // below clears the bitrate and uses a qp option this encoder does not expose.
if (info.video_bit_rate >= 1000
#if (LIBAVCODEC_VERSION_MAJOR >= 58)
- && c->codec_id != AV_CODEC_ID_AV1
+ && (c->codec_id != AV_CODEC_ID_AV1 || info.vcodec == "av1_vaapi" || info.vcodec == "av1_vulkan")
#endif
) {
c->bit_rate = info.video_bit_rate;
@@ -1954,7 +2042,27 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
Logger::Instance()->AppendDebugMethod(
"Encode Device not present, using default");
}
- if (av_hwdevice_ctx_create(&hw_device_ctx,
+ // Encode on the card that renders, so VA-API can share its frames. That card
+ // may have no VA-API driver (e.g. NVIDIA in a hybrid laptop); then use the
+ // device chosen above.
+ bool device_created = false;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (hw_en_av_device_type == AV_HWDEVICE_TYPE_VULKAN) {
+ if (!VulkanRenderingEnabled() || CreateSharedVulkanDevice(&hw_device_ctx) < 0)
+ throw InvalidCodec("No Vulkan rendering device is available", path);
+ device_created = true;
+ }
+#endif
+#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__linux__)
+ if (hw_en_av_device_type == AV_HWDEVICE_TYPE_VAAPI && VulkanRenderingEnabled()) {
+ const std::string render_node = ActiveGpuDevice().render_node;
+ device_created = !render_node.empty() &&
+ av_hwdevice_ctx_create(&hw_device_ctx, hw_en_av_device_type, render_node.c_str(), NULL, 0) >= 0;
+ if (!render_node.empty() && !device_created)
+ Logger::Instance()->Log("No VA-API on " + render_node + "; encoding on the default VA-API device", Logger::LevelWarning);
+ }
+#endif
+ if (!device_created && av_hwdevice_ctx_create(&hw_device_ctx,
hw_en_av_device_type, adapter_ptr, NULL, 0) < 0)
{
Logger::Instance()->AppendDebugMethod(
@@ -2006,13 +2114,23 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
}
}
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ const bool vulkan_encoder = hw_en_av_pix_fmt == AV_PIX_FMT_VULKAN;
+#else
+ const bool vulkan_encoder = false; // Older FFmpeg has no AV_PIX_FMT_VULKAN
+#endif
switch (video_codec_ctx->codec_id) {
case AV_CODEC_ID_H264:
video_codec_ctx->max_b_frames = 0; // At least this GPU doesn't support b-frames
- video_codec_ctx->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE;
- av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0);
- av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0);
- av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN);
+ if (!vulkan_encoder) {
+ video_codec_ctx->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE;
+ av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0);
+ if (hw_en_av_pix_fmt == AV_PIX_FMT_CUDA)
+ av_opt_set_int(video_codec_ctx->priv_data, "zerolatency", 1, 0);
+ else
+ av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0);
+ av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN);
+ }
break;
case AV_CODEC_ID_HEVC:
// tested to work with defaults
@@ -2029,7 +2147,7 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
// set hw_frames_ctx for encoder's AVCodecContext
int err;
- if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height)) < 0)
+ if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height, hw_en_av_pix_fmt)) < 0)
{
Logger::Instance()->AppendDebugMethod(
"FFmpegWriter::open_video (set_hwframe_ctx) ERROR faled to set hwframe context",
@@ -2051,6 +2169,24 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
}
#endif
+ // Tag YUV outputs so players decode them as rendered: SDR BT.709 color,
+ // with the matrix players assume for this size (BT.709 for HD, BT.601 for
+ // SD). RGB outputs are left untagged.
+ {
+ const AVPixFmtDescriptor* desc = av_pix_fmt_desc_get(video_codec_ctx->pix_fmt);
+ const bool hardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL);
+ const bool rgb = desc && !hardware && (desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_PAL));
+ const bool gray = desc && !hardware && desc->nb_components <= 2;
+ if (desc && !rgb && !gray) {
+ bool full_range = false;
+ NormalizeDeprecatedPixFmt(video_codec_ctx->pix_fmt, full_range);
+ video_codec_ctx->color_primaries = AVCOL_PRI_BT709;
+ video_codec_ctx->color_trc = AVCOL_TRC_BT709;
+ video_codec_ctx->colorspace = color::EncodeMatrix(info.width, info.height);
+ video_codec_ctx->color_range = full_range ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG;
+ }
+ }
+
/* open the codec */
if (avcodec_open2(video_codec_ctx, codec, &opts) < 0)
throw InvalidCodec("Could not open video codec", path);
@@ -2610,6 +2746,102 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) {
if (src_w == 1 && src_h == 1)
return;
+#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__) && USE_HW_ACCEL
+ if (!gpu_export_failed && hw_en_on && hw_en_supported &&
+ video_codec_ctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX && video_codec_ctx->hw_frames_ctx) {
+ auto surface = frame->GetDeferredImage();
+ if (surface) {
+ AVFrame* encoded = av_frame_alloc();
+ try {
+ if (!encoded) throw std::bad_alloc();
+ auto native = std::dynamic_pointer_cast(surface);
+ const auto* context = native ? reinterpret_cast(native->GetFrame()->hw_frames_ctx->data) : nullptr;
+ if (src_w != info.width || src_h != info.height || !context || context->sw_format != AV_PIX_FMT_RGBA) {
+ QTransform scale;
+ scale.scale(double(info.width) / src_w, double(info.height) / src_h);
+ surface = RenderVulkanImage(surface, nullptr, info.width, info.height, scale, 1.0, Qt::black);
+ }
+ // Render directly into the encoder's own Core Video pixel buffer.
+ // No CPU download, upload, or separate NV12 image copy is needed.
+ const int allocated = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0);
+ if (allocated < 0) throw std::runtime_error("Allocate VideoToolbox encoder frame: " + av_err2string(allocated));
+ std::unique_ptr target(ImportVideoToolboxFrame(encoded, true),
+ [](AVFrame* value) { av_frame_free(&value); });
+ auto converted = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface, target.get()));
+ if (!converted) throw std::runtime_error("No shared VideoToolbox conversion result");
+ // VideoToolbox does not consume Vulkan timeline semaphores. Retire
+ // GPU writes before handing the same IOSurface to its encoder.
+ WaitMacVulkanFrame(converted->GetFrame());
+ const int copied = av_frame_copy_props(encoded, converted->GetFrame());
+ if (copied < 0) throw std::bad_alloc();
+ add_avframe(frame, encoded);
+ encoded = nullptr; // add_avframe owns the retained encoder frame.
+ if (!gpu_export_logged) {
+ gpu_export_logged = true;
+ Logger::Instance()->Log("Vulkan export: encoder-owned VideoToolbox IOSurface (no CPU copy)", Logger::LevelInfo);
+ }
+ return;
+ } catch (const std::exception& error) {
+ av_frame_free(&encoded);
+ gpu_export_failed = true;
+ Logger::Instance()->Log(std::string("VideoToolbox GPU export CPU fallback: ") + error.what(), Logger::LevelWarning);
+ }
+ }
+ }
+#endif
+#if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL
+ // Native Vulkan writes directly into encoder-owned NV12. NVENC retains its
+ // device-to-device transfer path; other encoders use the established CPU path.
+ const bool native_vulkan = video_codec_ctx->pix_fmt == AV_PIX_FMT_VULKAN;
+ if (!gpu_export_failed && hw_en_on && hw_en_supported && video_codec_ctx->hw_frames_ctx &&
+ (native_vulkan || video_codec_ctx->pix_fmt == AV_PIX_FMT_CUDA)) {
+ auto surface = frame->GetDeferredImage();
+ if (surface) {
+ AVFrame* encoded = nullptr;
+ try {
+ auto native = std::dynamic_pointer_cast(surface);
+ const auto* context = native ? reinterpret_cast(native->GetFrame()->hw_frames_ctx->data) : nullptr;
+ if (src_w != info.width || src_h != info.height || !context || context->sw_format != AV_PIX_FMT_RGBA) {
+ QTransform scale;
+ scale.scale(double(info.width) / src_w, double(info.height) / src_h);
+ surface = RenderVulkanImage(surface, nullptr, info.width, info.height, scale, 1.0, Qt::black);
+ }
+ // NVENC keeps its existing separate-plane conversion and GPU
+ // transfer. Native Vulkan writes into the encoder's own frame.
+ std::shared_ptr nv12;
+ if (!native_vulkan)
+ nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface));
+ encoded = av_frame_alloc();
+ if (!encoded) throw std::bad_alloc();
+ const int allocated = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0);
+ if (allocated < 0) throw std::runtime_error("Allocate encoder frame: " + av_err2string(allocated));
+ if (native_vulkan) {
+ nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface, encoded));
+ } else {
+ const int status = av_hwframe_transfer_data(encoded, nv12->GetFrame(), 0);
+ if (status < 0) throw std::runtime_error("Vulkan/CUDA export transfer failed: " + av_err2string(status));
+ }
+
+ av_frame_copy_props(encoded, nv12->GetFrame());
+ add_avframe(frame, encoded);
+ if (!gpu_export_logged) {
+ gpu_export_logged = true;
+ Logger::Instance()->Log(native_vulkan ? "Vulkan export: encoder-owned GPU frames (no copy)"
+ : "Vulkan export: GPU transfer to NVENC", Logger::LevelInfo);
+ }
+ return;
+ } catch (const std::exception& error) {
+ av_frame_free(&encoded);
+ // Interop only touches the temporary scaled/NV12 surfaces. The readback
+ // below waits on the frame surface's semaphores, so it sees finished
+ // pixels; keep this export on the CPU path.
+ gpu_export_failed = true;
+ Logger::Instance()->Log(std::string("Vulkan export CPU fallback: ") + error.what(), Logger::LevelWarning);
+ }
+ }
+ }
+#endif
+
// Point persistent_src_frame->data to RGBA pixels
const uchar* pixels = frame->GetPixels();
if (!persistent_src_frame) {
@@ -2684,6 +2916,13 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) {
);
if (!img_convert_ctx)
throw ErrorEncodingVideo("Could not initialize sws context", -1);
+ // Encode with the matrix and range tagged in open_video().
+ bool dst_full_range = false;
+ NormalizeDeprecatedPixFmt(dst_fmt, dst_full_range);
+ const int* coefficients = sws_getCoefficients(
+ color::SwsColorspace(color::EncodeMatrix(info.width, info.height)));
+ sws_setColorspaceDetails(img_convert_ctx, coefficients, 1, coefficients,
+ dst_full_range ? 1 : 0, 0, 1 << 16, 1 << 16);
}
// Scale RGBA → dst_fmt into persistent_dst_buffer
@@ -2798,21 +3037,25 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra
// Assign the initial AVFrame PTS from the frame counter
frame_final->pts = video_timestamp;
#if USE_HW_ACCEL
- if (hw_en_on && hw_en_supported) {
+ if (hw_en_on && hw_en_supported
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ && !frame_final->hw_frames_ctx
+#endif
+ ) {
if (!(hw_frame = av_frame_alloc())) {
- std::clog << "Error code: av_hwframe_alloc\n";
+ throw OutOfMemory("Could not allocate hardware upload frame", path);
}
if (av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, hw_frame, 0) < 0) {
- std::clog << "Error code: av_hwframe_get_buffer\n";
- }
- if (!hw_frame->hw_frames_ctx) {
- std::clog << "Error hw_frames_ctx.\n";
+ av_frame_free(&hw_frame);
+ throw ErrorEncodingVideo("Could not allocate hardware upload buffer", -1);
}
// Keep the hardware format assigned by av_hwframe_get_buffer().
// NV12 describes the CPU source, not the encoder surface.
if ( av_hwframe_transfer_data(hw_frame, frame_final, 0) < 0) {
- std::clog << "Error while transferring frame data to surface.\n";
+ av_frame_free(&hw_frame);
+ throw ErrorEncodingVideo("Could not upload CPU frame to encoder", -1);
}
+ gpu_metrics::Record(gpu_metrics::Upload);
av_frame_copy_props(hw_frame, frame_final);
}
#endif // USE_HW_ACCEL
@@ -2825,12 +3068,23 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra
#if USE_HW_ACCEL
if (hw_en_on && hw_en_supported) {
- ret = avcodec_send_frame(video_codec_ctx, hw_frame); //hw_frame!!!
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ ret = avcodec_send_frame(video_codec_ctx, frame_final->hw_frames_ctx ? frame_final : hw_frame);
+#else
+ ret = avcodec_send_frame(video_codec_ctx, hw_frame);
+#endif
} else
#endif // USE_HW_ACCEL
{
ret = avcodec_send_frame(video_codec_ctx, frame_final);
}
+ if (ret >= 0) {
+#if USE_HW_ACCEL
+ gpu_metrics::Record(hw_en_on && hw_en_supported ? gpu_metrics::EncodeGpu : gpu_metrics::EncodeCpu);
+#else
+ gpu_metrics::Record(gpu_metrics::EncodeCpu);
+#endif
+ }
error_code = ret;
if (ret < 0 ) {
Logger::Instance()->AppendDebugMethod(
diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h
index 1859cb866..c05c66cb6 100644
--- a/src/FFmpegWriter.h
+++ b/src/FFmpegWriter.h
@@ -136,10 +136,23 @@ namespace openshot {
uint8_t *audio_outbuf;
uint8_t *audio_encoder_buffer;
+#if USE_HW_ACCEL
+ // Hardware encoder ownership belongs to this export, never another writer.
+ bool hw_en_on = false;
+ bool hw_en_supported = false;
+ AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE;
+ AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_NONE;
+ AVBufferRef* hw_device_ctx = nullptr;
+ AVFrame* hw_frame = nullptr;
+#endif
AVFrame *persistent_src_frame = nullptr;
AVFrame *persistent_dst_frame = nullptr;
uint8_t *persistent_dst_buffer = nullptr;
int persistent_dst_size = 0;
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ bool gpu_export_failed = false; ///< Encode remaining frames from CPU pixels
+ bool gpu_export_logged = false; ///< The GPU export path was reported once
+#endif
int audio_outbuf_size;
int audio_input_frame_size;
@@ -163,7 +176,7 @@ namespace openshot {
AVStream *add_audio_stream();
/// Add a video output stream
- AVStream *add_video_stream();
+ AVStream *add_video_stream(AVStream* existing = nullptr);
/// Allocate an AVFrame object
AVFrame *allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer);
@@ -202,6 +215,7 @@ namespace openshot {
void write_frame(std::shared_ptr frame);
public:
+ void WriteMeasuredFrame(ReaderBase* reader, int64_t number);
/// @brief Constructor for FFmpegWriter.
/// Throws an exception on failure to open path.
diff --git a/src/Frame.cpp b/src/Frame.cpp
index 1862a82d0..6447326b2 100644
--- a/src/Frame.cpp
+++ b/src/Frame.cpp
@@ -14,11 +14,13 @@
#include // for std::chrono::milliseconds
#include
#include
+#include
#include "Frame.h"
#include "AudioBufferSource.h"
#include "AudioResampler.h"
#include "QtUtilities.h"
+#include "GpuMetrics.h"
#include
#include
@@ -86,6 +88,13 @@ Frame& Frame::operator= (const Frame& other)
// Copy data and pointers from another Frame instance
void Frame::DeepCopy(const Frame& other)
{
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ if (this == &other)
+ return;
+ std::scoped_lock lock(addingImageMutex, other.addingImageMutex);
+ deferred_image = other.deferred_image;
+ image.reset();
+#endif
number = other.number;
capture_timestamp = other.capture_timestamp;
channels = other.channels;
@@ -381,11 +390,18 @@ juce::AudioBuffer *Frame::GetAudioSampleBuffer()
// Get the size in bytes of this frame (rough estimate)
int64_t Frame::GetBytes()
{
+#ifdef OPENSHOT_ENABLE_GPU_FRAME
+ const std::lock_guard lock(addingImageMutex);
+#endif
int64_t total_bytes = 0;
if (image) {
total_bytes += static_cast